1#!/bin/sh 2 3test_description='git rev-list using object filtering' 4 5. ./test-lib.sh 6 7# Test the blob:none filter. 8 9test_expect_success 'setup r1' ' 10 echo "{print \$1}" >print_1.awk && 11 echo "{print \$2}" >print_2.awk && 12 13 git init r1 && 14 for n in 1 2 3 4 5 15 do 16 echo "This is file: $n" > r1/file.$n 17 git -C r1 add file.$n 18 git -C r1 commit -m "$n" 19 done 20' 21 22test_expect_success 'verify blob:none omits all 5 blobs' ' 23 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \ 24 >ls_files_result && 25 awk -f print_2.awk ls_files_result | 26 sort >expected && 27 28 git -C r1 rev-list --quiet --objects --filter-print-omitted \ 29 --filter=blob:none HEAD >revs && 30 awk -f print_1.awk revs | 31 sed "s/~//" | 32 sort >observed && 33 34 test_cmp expected observed 35' 36 37test_expect_success 'specify blob explicitly prevents filtering' ' 38 file_3=$(git -C r1 ls-files -s file.3 | 39 awk -f print_2.awk) && 40 41 file_4=$(git -C r1 ls-files -s file.4 | 42 awk -f print_2.awk) && 43 44 git -C r1 rev-list --objects --filter=blob:none HEAD $file_3 >observed && 45 grep "$file_3" observed && 46 ! grep "$file_4" observed 47' 48 49test_expect_success 'verify emitted+omitted == all' ' 50 git -C r1 rev-list --objects HEAD >revs && 51 awk -f print_1.awk revs | 52 sort >expected && 53 54 git -C r1 rev-list --objects --filter-print-omitted --filter=blob:none \ 55 HEAD >revs && 56 awk -f print_1.awk revs | 57 sed "s/~//" | 58 sort >observed && 59 60 test_cmp expected observed 61' 62 63 64# Test blob:limit=<n>[kmg] filter. 65# We boundary test around the size parameter. The filter is strictly less than 66# the value, so size 500 and 1000 should have the same results, but 1001 should 67# filter more. 68 69test_expect_success 'setup r2' ' 70 git init r2 && 71 for n in 1000 10000 72 do 73 printf "%"$n"s" X > r2/large.$n 74 git -C r2 add large.$n 75 git -C r2 commit -m "$n" 76 done 77' 78 79test_expect_success 'verify blob:limit=500 omits all blobs' ' 80 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result && 81 awk -f print_2.awk ls_files_result | 82 sort >expected && 83 84 git -C r2 rev-list --quiet --objects --filter-print-omitted \ 85 --filter=blob:limit=500 HEAD >revs && 86 awk -f print_1.awk revs | 87 sed "s/~//" | 88 sort >observed && 89 90 test_cmp expected observed 91' 92 93test_expect_success 'verify emitted+omitted == all' ' 94 git -C r2 rev-list --objects HEAD >revs && 95 awk -f print_1.awk revs | 96 sort >expected && 97 98 git -C r2 rev-list --objects --filter-print-omitted \ 99 --filter=blob:limit=500 HEAD >revs && 100 awk -f print_1.awk revs | 101 sed "s/~//" | 102 sort >observed && 103 104 test_cmp expected observed 105' 106 107test_expect_success 'verify blob:limit=1000' ' 108 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result && 109 awk -f print_2.awk ls_files_result | 110 sort >expected && 111 112 git -C r2 rev-list --quiet --objects --filter-print-omitted \ 113 --filter=blob:limit=1000 HEAD >revs && 114 awk -f print_1.awk revs | 115 sed "s/~//" | 116 sort >observed && 117 118 test_cmp expected observed 119' 120 121test_expect_success 'verify blob:limit=1001' ' 122 git -C r2 ls-files -s large.10000 >ls_files_result && 123 awk -f print_2.awk ls_files_result | 124 sort >expected && 125 126 git -C r2 rev-list --quiet --objects --filter-print-omitted \ 127 --filter=blob:limit=1001 HEAD >revs && 128 awk -f print_1.awk revs | 129 sed "s/~//" | 130 sort >observed && 131 132 test_cmp expected observed 133' 134 135test_expect_success 'verify blob:limit=1k' ' 136 git -C r2 ls-files -s large.10000 >ls_files_result && 137 awk -f print_2.awk ls_files_result | 138 sort >expected && 139 140 git -C r2 rev-list --quiet --objects --filter-print-omitted \ 141 --filter=blob:limit=1k HEAD >revs && 142 awk -f print_1.awk revs | 143 sed "s/~//" | 144 sort >observed && 145 146 test_cmp expected observed 147' 148 149test_expect_success 'verify blob:limit=1m' ' 150 git -C r2 rev-list --quiet --objects --filter-print-omitted \ 151 --filter=blob:limit=1m HEAD >revs && 152 awk -f print_1.awk revs | 153 sed "s/~//" | 154 sort >observed && 155 156 test_must_be_empty observed 157' 158 159# Test sparse:path=<path> filter. 160# !!!! 161# NOTE: sparse:path filter support has been dropped for security reasons, 162# so the tests have been changed to make sure that using it fails. 163# !!!! 164# Use a local file containing a sparse-checkout specification to filter 165# out blobs not required for the corresponding sparse-checkout. We do not 166# require sparse-checkout to actually be enabled. 167 168test_expect_success 'setup r3' ' 169 git init r3 && 170 mkdir r3/dir1 && 171 for n in sparse1 sparse2 172 do 173 echo "This is file: $n" > r3/$n 174 git -C r3 add $n 175 echo "This is file: dir1/$n" > r3/dir1/$n 176 git -C r3 add dir1/$n 177 done && 178 git -C r3 commit -m "sparse" && 179 echo dir1/ >pattern1 && 180 echo sparse1 >pattern2 181' 182 183test_expect_success 'verify sparse:path=pattern1 fails' ' 184 test_must_fail git -C r3 rev-list --quiet --objects \ 185 --filter-print-omitted --filter=sparse:path=../pattern1 HEAD 186' 187 188test_expect_success 'verify sparse:path=pattern2 fails' ' 189 test_must_fail git -C r3 rev-list --quiet --objects \ 190 --filter-print-omitted --filter=sparse:path=../pattern2 HEAD 191' 192 193# Test sparse:oid=<oid-ish> filter. 194# Use a blob containing a sparse-checkout specification to filter 195# out blobs not required for the corresponding sparse-checkout. We do not 196# require sparse-checkout to actually be enabled. 197 198test_expect_success 'setup r3 part 2' ' 199 echo dir1/ >r3/pattern && 200 git -C r3 add pattern && 201 git -C r3 commit -m "pattern" 202' 203 204test_expect_success 'verify sparse:oid=OID omits top-level files' ' 205 git -C r3 ls-files -s pattern sparse1 sparse2 >ls_files_result && 206 awk -f print_2.awk ls_files_result | 207 sort >expected && 208 209 oid=$(git -C r3 ls-files -s pattern | awk -f print_2.awk) && 210 211 git -C r3 rev-list --quiet --objects --filter-print-omitted \ 212 --filter=sparse:oid=$oid HEAD >revs && 213 awk -f print_1.awk revs | 214 sed "s/~//" | 215 sort >observed && 216 217 test_cmp expected observed 218' 219 220test_expect_success 'verify sparse:oid=oid-ish omits top-level files' ' 221 git -C r3 ls-files -s pattern sparse1 sparse2 >ls_files_result && 222 awk -f print_2.awk ls_files_result | 223 sort >expected && 224 225 git -C r3 rev-list --quiet --objects --filter-print-omitted \ 226 --filter=sparse:oid=master:pattern HEAD >revs && 227 awk -f print_1.awk revs | 228 sed "s/~//" | 229 sort >observed && 230 231 test_cmp expected observed 232' 233 234test_expect_success 'rev-list W/ --missing=print and --missing=allow-any for trees' ' 235 TREE=$(git -C r3 rev-parse HEAD:dir1) && 236 237 # Create a spare repo because we will be deleting objects from this one. 238 git clone r3 r3.b && 239 240 rm r3.b/.git/objects/$(echo $TREE | sed "s|^..|&/|") && 241 242 git -C r3.b rev-list --quiet --missing=print --objects HEAD \ 243 >missing_objs 2>rev_list_err && 244 echo "?$TREE" >expected && 245 test_cmp expected missing_objs && 246 247 # do not complain when a missing tree cannot be parsed 248 test_must_be_empty rev_list_err && 249 250 git -C r3.b rev-list --missing=allow-any --objects HEAD \ 251 >objs 2>rev_list_err && 252 ! grep $TREE objs && 253 test_must_be_empty rev_list_err 254' 255 256# Test tree:0 filter. 257 258test_expect_success 'verify tree:0 includes trees in "filtered" output' ' 259 git -C r3 rev-list --quiet --objects --filter-print-omitted \ 260 --filter=tree:0 HEAD >revs && 261 262 awk -f print_1.awk revs | 263 sed s/~// | 264 xargs -n1 git -C r3 cat-file -t >unsorted_filtered_types && 265 266 sort -u unsorted_filtered_types >filtered_types && 267 test_write_lines blob tree >expected && 268 test_cmp expected filtered_types 269' 270 271# Make sure tree:0 does not iterate through any trees. 272 273test_expect_success 'verify skipping tree iteration when not collecting omits' ' 274 GIT_TRACE=1 git -C r3 rev-list \ 275 --objects --filter=tree:0 HEAD 2>filter_trace && 276 grep "Skipping contents of tree [.][.][.]" filter_trace >actual && 277 # One line for each commit traversed. 278 test_line_count = 2 actual && 279 280 # Make sure no other trees were considered besides the root. 281 ! grep "Skipping contents of tree [^.]" filter_trace && 282 283 # Try this again with "combine:". If both sub-filters are skipping 284 # trees, the composite filter should also skip trees. This is not 285 # important unless the user does combine:tree:X+tree:Y or another filter 286 # besides "tree:" is implemented in the future which can skip trees. 287 GIT_TRACE=1 git -C r3 rev-list \ 288 --objects --filter=combine:tree:1+tree:3 HEAD 2>filter_trace && 289 290 # Only skip the dir1/ tree, which is shared between the two commits. 291 grep "Skipping contents of tree " filter_trace >actual && 292 test_write_lines "Skipping contents of tree dir1/..." >expected && 293 test_cmp expected actual 294' 295 296# Test tree:# filters. 297 298expect_has () { 299 commit=$1 && 300 name=$2 && 301 302 hash=$(git -C r3 rev-parse $commit:$name) && 303 grep "^$hash $name$" actual 304} 305 306test_expect_success 'verify tree:1 includes root trees' ' 307 git -C r3 rev-list --objects --filter=tree:1 HEAD >actual && 308 309 # We should get two root directories and two commits. 310 expect_has HEAD "" && 311 expect_has HEAD~1 "" && 312 test_line_count = 4 actual 313' 314 315test_expect_success 'verify tree:2 includes root trees and immediate children' ' 316 git -C r3 rev-list --objects --filter=tree:2 HEAD >actual && 317 318 expect_has HEAD "" && 319 expect_has HEAD~1 "" && 320 expect_has HEAD dir1 && 321 expect_has HEAD pattern && 322 expect_has HEAD sparse1 && 323 expect_has HEAD sparse2 && 324 325 # There are also 2 commit objects 326 test_line_count = 8 actual 327' 328 329test_expect_success 'verify tree:3 includes everything expected' ' 330 git -C r3 rev-list --objects --filter=tree:3 HEAD >actual && 331 332 expect_has HEAD "" && 333 expect_has HEAD~1 "" && 334 expect_has HEAD dir1 && 335 expect_has HEAD dir1/sparse1 && 336 expect_has HEAD dir1/sparse2 && 337 expect_has HEAD pattern && 338 expect_has HEAD sparse1 && 339 expect_has HEAD sparse2 && 340 341 # There are also 2 commit objects 342 test_line_count = 10 actual 343' 344 345test_expect_success 'combine:... for a simple combination' ' 346 git -C r3 rev-list --objects --filter=combine:tree:2+blob:none HEAD \ 347 >actual && 348 349 expect_has HEAD "" && 350 expect_has HEAD~1 "" && 351 expect_has HEAD dir1 && 352 353 # There are also 2 commit objects 354 test_line_count = 5 actual && 355 356 cp actual expected && 357 358 # Try again using repeated --filter - this is equivalent to a manual 359 # combine with "combine:...+..." 360 git -C r3 rev-list --objects --filter=combine:tree:2 \ 361 --filter=blob:none HEAD >actual && 362 363 test_cmp expected actual 364' 365 366test_expect_success 'combine:... with URL encoding' ' 367 git -C r3 rev-list --objects \ 368 --filter=combine:tree%3a2+blob:%6Eon%65 HEAD >actual && 369 370 expect_has HEAD "" && 371 expect_has HEAD~1 "" && 372 expect_has HEAD dir1 && 373 374 # There are also 2 commit objects 375 test_line_count = 5 actual 376' 377 378expect_invalid_filter_spec () { 379 spec="$1" && 380 err="$2" && 381 382 test_must_fail git -C r3 rev-list --objects --filter="$spec" HEAD \ 383 >actual 2>actual_stderr && 384 test_must_be_empty actual && 385 test_i18ngrep "$err" actual_stderr 386} 387 388test_expect_success 'combine:... while URL-encoding things that should not be' ' 389 expect_invalid_filter_spec combine%3Atree:2+blob:none \ 390 "invalid filter-spec" 391' 392 393test_expect_success 'combine: with nothing after the :' ' 394 expect_invalid_filter_spec combine: "expected something after combine:" 395' 396 397test_expect_success 'parse error in first sub-filter in combine:' ' 398 expect_invalid_filter_spec combine:tree:asdf+blob:none \ 399 "expected .tree:<depth>." 400' 401 402test_expect_success 'combine:... with non-encoded reserved chars' ' 403 expect_invalid_filter_spec combine:tree:2+sparse:@xyz \ 404 "must escape char in sub-filter-spec: .@." && 405 expect_invalid_filter_spec combine:tree:2+sparse:\` \ 406 "must escape char in sub-filter-spec: .\`." && 407 expect_invalid_filter_spec combine:tree:2+sparse:~abc \ 408 "must escape char in sub-filter-spec: .\~." 409' 410 411test_expect_success 'validate err msg for "combine:<valid-filter>+"' ' 412 expect_invalid_filter_spec combine:tree:2+ "expected .tree:<depth>." 413' 414 415test_expect_success 'combine:... with edge-case hex digits: Ff Aa 0 9' ' 416 git -C r3 rev-list --objects --filter="combine:tree:2+bl%6Fb:n%6fne" \ 417 HEAD >actual && 418 test_line_count = 5 actual && 419 git -C r3 rev-list --objects --filter="combine:tree%3A2+blob%3anone" \ 420 HEAD >actual && 421 test_line_count = 5 actual && 422 git -C r3 rev-list --objects --filter="combine:tree:%30" HEAD >actual && 423 test_line_count = 2 actual && 424 git -C r3 rev-list --objects --filter="combine:tree:%39+blob:none" \ 425 HEAD >actual && 426 test_line_count = 5 actual 427' 428 429test_expect_success 'add sparse pattern blobs whose paths have reserved chars' ' 430 cp r3/pattern r3/pattern1+renamed% && 431 cp r3/pattern "r3/p;at%ter+n" && 432 cp r3/pattern r3/^~pattern && 433 git -C r3 add pattern1+renamed% "p;at%ter+n" ^~pattern && 434 git -C r3 commit -m "add sparse pattern files with reserved chars" 435' 436 437test_expect_success 'combine:... with more than two sub-filters' ' 438 git -C r3 rev-list --objects \ 439 --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern \ 440 HEAD >actual && 441 442 expect_has HEAD "" && 443 expect_has HEAD~1 "" && 444 expect_has HEAD~2 "" && 445 expect_has HEAD dir1 && 446 expect_has HEAD dir1/sparse1 && 447 expect_has HEAD dir1/sparse2 && 448 449 # Should also have 3 commits 450 test_line_count = 9 actual && 451 452 # Try again, this time making sure the last sub-filter is only 453 # URL-decoded once. 454 cp actual expect && 455 456 git -C r3 rev-list --objects \ 457 --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern1%2brenamed%25 \ 458 HEAD >actual && 459 test_cmp expect actual && 460 461 # Use the same composite filter again, but with a pattern file name that 462 # requires encoding multiple characters, and use implicit filter 463 # combining. 464 test_when_finished "rm -f trace1" && 465 GIT_TRACE=$(pwd)/trace1 git -C r3 rev-list --objects \ 466 --filter=tree:3 --filter=blob:limit=40 \ 467 --filter=sparse:oid="master:p;at%ter+n" \ 468 HEAD >actual && 469 470 test_cmp expect actual && 471 grep "Add to combine filter-spec: sparse:oid=master:p%3bat%25ter%2bn" \ 472 trace1 && 473 474 # Repeat the above test, but this time, the characters to encode are in 475 # the LHS of the combined filter. 476 test_when_finished "rm -f trace2" && 477 GIT_TRACE=$(pwd)/trace2 git -C r3 rev-list --objects \ 478 --filter=sparse:oid=master:^~pattern \ 479 --filter=tree:3 --filter=blob:limit=40 \ 480 HEAD >actual && 481 482 test_cmp expect actual && 483 grep "Add to combine filter-spec: sparse:oid=master:%5e%7epattern" \ 484 trace2 485' 486 487# Test provisional omit collection logic with a repo that has objects appearing 488# at multiple depths - first deeper than the filter's threshold, then shallow. 489 490test_expect_success 'setup r4' ' 491 git init r4 && 492 493 echo foo > r4/foo && 494 mkdir r4/subdir && 495 echo bar > r4/subdir/bar && 496 497 mkdir r4/filt && 498 cp -r r4/foo r4/subdir r4/filt && 499 500 git -C r4 add foo subdir filt && 501 git -C r4 commit -m "commit msg" 502' 503 504expect_has_with_different_name () { 505 repo=$1 && 506 name=$2 && 507 508 hash=$(git -C $repo rev-parse HEAD:$name) && 509 ! grep "^$hash $name$" actual && 510 grep "^$hash " actual && 511 ! grep "~$hash" actual 512} 513 514test_expect_success 'test tree:# filter provisional omit for blob and tree' ' 515 git -C r4 rev-list --objects --filter-print-omitted --filter=tree:2 \ 516 HEAD >actual && 517 expect_has_with_different_name r4 filt/foo && 518 expect_has_with_different_name r4 filt/subdir 519' 520 521test_expect_success 'verify skipping tree iteration when collecting omits' ' 522 GIT_TRACE=1 git -C r4 rev-list --filter-print-omitted \ 523 --objects --filter=tree:0 HEAD 2>filter_trace && 524 grep "^Skipping contents of tree " filter_trace >actual && 525 526 echo "Skipping contents of tree subdir/..." >expect && 527 test_cmp expect actual 528' 529 530test_expect_success 'setup r5' ' 531 git init r5 && 532 mkdir -p r5/subdir && 533 534 echo 1 >r5/short-root && 535 echo 12345 >r5/long-root && 536 echo a >r5/subdir/short-subdir && 537 echo abcde >r5/subdir/long-subdir && 538 539 git -C r5 add short-root long-root subdir && 540 git -C r5 commit -m "commit msg" 541' 542 543test_expect_success 'verify collecting omits in combined: filter' ' 544 # Note that this test guards against the naive implementation of simply 545 # giving both filters the same "omits" set and expecting it to 546 # automatically merge them. 547 git -C r5 rev-list --objects --quiet --filter-print-omitted \ 548 --filter=combine:tree:2+blob:limit=3 HEAD >actual && 549 550 # Expect 0 trees/commits, 3 blobs omitted (all blobs except short-root) 551 omitted_1=$(echo 12345 | git hash-object --stdin) && 552 omitted_2=$(echo a | git hash-object --stdin) && 553 omitted_3=$(echo abcde | git hash-object --stdin) && 554 555 grep ~$omitted_1 actual && 556 grep ~$omitted_2 actual && 557 grep ~$omitted_3 actual && 558 test_line_count = 3 actual 559' 560 561# Test tree:<depth> where a tree is iterated to twice - once where a subentry is 562# too deep to be included, and again where the blob inside it is shallow enough 563# to be included. This makes sure we don't use LOFR_MARK_SEEN incorrectly (we 564# can't use it because a tree can be iterated over again at a lower depth). 565 566test_expect_success 'tree:<depth> where we iterate over tree at two levels' ' 567 git init r5 && 568 569 mkdir -p r5/a/subdir/b && 570 echo foo > r5/a/subdir/b/foo && 571 572 mkdir -p r5/subdir/b && 573 echo foo > r5/subdir/b/foo && 574 575 git -C r5 add a subdir && 576 git -C r5 commit -m "commit msg" && 577 578 git -C r5 rev-list --objects --filter=tree:4 HEAD >actual && 579 expect_has_with_different_name r5 a/subdir/b/foo 580' 581 582test_expect_success 'tree:<depth> which filters out blob but given as arg' ' 583 blob_hash=$(git -C r4 rev-parse HEAD:subdir/bar) && 584 585 git -C r4 rev-list --objects --filter=tree:1 HEAD $blob_hash >actual && 586 grep ^$blob_hash actual 587' 588 589# Delete some loose objects and use rev-list, but WITHOUT any filtering. 590# This models previously omitted objects that we did not receive. 591 592test_expect_success 'rev-list W/ --missing=print' ' 593 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \ 594 >ls_files_result && 595 awk -f print_2.awk ls_files_result | 596 sort >expected && 597 598 for id in `cat expected | sed "s|..|&/|"` 599 do 600 rm r1/.git/objects/$id 601 done && 602 603 git -C r1 rev-list --quiet --missing=print --objects HEAD >revs && 604 awk -f print_1.awk revs | 605 sed "s/?//" | 606 sort >observed && 607 608 test_cmp expected observed 609' 610 611test_expect_success 'rev-list W/O --missing fails' ' 612 test_must_fail git -C r1 rev-list --quiet --objects HEAD 613' 614 615test_expect_success 'rev-list W/ missing=allow-any' ' 616 git -C r1 rev-list --quiet --missing=allow-any --objects HEAD 617' 618 619# Test expansion of filter specs. 620 621test_expect_success 'expand blob limit in protocol' ' 622 git -C r2 config --local uploadpack.allowfilter 1 && 623 GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 clone \ 624 --filter=blob:limit=1k "file://$(pwd)/r2" limit && 625 ! grep "blob:limit=1k" trace && 626 grep "blob:limit=1024" trace 627' 628 629test_done 630