1#!/bin/sh 2 3test_description='exercise basic bitmap functionality' 4GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master 5export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME 6 7. ./test-lib.sh 8. "$TEST_DIRECTORY"/lib-bitmap.sh 9 10# t5310 deals only with single-pack bitmaps, so don't write MIDX bitmaps in 11# their place. 12GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 13 14objpath () { 15 echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')" 16} 17 18# show objects present in pack ($1 should be associated *.idx) 19list_packed_objects () { 20 git show-index <"$1" >object-list && 21 cut -d' ' -f2 object-list 22} 23 24# has_any pattern-file content-file 25# tests whether content-file has any entry from pattern-file with entries being 26# whole lines. 27has_any () { 28 grep -Ff "$1" "$2" 29} 30 31setup_bitmap_history 32 33test_expect_success 'setup writing bitmaps during repack' ' 34 git config repack.writeBitmaps true 35' 36 37test_expect_success 'full repack creates bitmaps' ' 38 GIT_TRACE2_EVENT_NESTING=4 GIT_TRACE2_EVENT="$(pwd)/trace" \ 39 git repack -ad && 40 ls .git/objects/pack/ | grep bitmap >output && 41 test_line_count = 1 output && 42 grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace && 43 grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace 44' 45 46basic_bitmap_tests 47 48test_expect_success 'incremental repack fails when bitmaps are requested' ' 49 test_commit more-1 && 50 test_must_fail git repack -d 2>err && 51 test_i18ngrep "Incremental repacks are incompatible with bitmap" err 52' 53 54test_expect_success 'incremental repack can disable bitmaps' ' 55 test_commit more-2 && 56 git repack -d --no-write-bitmap-index 57' 58 59test_expect_success 'pack-objects respects --local (non-local loose)' ' 60 git init --bare alt.git && 61 echo $(pwd)/alt.git/objects >.git/objects/info/alternates && 62 echo content1 >file1 && 63 # non-local loose object which is not present in bitmapped pack 64 altblob=$(GIT_DIR=alt.git git hash-object -w file1) && 65 # non-local loose object which is also present in bitmapped pack 66 git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin && 67 git add file1 && 68 test_tick && 69 git commit -m commit_file1 && 70 echo HEAD | git pack-objects --local --stdout --revs >1.pack && 71 git index-pack 1.pack && 72 list_packed_objects 1.idx >1.objects && 73 printf "%s\n" "$altblob" "$blob" >nonlocal-loose && 74 ! has_any nonlocal-loose 1.objects 75' 76 77test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' ' 78 echo content2 >file2 && 79 blob2=$(git hash-object -w file2) && 80 git add file2 && 81 test_tick && 82 git commit -m commit_file2 && 83 printf "%s\n" "$blob2" "$bitmaptip" >keepobjects && 84 pack2=$(git pack-objects pack2 <keepobjects) && 85 mv pack2-$pack2.* .git/objects/pack/ && 86 >.git/objects/pack/pack2-$pack2.keep && 87 rm $(objpath $blob2) && 88 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack && 89 git index-pack 2a.pack && 90 list_packed_objects 2a.idx >2a.objects && 91 ! has_any keepobjects 2a.objects 92' 93 94test_expect_success 'pack-objects respects --local (non-local pack)' ' 95 mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ && 96 echo HEAD | git pack-objects --local --stdout --revs >2b.pack && 97 git index-pack 2b.pack && 98 list_packed_objects 2b.idx >2b.objects && 99 ! has_any keepobjects 2b.objects 100' 101 102test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' ' 103 ls .git/objects/pack/ | grep bitmap >output && 104 test_line_count = 1 output && 105 packbitmap=$(basename $(cat output) .bitmap) && 106 list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects && 107 test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" && 108 >.git/objects/pack/$packbitmap.keep && 109 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack && 110 git index-pack 3a.pack && 111 list_packed_objects 3a.idx >3a.objects && 112 ! has_any packbitmap.objects 3a.objects 113' 114 115test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' ' 116 mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ && 117 rm -f .git/objects/pack/multi-pack-index && 118 test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" && 119 echo HEAD | git pack-objects --local --stdout --revs >3b.pack && 120 git index-pack 3b.pack && 121 list_packed_objects 3b.idx >3b.objects && 122 ! has_any packbitmap.objects 3b.objects 123' 124 125test_expect_success 'pack-objects to file can use bitmap' ' 126 # make sure we still have 1 bitmap index from previous tests 127 ls .git/objects/pack/ | grep bitmap >output && 128 test_line_count = 1 output && 129 # verify equivalent packs are generated with/without using bitmap index 130 packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) && 131 packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) && 132 list_packed_objects packa-$packasha1.idx >packa.objects && 133 list_packed_objects packb-$packbsha1.idx >packb.objects && 134 test_cmp packa.objects packb.objects 135' 136 137test_expect_success 'full repack, reusing previous bitmaps' ' 138 git repack -ad && 139 ls .git/objects/pack/ | grep bitmap >output && 140 test_line_count = 1 output 141' 142 143test_expect_success 'fetch (full bitmap)' ' 144 git --git-dir=clone.git fetch origin second:second && 145 git rev-parse HEAD >expect && 146 git --git-dir=clone.git rev-parse HEAD >actual && 147 test_cmp expect actual 148' 149 150test_expect_success 'create objects for missing-HAVE tests' ' 151 blob=$(echo "missing have" | git hash-object -w --stdin) && 152 tree=$(printf "100644 blob $blob\tfile\n" | git mktree) && 153 parent=$(echo parent | git commit-tree $tree) && 154 commit=$(echo commit | git commit-tree $tree -p $parent) && 155 cat >revs <<-EOF 156 HEAD 157 ^HEAD^ 158 ^$commit 159 EOF 160' 161 162test_expect_success 'pack-objects respects --incremental' ' 163 cat >revs2 <<-EOF && 164 HEAD 165 $commit 166 EOF 167 git pack-objects --incremental --stdout --revs <revs2 >4.pack && 168 git index-pack 4.pack && 169 list_packed_objects 4.idx >4.objects && 170 test_line_count = 4 4.objects && 171 git rev-list --objects $commit >revlist && 172 cut -d" " -f1 revlist |sort >objects && 173 test_cmp 4.objects objects 174' 175 176test_expect_success 'pack with missing blob' ' 177 rm $(objpath $blob) && 178 git pack-objects --stdout --revs <revs >/dev/null 179' 180 181test_expect_success 'pack with missing tree' ' 182 rm $(objpath $tree) && 183 git pack-objects --stdout --revs <revs >/dev/null 184' 185 186test_expect_success 'pack with missing parent' ' 187 rm $(objpath $parent) && 188 git pack-objects --stdout --revs <revs >/dev/null 189' 190 191test_expect_success JGIT,SHA1 'we can read jgit bitmaps' ' 192 git clone --bare . compat-jgit.git && 193 ( 194 cd compat-jgit.git && 195 rm -f objects/pack/*.bitmap && 196 jgit gc && 197 git rev-list --test-bitmap HEAD 198 ) 199' 200 201test_expect_success JGIT,SHA1 'jgit can read our bitmaps' ' 202 git clone --bare . compat-us.git && 203 ( 204 cd compat-us.git && 205 git repack -adb && 206 # jgit gc will barf if it does not like our bitmaps 207 jgit gc 208 ) 209' 210 211test_expect_success 'splitting packs does not generate bogus bitmaps' ' 212 test-tool genrandom foo $((1024 * 1024)) >rand && 213 git add rand && 214 git commit -m "commit with big file" && 215 git -c pack.packSizeLimit=500k repack -adb && 216 git init --bare no-bitmaps.git && 217 git -C no-bitmaps.git fetch .. HEAD 218' 219 220test_expect_success 'set up reusable pack' ' 221 rm -f .git/objects/pack/*.keep && 222 git repack -adb && 223 reusable_pack () { 224 git for-each-ref --format="%(objectname)" | 225 git pack-objects --delta-base-offset --revs --stdout "$@" 226 } 227' 228 229test_expect_success 'pack reuse respects --honor-pack-keep' ' 230 test_when_finished "rm -f .git/objects/pack/*.keep" && 231 for i in .git/objects/pack/*.pack 232 do 233 >${i%.pack}.keep 234 done && 235 reusable_pack --honor-pack-keep >empty.pack && 236 git index-pack empty.pack && 237 git show-index <empty.idx >actual && 238 test_must_be_empty actual 239' 240 241test_expect_success 'pack reuse respects --local' ' 242 mv .git/objects/pack/* alt.git/objects/pack/ && 243 test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" && 244 reusable_pack --local >empty.pack && 245 git index-pack empty.pack && 246 git show-index <empty.idx >actual && 247 test_must_be_empty actual 248' 249 250test_expect_success 'pack reuse respects --incremental' ' 251 reusable_pack --incremental >empty.pack && 252 git index-pack empty.pack && 253 git show-index <empty.idx >actual && 254 test_must_be_empty actual 255' 256 257test_expect_success 'truncated bitmap fails gracefully (ewah)' ' 258 test_config pack.writebitmaphashcache false && 259 git repack -ad && 260 git rev-list --use-bitmap-index --count --all >expect && 261 bitmap=$(ls .git/objects/pack/*.bitmap) && 262 test_when_finished "rm -f $bitmap" && 263 test_copy_bytes 256 <$bitmap >$bitmap.tmp && 264 mv -f $bitmap.tmp $bitmap && 265 git rev-list --use-bitmap-index --count --all >actual 2>stderr && 266 test_cmp expect actual && 267 test_i18ngrep corrupt.ewah.bitmap stderr 268' 269 270test_expect_success 'truncated bitmap fails gracefully (cache)' ' 271 git repack -ad && 272 git rev-list --use-bitmap-index --count --all >expect && 273 bitmap=$(ls .git/objects/pack/*.bitmap) && 274 test_when_finished "rm -f $bitmap" && 275 test_copy_bytes 512 <$bitmap >$bitmap.tmp && 276 mv -f $bitmap.tmp $bitmap && 277 git rev-list --use-bitmap-index --count --all >actual 2>stderr && 278 test_cmp expect actual && 279 test_i18ngrep corrupted.bitmap.index stderr 280' 281 282# Create a state of history with these properties: 283# 284# - refs that allow a client to fetch some new history, while sharing some old 285# history with the server; we use branches delta-reuse-old and 286# delta-reuse-new here 287# 288# - the new history contains an object that is stored on the server as a delta 289# against a base that is in the old history 290# 291# - the base object is not immediately reachable from the tip of the old 292# history; finding it would involve digging down through history we know the 293# other side has 294# 295# This should result in a state where fetching from old->new would not 296# traditionally reuse the on-disk delta (because we'd have to dig to realize 297# that the client has it), but we will do so if bitmaps can tell us cheaply 298# that the other side has it. 299test_expect_success 'set up thin delta-reuse parent' ' 300 # This first commit contains the buried base object. 301 test-tool genrandom delta 16384 >file && 302 git add file && 303 git commit -m "delta base" && 304 base=$(git rev-parse --verify HEAD:file) && 305 306 # These intermediate commits bury the base back in history. 307 # This becomes the "old" state. 308 for i in 1 2 3 4 5 309 do 310 echo $i >file && 311 git commit -am "intermediate $i" || return 1 312 done && 313 git branch delta-reuse-old && 314 315 # And now our new history has a delta against the buried base. Note 316 # that this must be smaller than the original file, since pack-objects 317 # prefers to create deltas from smaller objects to larger. 318 test-tool genrandom delta 16300 >file && 319 git commit -am "delta result" && 320 delta=$(git rev-parse --verify HEAD:file) && 321 git branch delta-reuse-new && 322 323 # Repack with bitmaps and double check that we have the expected delta 324 # relationship. 325 git repack -adb && 326 have_delta $delta $base 327' 328 329# Now we can sanity-check the non-bitmap behavior (that the server is not able 330# to reuse the delta). This isn't strictly something we care about, so this 331# test could be scrapped in the future. But it makes sure that the next test is 332# actually triggering the feature we want. 333# 334# Note that our tools for working with on-the-wire "thin" packs are limited. So 335# we actually perform the fetch, retain the resulting pack, and inspect the 336# result. 337test_expect_success 'fetch without bitmaps ignores delta against old base' ' 338 test_config pack.usebitmaps false && 339 test_when_finished "rm -rf client.git" && 340 git init --bare client.git && 341 ( 342 cd client.git && 343 git config transfer.unpackLimit 1 && 344 git fetch .. delta-reuse-old:delta-reuse-old && 345 git fetch .. delta-reuse-new:delta-reuse-new && 346 have_delta $delta $ZERO_OID 347 ) 348' 349 350# And do the same for the bitmap case, where we do expect to find the delta. 351test_expect_success 'fetch with bitmaps can reuse old base' ' 352 test_config pack.usebitmaps true && 353 test_when_finished "rm -rf client.git" && 354 git init --bare client.git && 355 ( 356 cd client.git && 357 git config transfer.unpackLimit 1 && 358 git fetch .. delta-reuse-old:delta-reuse-old && 359 git fetch .. delta-reuse-new:delta-reuse-new && 360 have_delta $delta $base 361 ) 362' 363 364test_expect_success 'pack.preferBitmapTips' ' 365 git init repo && 366 test_when_finished "rm -fr repo" && 367 ( 368 cd repo && 369 370 # create enough commits that not all are receive bitmap 371 # coverage even if they are all at the tip of some reference. 372 test_commit_bulk --message="%s" 103 && 373 374 git rev-list HEAD >commits.raw && 375 sort <commits.raw >commits && 376 377 git log --format="create refs/tags/%s %H" HEAD >refs && 378 git update-ref --stdin <refs && 379 380 git repack -adb && 381 test-tool bitmap list-commits | sort >bitmaps && 382 383 # remember which commits did not receive bitmaps 384 comm -13 bitmaps commits >before && 385 test_file_not_empty before && 386 387 # mark the commits which did not receive bitmaps as preferred, 388 # and generate the bitmap again 389 perl -pe "s{^}{create refs/tags/include/$. }" <before | 390 git update-ref --stdin && 391 git -c pack.preferBitmapTips=refs/tags/include repack -adb && 392 393 # finally, check that the commit(s) without bitmap coverage 394 # are not the same ones as before 395 test-tool bitmap list-commits | sort >bitmaps && 396 comm -13 bitmaps commits >after && 397 398 ! test_cmp before after 399 ) 400' 401 402test_done 403