1#!/bin/sh
2
3test_description='exercise basic bitmap functionality'
4GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
5export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
6
7. ./test-lib.sh
8. "$TEST_DIRECTORY"/lib-bitmap.sh
9
10# t5310 deals only with single-pack bitmaps, so don't write MIDX bitmaps in
11# their place.
12GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
13
14objpath () {
15	echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')"
16}
17
18# show objects present in pack ($1 should be associated *.idx)
19list_packed_objects () {
20	git show-index <"$1" >object-list &&
21	cut -d' ' -f2 object-list
22}
23
24# has_any pattern-file content-file
25# tests whether content-file has any entry from pattern-file with entries being
26# whole lines.
27has_any () {
28	grep -Ff "$1" "$2"
29}
30
31setup_bitmap_history
32
33test_expect_success 'setup writing bitmaps during repack' '
34	git config repack.writeBitmaps true
35'
36
37test_expect_success 'full repack creates bitmaps' '
38	GIT_TRACE2_EVENT_NESTING=4 GIT_TRACE2_EVENT="$(pwd)/trace" \
39		git repack -ad &&
40	ls .git/objects/pack/ | grep bitmap >output &&
41	test_line_count = 1 output &&
42	grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
43	grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
44'
45
46basic_bitmap_tests
47
48test_expect_success 'incremental repack fails when bitmaps are requested' '
49	test_commit more-1 &&
50	test_must_fail git repack -d 2>err &&
51	test_i18ngrep "Incremental repacks are incompatible with bitmap" err
52'
53
54test_expect_success 'incremental repack can disable bitmaps' '
55	test_commit more-2 &&
56	git repack -d --no-write-bitmap-index
57'
58
59test_expect_success 'pack-objects respects --local (non-local loose)' '
60	git init --bare alt.git &&
61	echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
62	echo content1 >file1 &&
63	# non-local loose object which is not present in bitmapped pack
64	altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
65	# non-local loose object which is also present in bitmapped pack
66	git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
67	git add file1 &&
68	test_tick &&
69	git commit -m commit_file1 &&
70	echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
71	git index-pack 1.pack &&
72	list_packed_objects 1.idx >1.objects &&
73	printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
74	! has_any nonlocal-loose 1.objects
75'
76
77test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
78	echo content2 >file2 &&
79	blob2=$(git hash-object -w file2) &&
80	git add file2 &&
81	test_tick &&
82	git commit -m commit_file2 &&
83	printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
84	pack2=$(git pack-objects pack2 <keepobjects) &&
85	mv pack2-$pack2.* .git/objects/pack/ &&
86	>.git/objects/pack/pack2-$pack2.keep &&
87	rm $(objpath $blob2) &&
88	echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
89	git index-pack 2a.pack &&
90	list_packed_objects 2a.idx >2a.objects &&
91	! has_any keepobjects 2a.objects
92'
93
94test_expect_success 'pack-objects respects --local (non-local pack)' '
95	mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
96	echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
97	git index-pack 2b.pack &&
98	list_packed_objects 2b.idx >2b.objects &&
99	! has_any keepobjects 2b.objects
100'
101
102test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
103	ls .git/objects/pack/ | grep bitmap >output &&
104	test_line_count = 1 output &&
105	packbitmap=$(basename $(cat output) .bitmap) &&
106	list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
107	test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
108	>.git/objects/pack/$packbitmap.keep &&
109	echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
110	git index-pack 3a.pack &&
111	list_packed_objects 3a.idx >3a.objects &&
112	! has_any packbitmap.objects 3a.objects
113'
114
115test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
116	mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
117	rm -f .git/objects/pack/multi-pack-index &&
118	test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
119	echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
120	git index-pack 3b.pack &&
121	list_packed_objects 3b.idx >3b.objects &&
122	! has_any packbitmap.objects 3b.objects
123'
124
125test_expect_success 'pack-objects to file can use bitmap' '
126	# make sure we still have 1 bitmap index from previous tests
127	ls .git/objects/pack/ | grep bitmap >output &&
128	test_line_count = 1 output &&
129	# verify equivalent packs are generated with/without using bitmap index
130	packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
131	packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
132	list_packed_objects packa-$packasha1.idx >packa.objects &&
133	list_packed_objects packb-$packbsha1.idx >packb.objects &&
134	test_cmp packa.objects packb.objects
135'
136
137test_expect_success 'full repack, reusing previous bitmaps' '
138	git repack -ad &&
139	ls .git/objects/pack/ | grep bitmap >output &&
140	test_line_count = 1 output
141'
142
143test_expect_success 'fetch (full bitmap)' '
144	git --git-dir=clone.git fetch origin second:second &&
145	git rev-parse HEAD >expect &&
146	git --git-dir=clone.git rev-parse HEAD >actual &&
147	test_cmp expect actual
148'
149
150test_expect_success 'create objects for missing-HAVE tests' '
151	blob=$(echo "missing have" | git hash-object -w --stdin) &&
152	tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
153	parent=$(echo parent | git commit-tree $tree) &&
154	commit=$(echo commit | git commit-tree $tree -p $parent) &&
155	cat >revs <<-EOF
156	HEAD
157	^HEAD^
158	^$commit
159	EOF
160'
161
162test_expect_success 'pack-objects respects --incremental' '
163	cat >revs2 <<-EOF &&
164	HEAD
165	$commit
166	EOF
167	git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
168	git index-pack 4.pack &&
169	list_packed_objects 4.idx >4.objects &&
170	test_line_count = 4 4.objects &&
171	git rev-list --objects $commit >revlist &&
172	cut -d" " -f1 revlist |sort >objects &&
173	test_cmp 4.objects objects
174'
175
176test_expect_success 'pack with missing blob' '
177	rm $(objpath $blob) &&
178	git pack-objects --stdout --revs <revs >/dev/null
179'
180
181test_expect_success 'pack with missing tree' '
182	rm $(objpath $tree) &&
183	git pack-objects --stdout --revs <revs >/dev/null
184'
185
186test_expect_success 'pack with missing parent' '
187	rm $(objpath $parent) &&
188	git pack-objects --stdout --revs <revs >/dev/null
189'
190
191test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
192	git clone --bare . compat-jgit.git &&
193	(
194		cd compat-jgit.git &&
195		rm -f objects/pack/*.bitmap &&
196		jgit gc &&
197		git rev-list --test-bitmap HEAD
198	)
199'
200
201test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
202	git clone --bare . compat-us.git &&
203	(
204		cd compat-us.git &&
205		git repack -adb &&
206		# jgit gc will barf if it does not like our bitmaps
207		jgit gc
208	)
209'
210
211test_expect_success 'splitting packs does not generate bogus bitmaps' '
212	test-tool genrandom foo $((1024 * 1024)) >rand &&
213	git add rand &&
214	git commit -m "commit with big file" &&
215	git -c pack.packSizeLimit=500k repack -adb &&
216	git init --bare no-bitmaps.git &&
217	git -C no-bitmaps.git fetch .. HEAD
218'
219
220test_expect_success 'set up reusable pack' '
221	rm -f .git/objects/pack/*.keep &&
222	git repack -adb &&
223	reusable_pack () {
224		git for-each-ref --format="%(objectname)" |
225		git pack-objects --delta-base-offset --revs --stdout "$@"
226	}
227'
228
229test_expect_success 'pack reuse respects --honor-pack-keep' '
230	test_when_finished "rm -f .git/objects/pack/*.keep" &&
231	for i in .git/objects/pack/*.pack
232	do
233		>${i%.pack}.keep
234	done &&
235	reusable_pack --honor-pack-keep >empty.pack &&
236	git index-pack empty.pack &&
237	git show-index <empty.idx >actual &&
238	test_must_be_empty actual
239'
240
241test_expect_success 'pack reuse respects --local' '
242	mv .git/objects/pack/* alt.git/objects/pack/ &&
243	test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
244	reusable_pack --local >empty.pack &&
245	git index-pack empty.pack &&
246	git show-index <empty.idx >actual &&
247	test_must_be_empty actual
248'
249
250test_expect_success 'pack reuse respects --incremental' '
251	reusable_pack --incremental >empty.pack &&
252	git index-pack empty.pack &&
253	git show-index <empty.idx >actual &&
254	test_must_be_empty actual
255'
256
257test_expect_success 'truncated bitmap fails gracefully (ewah)' '
258	test_config pack.writebitmaphashcache false &&
259	git repack -ad &&
260	git rev-list --use-bitmap-index --count --all >expect &&
261	bitmap=$(ls .git/objects/pack/*.bitmap) &&
262	test_when_finished "rm -f $bitmap" &&
263	test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
264	mv -f $bitmap.tmp $bitmap &&
265	git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
266	test_cmp expect actual &&
267	test_i18ngrep corrupt.ewah.bitmap stderr
268'
269
270test_expect_success 'truncated bitmap fails gracefully (cache)' '
271	git repack -ad &&
272	git rev-list --use-bitmap-index --count --all >expect &&
273	bitmap=$(ls .git/objects/pack/*.bitmap) &&
274	test_when_finished "rm -f $bitmap" &&
275	test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
276	mv -f $bitmap.tmp $bitmap &&
277	git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
278	test_cmp expect actual &&
279	test_i18ngrep corrupted.bitmap.index stderr
280'
281
282# Create a state of history with these properties:
283#
284#  - refs that allow a client to fetch some new history, while sharing some old
285#    history with the server; we use branches delta-reuse-old and
286#    delta-reuse-new here
287#
288#  - the new history contains an object that is stored on the server as a delta
289#    against a base that is in the old history
290#
291#  - the base object is not immediately reachable from the tip of the old
292#    history; finding it would involve digging down through history we know the
293#    other side has
294#
295# This should result in a state where fetching from old->new would not
296# traditionally reuse the on-disk delta (because we'd have to dig to realize
297# that the client has it), but we will do so if bitmaps can tell us cheaply
298# that the other side has it.
299test_expect_success 'set up thin delta-reuse parent' '
300	# This first commit contains the buried base object.
301	test-tool genrandom delta 16384 >file &&
302	git add file &&
303	git commit -m "delta base" &&
304	base=$(git rev-parse --verify HEAD:file) &&
305
306	# These intermediate commits bury the base back in history.
307	# This becomes the "old" state.
308	for i in 1 2 3 4 5
309	do
310		echo $i >file &&
311		git commit -am "intermediate $i" || return 1
312	done &&
313	git branch delta-reuse-old &&
314
315	# And now our new history has a delta against the buried base. Note
316	# that this must be smaller than the original file, since pack-objects
317	# prefers to create deltas from smaller objects to larger.
318	test-tool genrandom delta 16300 >file &&
319	git commit -am "delta result" &&
320	delta=$(git rev-parse --verify HEAD:file) &&
321	git branch delta-reuse-new &&
322
323	# Repack with bitmaps and double check that we have the expected delta
324	# relationship.
325	git repack -adb &&
326	have_delta $delta $base
327'
328
329# Now we can sanity-check the non-bitmap behavior (that the server is not able
330# to reuse the delta). This isn't strictly something we care about, so this
331# test could be scrapped in the future. But it makes sure that the next test is
332# actually triggering the feature we want.
333#
334# Note that our tools for working with on-the-wire "thin" packs are limited. So
335# we actually perform the fetch, retain the resulting pack, and inspect the
336# result.
337test_expect_success 'fetch without bitmaps ignores delta against old base' '
338	test_config pack.usebitmaps false &&
339	test_when_finished "rm -rf client.git" &&
340	git init --bare client.git &&
341	(
342		cd client.git &&
343		git config transfer.unpackLimit 1 &&
344		git fetch .. delta-reuse-old:delta-reuse-old &&
345		git fetch .. delta-reuse-new:delta-reuse-new &&
346		have_delta $delta $ZERO_OID
347	)
348'
349
350# And do the same for the bitmap case, where we do expect to find the delta.
351test_expect_success 'fetch with bitmaps can reuse old base' '
352	test_config pack.usebitmaps true &&
353	test_when_finished "rm -rf client.git" &&
354	git init --bare client.git &&
355	(
356		cd client.git &&
357		git config transfer.unpackLimit 1 &&
358		git fetch .. delta-reuse-old:delta-reuse-old &&
359		git fetch .. delta-reuse-new:delta-reuse-new &&
360		have_delta $delta $base
361	)
362'
363
364test_expect_success 'pack.preferBitmapTips' '
365	git init repo &&
366	test_when_finished "rm -fr repo" &&
367	(
368		cd repo &&
369
370		# create enough commits that not all are receive bitmap
371		# coverage even if they are all at the tip of some reference.
372		test_commit_bulk --message="%s" 103 &&
373
374		git rev-list HEAD >commits.raw &&
375		sort <commits.raw >commits &&
376
377		git log --format="create refs/tags/%s %H" HEAD >refs &&
378		git update-ref --stdin <refs &&
379
380		git repack -adb &&
381		test-tool bitmap list-commits | sort >bitmaps &&
382
383		# remember which commits did not receive bitmaps
384		comm -13 bitmaps commits >before &&
385		test_file_not_empty before &&
386
387		# mark the commits which did not receive bitmaps as preferred,
388		# and generate the bitmap again
389		perl -pe "s{^}{create refs/tags/include/$. }" <before |
390			git update-ref --stdin &&
391		git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
392
393		# finally, check that the commit(s) without bitmap coverage
394		# are not the same ones as before
395		test-tool bitmap list-commits | sort >bitmaps &&
396		comm -13 bitmaps commits >after &&
397
398		! test_cmp before after
399	)
400'
401
402test_done
403