1 /*-
2  * Copyright (c) 2003-2007,2013 Tim Kientzle
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25 #include "test.h"
26 __FBSDID("$FreeBSD$");
27 
28 #include <errno.h>
29 #include <stdlib.h>
30 #include <string.h>
31 
32 /*
33  * This is a somewhat tricky test that verifies the ability to
34  * write and read very large entries to zip archives.
35  *
36  * See test_tar_large.c for more information about the machinery
37  * being used here.
38  */
39 
40 static size_t nullsize;
41 static void *nulldata;
42 
43 struct fileblock {
44 	struct fileblock *next;
45 	int	size;
46 	void *buff;
47 	int64_t gap_size; /* Size of following gap */
48 };
49 
50 struct fileblocks {
51 	int64_t filesize;
52 	int64_t fileposition;
53 	int64_t gap_remaining;
54 	void *buff;
55 	struct fileblock *first;
56 	struct fileblock *current;
57 	struct fileblock *last;
58 };
59 
60 /* The following size definitions simplify things below. */
61 #define KB ((int64_t)1024)
62 #define MB ((int64_t)1024 * KB)
63 #define GB ((int64_t)1024 * MB)
64 #define TB ((int64_t)1024 * GB)
65 
66 static int64_t	memory_read_skip(struct archive *, void *, int64_t request);
67 static ssize_t	memory_read(struct archive *, void *, const void **buff);
68 static ssize_t	memory_write(struct archive *, void *, const void *, size_t);
69 
70 static uint16_t le16(const void *_p) {
71 	const uint8_t *p = _p;
72 	return p[0] | (p[1] << 8);
73 }
74 
75 static uint32_t le32(const void *_p) {
76 	const uint8_t *p = _p;
77 	return le16(p) | ((uint32_t)le16(p + 2) << 16);
78 }
79 
80 static uint64_t le64(const void *_p) {
81 	const uint8_t *p = _p;
82 	return le32(p) | ((uint64_t)le32(p + 4) << 32);
83 }
84 
85 static ssize_t
86 memory_write(struct archive *a, void *_private, const void *buff, size_t size)
87 {
88 	struct fileblocks *private = _private;
89 	struct fileblock *block;
90 
91 	(void)a;
92 
93 	if ((const char *)nulldata <= (const char *)buff
94 	    && (const char *)buff < (const char *)nulldata + nullsize) {
95 		/* We don't need to store a block of gap data. */
96 		private->last->gap_size += (int64_t)size;
97 	} else {
98 		/* Yes, we're assuming the very first write is metadata. */
99 		/* It's header or metadata, copy and save it. */
100 		block = (struct fileblock *)malloc(sizeof(*block));
101 		memset(block, 0, sizeof(*block));
102 		block->size = (int)size;
103 		block->buff = malloc(size);
104 		memcpy(block->buff, buff, size);
105 		if (private->last == NULL) {
106 			private->first = private->last = block;
107 		} else {
108 			private->last->next = block;
109 			private->last = block;
110 		}
111 		block->next = NULL;
112 	}
113 	private->filesize += size;
114 	return ((long)size);
115 }
116 
117 static ssize_t
118 memory_read(struct archive *a, void *_private, const void **buff)
119 {
120 	struct fileblocks *private = _private;
121 	ssize_t size;
122 
123 	(void)a;
124 
125 	while (private->current != NULL && private->buff == NULL && private->gap_remaining == 0) {
126 		private->current = private->current->next;
127 		if (private->current != NULL) {
128 			private->buff = private->current->buff;
129 			private->gap_remaining = private->current->gap_size;
130 		}
131 	}
132 
133 	if (private->current == NULL)
134 		return (0);
135 
136 	/* If there's real data, return that. */
137 	if (private->buff != NULL) {
138 		*buff = private->buff;
139 		size = ((char *)private->current->buff + private->current->size)
140 		    - (char *)private->buff;
141 		private->buff = NULL;
142 		private->fileposition += size;
143 		return (size);
144 	}
145 
146 	/* Big gap: too big to return all at once, so just return some. */
147 	if (private->gap_remaining > (int64_t)nullsize) {
148 		private->gap_remaining -= nullsize;
149 		*buff = nulldata;
150 		private->fileposition += nullsize;
151 		return (nullsize);
152 	}
153 
154 	/* Small gap: finish the gap and prep for next block. */
155 	if (private->gap_remaining > 0) {
156 		size = (ssize_t)private->gap_remaining;
157 		*buff = nulldata;
158 		private->gap_remaining = 0;
159 		private->fileposition += size;
160 
161 		private->current = private->current->next;
162 		if (private->current != NULL) {
163 			private->buff = private->current->buff;
164 			private->gap_remaining = private->current->gap_size;
165 		}
166 
167 		return (size);
168 	}
169 	fprintf(stderr, "\n\n\nInternal failure\n\n\n");
170 	exit(1);
171 }
172 
173 static int
174 memory_read_open(struct archive *a, void *_private)
175 {
176 	struct fileblocks *private = _private;
177 
178 	(void)a; /* UNUSED */
179 
180 	private->current = private->first;
181 	private->fileposition = 0;
182 	if (private->current != NULL) {
183 		private->buff = private->current->buff;
184 		private->gap_remaining = private->current->gap_size;
185 	}
186 	return (ARCHIVE_OK);
187 }
188 
189 static int64_t
190 memory_read_seek(struct archive *a, void *_private, int64_t offset, int whence)
191 {
192 	struct fileblocks *private = _private;
193 
194 	(void)a;
195 	if (whence == SEEK_END) {
196 		offset = private->filesize + offset;
197 	} else if (whence == SEEK_CUR) {
198 		offset = private->fileposition + offset;
199 	}
200 
201 	if (offset < 0) {
202 		fprintf(stderr, "\n\n\nInternal failure: negative seek\n\n\n");
203 		exit(1);
204 	}
205 
206 	/* We've converted the request into a SEEK_SET. */
207 	private->fileposition = offset;
208 
209 	/* Walk the block list to find the new position. */
210 	offset = 0;
211 	private->current = private->first;
212 	while (private->current != NULL) {
213 		if (offset + private->current->size > private->fileposition) {
214 			/* Position is in this block. */
215 			private->buff = (char *)private->current->buff
216 			    + private->fileposition - offset;
217 			private->gap_remaining = private->current->gap_size;
218 			return private->fileposition;
219 		}
220 		offset += private->current->size;
221 		if (offset + private->current->gap_size > private->fileposition) {
222 			/* Position is in this gap. */
223 			private->buff = NULL;
224 			private->gap_remaining = private->current->gap_size
225 			    - (private->fileposition - offset);
226 			return private->fileposition;
227 		}
228 		offset += private->current->gap_size;
229 		/* Skip to next block. */
230 		private->current = private->current->next;
231 	}
232 	if (private->fileposition == private->filesize) {
233 		return private->fileposition;
234 	}
235 	fprintf(stderr, "\n\n\nInternal failure: over-sized seek\n\n\n");
236 	exit(1);
237 }
238 
239 static int64_t
240 memory_read_skip(struct archive *a, void *_private, int64_t skip)
241 {
242 	struct fileblocks *private = _private;
243 	int64_t old_position = private->fileposition;
244 	int64_t new_position = memory_read_seek(a, _private, skip, SEEK_CUR);
245 	return (new_position - old_position);
246 }
247 
248 static struct fileblocks *
249 fileblocks_new(void)
250 {
251 	struct fileblocks *fileblocks;
252 
253 	fileblocks = calloc(1, sizeof(struct fileblocks));
254 	return fileblocks;
255 }
256 
257 static void
258 fileblocks_free(struct fileblocks *fileblocks)
259 {
260 	while (fileblocks->first != NULL) {
261 		struct fileblock *b = fileblocks->first;
262 		fileblocks->first = fileblocks->first->next;
263 		free(b->buff);
264 		free(b);
265 	}
266 	free(fileblocks);
267 }
268 
269 
270 /* The sizes of the entries we're going to generate. */
271 static int64_t test_sizes[] = {
272 	/* Test for 32-bit signed overflow. */
273 	2 * GB - 1, 2 * GB, 2 * GB + 1,
274 	/* Test for 32-bit unsigned overflow. */
275 	4 * GB - 1, 4 * GB, 4 * GB + 1,
276 	/* And beyond ... because we can. */
277 	16 * GB - 1, 16 * GB, 16 * GB + 1,
278 	64 * GB - 1, 64 * GB, 64 * GB + 1,
279 	256 * GB - 1, 256 * GB, 256 * GB + 1,
280 	1 * TB,
281 	0
282 };
283 
284 
285 static void
286 verify_large_zip(struct archive *a, struct fileblocks *fileblocks)
287 {
288 	char namebuff[64];
289 	struct archive_entry *ae;
290 	int i;
291 
292 	assertEqualIntA(a, ARCHIVE_OK,
293 	    archive_read_set_options(a, "zip:ignorecrc32"));
294 	assertEqualIntA(a, ARCHIVE_OK,
295 	    archive_read_set_open_callback(a, memory_read_open));
296 	assertEqualIntA(a, ARCHIVE_OK,
297 	    archive_read_set_read_callback(a, memory_read));
298 	assertEqualIntA(a, ARCHIVE_OK,
299 	    archive_read_set_skip_callback(a, memory_read_skip));
300 	assertEqualIntA(a, ARCHIVE_OK,
301 	    archive_read_set_seek_callback(a, memory_read_seek));
302 	assertEqualIntA(a, ARCHIVE_OK,
303 	    archive_read_set_callback_data(a, fileblocks));
304 	assertEqualIntA(a, ARCHIVE_OK, archive_read_open1(a));
305 
306 	/*
307 	 * Read entries back.
308 	 */
309 	for (i = 0; test_sizes[i] > 0; i++) {
310 		assertEqualIntA(a, ARCHIVE_OK,
311 		    archive_read_next_header(a, &ae));
312 		snprintf(namebuff, sizeof(namebuff), "file_%d", i);
313 		assertEqualString(namebuff, archive_entry_pathname(ae));
314 		assertEqualInt(test_sizes[i], archive_entry_size(ae));
315 	}
316 	assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
317 	assertEqualString("lastfile", archive_entry_pathname(ae));
318 
319 	assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
320 
321 	/* Close out the archive. */
322 	assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
323 }
324 
325 DEFINE_TEST(test_write_format_zip_large)
326 {
327 	int i;
328 	char namebuff[64];
329 	struct fileblocks *fileblocks = fileblocks_new();
330 	struct archive_entry *ae;
331 	struct archive *a;
332 	const char *p;
333 	const char *cd_start, *zip64_eocd, *zip64_locator, *eocd;
334 	int64_t cd_size;
335 	char *buff;
336 	int64_t  filesize;
337 	size_t writesize, buffsize, s;
338 
339 	nullsize = (size_t)(1 * MB);
340 	nulldata = malloc(nullsize);
341 	memset(nulldata, 0xAA, nullsize);
342 
343 	/*
344 	 * Open an archive for writing.
345 	 */
346 	a = archive_write_new();
347 	archive_write_set_format_zip(a);
348 	assertEqualIntA(a, ARCHIVE_OK,
349 	    archive_write_set_options(a, "zip:compression=store"));
350 	assertEqualIntA(a, ARCHIVE_OK,
351 	    archive_write_set_options(a, "zip:fakecrc32"));
352 	assertEqualIntA(a, ARCHIVE_OK,
353 	    archive_write_set_bytes_per_block(a, 0)); /* No buffering. */
354 	assertEqualIntA(a, ARCHIVE_OK,
355 	    archive_write_open(a, fileblocks, NULL, memory_write, NULL));
356 
357 	/*
358 	 * Write a series of large files to it.
359 	 */
360 	for (i = 0; test_sizes[i] != 0; i++) {
361 		assert((ae = archive_entry_new()) != NULL);
362 		snprintf(namebuff, sizeof(namebuff), "file_%d", i);
363 		archive_entry_copy_pathname(ae, namebuff);
364 		archive_entry_set_mode(ae, S_IFREG | 0755);
365 		filesize = test_sizes[i];
366 		archive_entry_set_size(ae, filesize);
367 
368 		assertEqualIntA(a, ARCHIVE_OK,
369 		    archive_write_header(a, ae));
370 		archive_entry_free(ae);
371 
372 		/*
373 		 * Write the actual data to the archive.
374 		 */
375 		while (filesize > 0) {
376 			writesize = nullsize;
377 			if ((int64_t)writesize > filesize)
378 				writesize = (size_t)filesize;
379 			assertEqualIntA(a, (int)writesize,
380 			    (int)archive_write_data(a, nulldata, writesize));
381 			filesize -= writesize;
382 		}
383 	}
384 
385 	assert((ae = archive_entry_new()) != NULL);
386 	archive_entry_copy_pathname(ae, "lastfile");
387 	archive_entry_set_mode(ae, S_IFREG | 0755);
388 	assertA(0 == archive_write_header(a, ae));
389 	archive_entry_free(ae);
390 
391 	/* Close out the archive. */
392 	assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
393 	assertEqualInt(ARCHIVE_OK, archive_write_free(a));
394 
395 	/*
396 	 * Read back with seeking reader:
397 	 */
398 	a = archive_read_new();
399 	assertEqualIntA(a, ARCHIVE_OK,
400 	    archive_read_support_format_zip_seekable(a));
401 	verify_large_zip(a, fileblocks);
402 	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
403 
404 	/*
405 	 * Read back with streaming reader:
406 	 */
407 	a = archive_read_new();
408 	assertEqualIntA(a, ARCHIVE_OK,
409 	    archive_read_support_format_zip_streamable(a));
410 	verify_large_zip(a, fileblocks);
411 	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
412 
413 	/*
414 	 * Manually verify some of the final bytes of the archives.
415 	 */
416 	/* Collect the final bytes together */
417 #define FINAL_SIZE 8192
418 	buff = malloc(FINAL_SIZE);
419 	buffsize = 0;
420 	memory_read_open(NULL, fileblocks);
421 	memory_read_seek(NULL, fileblocks, -FINAL_SIZE, SEEK_END);
422 	while ((s = memory_read(NULL, fileblocks, (const void **)&p)) > 0) {
423 		memcpy(buff + buffsize, p, s);
424 		buffsize += s;
425 	}
426 	assertEqualInt(buffsize, FINAL_SIZE);
427 
428 	p = buff + buffsize;
429 
430 	/* Verify regular end-of-central-directory record */
431 	eocd = p - 22;
432 	assertEqualMem(eocd, "PK\005\006\0\0\0\0", 8);
433 	assertEqualMem(eocd + 8, "\021\0\021\0", 4); /* 17 entries total */
434 	cd_size = le32(eocd + 12);
435 	/* Start of CD offset should be 0xffffffff */
436 	assertEqualMem(eocd + 16, "\xff\xff\xff\xff", 4);
437 	assertEqualMem(eocd + 20, "\0\0", 2);	/* No Zip comment */
438 
439 	/* Verify Zip64 locator */
440 	zip64_locator = p - 42;
441 	assertEqualMem(zip64_locator, "PK\006\007\0\0\0\0", 8);
442 	zip64_eocd = p - (fileblocks->filesize - le64(zip64_locator + 8));
443 	assertEqualMem(zip64_locator + 16, "\001\0\0\0", 4);
444 
445 	/* Verify Zip64 end-of-cd record. */
446 	assert(zip64_eocd == p - 98);
447 	assertEqualMem(zip64_eocd, "PK\006\006", 4);
448 	assertEqualInt(44, le64(zip64_eocd + 4)); // Size of EoCD record - 12
449 	assertEqualMem(zip64_eocd + 12, "\055\0", 2);  // Made by version: 45
450 	assertEqualMem(zip64_eocd + 14, "\055\0", 2);  // Requires version: 45
451 	assertEqualMem(zip64_eocd + 16, "\0\0\0\0", 4); // This disk
452 	assertEqualMem(zip64_eocd + 20, "\0\0\0\0", 4); // Total disks
453 	assertEqualInt(17, le64(zip64_eocd + 24));  // Entries on this disk
454 	assertEqualInt(17, le64(zip64_eocd + 32));  // Total entries
455 	cd_size = le64(zip64_eocd + 40);
456 	cd_start = p - (fileblocks->filesize - le64(zip64_eocd + 48));
457 
458 	assert(cd_start + cd_size == zip64_eocd);
459 
460 	assertEqualInt(le64(zip64_eocd + 48) // Start of CD
461 	    + cd_size
462 	    + 56 // Size of Zip64 EOCD
463 	    + 20 // Size of Zip64 locator
464 	    + 22, // Size of EOCD
465 	    fileblocks->filesize);
466 
467 	// TODO: Scan entire Central Directory, sanity-check all data
468 	assertEqualMem(cd_start, "PK\001\002", 4);
469 
470 	fileblocks_free(fileblocks);
471 	free(buff);
472 	free(nulldata);
473 }
474