1 /*-
2 * Copyright (c) 2003-2007 Tim Kientzle
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25 #include "test.h"
26 __FBSDID("$FreeBSD: head/lib/libarchive/test/test_tar_large.c 201247 2009-12-30 05:59:21Z kientzle $");
27
28 #include <errno.h>
29 #include <stdlib.h>
30 #include <string.h>
31
32 /*
33 * This is a somewhat tricky test that verifies the ability to
34 * write and read very large entries to tar archives. It
35 * writes entries from 2GB up to 1TB to an archive in memory.
36 * The memory storage here carefully avoids actually storing
37 * any part of the file bodies, so it runs very quickly and requires
38 * very little memory. If you're willing to wait a few minutes,
39 * you should be able to exercise petabyte entries with this code.
40 */
41
42 /*
43 * Each file is built up by duplicating the following block.
44 */
45 static size_t filedatasize;
46 static void *filedata;
47
48 /*
49 * We store the archive as blocks of data generated by libarchive,
50 * each possibly followed by bytes of file data.
51 */
52 struct memblock {
53 struct memblock *next;
54 size_t size;
55 void *buff;
56 int64_t filebytes;
57 };
58
59 /*
60 * The total memory store is just a list of memblocks plus
61 * some accounting overhead.
62 */
63 struct memdata {
64 int64_t filebytes;
65 void *buff;
66 struct memblock *first;
67 struct memblock *last;
68 };
69
70 /* The following size definitions simplify things below. */
71 #define KB ((int64_t)1024)
72 #define MB ((int64_t)1024 * KB)
73 #define GB ((int64_t)1024 * MB)
74 #define TB ((int64_t)1024 * GB)
75
76 static int64_t memory_read_skip(struct archive *, void *, int64_t request);
77 static ssize_t memory_read(struct archive *, void *, const void **buff);
78 static ssize_t memory_write(struct archive *, void *, const void *, size_t);
79
80
81 static ssize_t
memory_write(struct archive * a,void * _private,const void * buff,size_t size)82 memory_write(struct archive *a, void *_private, const void *buff, size_t size)
83 {
84 struct memdata *private = _private;
85 struct memblock *block;
86
87 (void)a;
88
89 /*
90 * Since libarchive tries to behave in a zero-copy manner, if
91 * you give a pointer to filedata to the library, a pointer
92 * into that data will (usually) pop out here. This way, we
93 * can tell the difference between filedata and library header
94 * and metadata.
95 */
96 if ((const char *)filedata <= (const char *)buff
97 && (const char *)buff < (const char *)filedata + filedatasize) {
98 /* We don't need to store a block of file data. */
99 private->last->filebytes += (int64_t)size;
100 } else {
101 /* Yes, we're assuming the very first write is metadata. */
102 /* It's header or metadata, copy and save it. */
103 block = (struct memblock *)malloc(sizeof(*block));
104 memset(block, 0, sizeof(*block));
105 block->size = size;
106 block->buff = malloc(size);
107 memcpy(block->buff, buff, size);
108 if (private->last == NULL) {
109 private->first = private->last = block;
110 } else {
111 private->last->next = block;
112 private->last = block;
113 }
114 block->next = NULL;
115 }
116 return ((long)size);
117 }
118
119 static ssize_t
memory_read(struct archive * a,void * _private,const void ** buff)120 memory_read(struct archive *a, void *_private, const void **buff)
121 {
122 struct memdata *private = _private;
123 struct memblock *block;
124 ssize_t size;
125
126 (void)a;
127
128 free(private->buff);
129 private->buff = NULL;
130 if (private->first == NULL) {
131 private->last = NULL;
132 return (ARCHIVE_EOF);
133 }
134 if (private->filebytes > 0) {
135 /*
136 * We're returning file bytes, simulate it by
137 * passing blocks from the template data.
138 */
139 if (private->filebytes > (int64_t)filedatasize)
140 size = (ssize_t)filedatasize;
141 else
142 size = (ssize_t)private->filebytes;
143 private->filebytes -= size;
144 *buff = filedata;
145 } else {
146 /*
147 * We need to get some real data to return.
148 */
149 block = private->first;
150 private->first = block->next;
151 size = (ssize_t)block->size;
152 if (block->buff != NULL) {
153 private->buff = block->buff;
154 *buff = block->buff;
155 } else {
156 private->buff = NULL;
157 *buff = filedata;
158 }
159 private->filebytes = block->filebytes;
160 free(block);
161 }
162 return (size);
163 }
164
165
166 static int64_t
memory_read_skip(struct archive * a,void * _private,int64_t skip)167 memory_read_skip(struct archive *a, void *_private, int64_t skip)
168 {
169 struct memdata *private = _private;
170
171 (void)a;
172
173 if (private->first == NULL) {
174 private->last = NULL;
175 return (0);
176 }
177 if (private->filebytes > 0) {
178 if (private->filebytes < skip)
179 skip = (off_t)private->filebytes;
180 private->filebytes -= skip;
181 } else {
182 skip = 0;
183 }
184 return (skip);
185 }
186
DEFINE_TEST(test_tar_large)187 DEFINE_TEST(test_tar_large)
188 {
189 /* The sizes of the entries we're going to generate. */
190 static int64_t tests[] = {
191 /* Test for 32-bit signed overflow. */
192 2 * GB - 1, 2 * GB, 2 * GB + 1,
193 /* Test for 32-bit unsigned overflow. */
194 4 * GB - 1, 4 * GB, 4 * GB + 1,
195 /* 8GB is the "official" max for ustar. */
196 8 * GB - 1, 8 * GB, 8 * GB + 1,
197 /* Bend ustar a tad and you can get 64GB (12 octal digits). */
198 64 * GB - 1, 64 * GB,
199 /* And larger entries that require non-ustar extensions. */
200 256 * GB, 1 * TB, 0 };
201 int i;
202 char namebuff[64];
203 struct memdata memdata;
204 struct archive_entry *ae;
205 struct archive *a;
206 int64_t filesize;
207 size_t writesize;
208
209 filedatasize = (size_t)(1 * MB);
210 filedata = malloc(filedatasize);
211 memset(filedata, 0xAA, filedatasize);
212 memset(&memdata, 0, sizeof(memdata));
213
214 /*
215 * Open an archive for writing.
216 */
217 a = archive_write_new();
218 archive_write_set_format_pax_restricted(a);
219 archive_write_set_bytes_per_block(a, 0); /* No buffering. */
220 archive_write_open(a, &memdata, NULL, memory_write, NULL);
221
222 /*
223 * Write a series of large files to it.
224 */
225 for (i = 0; tests[i] != 0; i++) {
226 assert((ae = archive_entry_new()) != NULL);
227 sprintf(namebuff, "file_%d", i);
228 archive_entry_copy_pathname(ae, namebuff);
229 archive_entry_set_mode(ae, S_IFREG | 0755);
230 filesize = tests[i];
231
232 archive_entry_set_size(ae, filesize);
233
234 assertA(0 == archive_write_header(a, ae));
235 archive_entry_free(ae);
236
237 /*
238 * Write the actual data to the archive.
239 */
240 while (filesize > 0) {
241 writesize = filedatasize;
242 if ((int64_t)writesize > filesize)
243 writesize = (size_t)filesize;
244 assertA((int)writesize
245 == archive_write_data(a, filedata, writesize));
246 filesize -= writesize;
247 }
248 }
249
250 assert((ae = archive_entry_new()) != NULL);
251 archive_entry_copy_pathname(ae, "lastfile");
252 archive_entry_set_mode(ae, S_IFREG | 0755);
253 assertA(0 == archive_write_header(a, ae));
254 archive_entry_free(ae);
255
256
257 /* Close out the archive. */
258 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a));
259 assertEqualInt(ARCHIVE_OK, archive_write_free(a));
260
261 /*
262 * Open the same archive for reading.
263 */
264 a = archive_read_new();
265 archive_read_support_format_tar(a);
266 archive_read_open2(a, &memdata, NULL,
267 memory_read, memory_read_skip, NULL);
268
269 /*
270 * Read entries back.
271 */
272 for (i = 0; tests[i] > 0; i++) {
273 assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
274 sprintf(namebuff, "file_%d", i);
275 assertEqualString(namebuff, archive_entry_pathname(ae));
276 assert(tests[i] == archive_entry_size(ae));
277 }
278 assertEqualIntA(a, 0, archive_read_next_header(a, &ae));
279 assertEqualString("lastfile", archive_entry_pathname(ae));
280
281 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae));
282
283 /* Close out the archive. */
284 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
285 assertEqualInt(ARCHIVE_OK, archive_read_free(a));
286
287 free(memdata.buff);
288 free(filedata);
289 }
290