1 // Copyright (c) 2017 Cloudflare, Inc. and contributors
2 // Licensed under the MIT License:
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 
22 #if KJ_HAS_ZLIB
23 
24 #include "gzip.h"
25 #include <kj/test.h>
26 #include <kj/debug.h>
27 #include <stdlib.h>
28 
29 namespace kj {
30 namespace {
31 
32 static const byte FOOBAR_GZIP[] = {
33   0x1F, 0x8B, 0x08, 0x00, 0xF9, 0x05, 0xB7, 0x59,
34   0x00, 0x03, 0x4B, 0xCB, 0xCF, 0x4F, 0x4A, 0x2C,
35   0x02, 0x00, 0x95, 0x1F, 0xF6, 0x9E, 0x06, 0x00,
36   0x00, 0x00,
37 };
38 
39 class MockInputStream: public InputStream {
40 public:
MockInputStream(kj::ArrayPtr<const byte> bytes,size_t blockSize)41   MockInputStream(kj::ArrayPtr<const byte> bytes, size_t blockSize)
42       : bytes(bytes), blockSize(blockSize) {}
43 
tryRead(void * buffer,size_t minBytes,size_t maxBytes)44   size_t tryRead(void* buffer, size_t minBytes, size_t maxBytes) override {
45     // Clamp max read to blockSize.
46     size_t n = kj::min(blockSize, maxBytes);
47 
48     // Unless that's less than minBytes -- in which case, use minBytes.
49     n = kj::max(n, minBytes);
50 
51     // But also don't read more data than we have.
52     n = kj::min(n, bytes.size());
53 
54     memcpy(buffer, bytes.begin(), n);
55     bytes = bytes.slice(n, bytes.size());
56     return n;
57   }
58 
59 private:
60   kj::ArrayPtr<const byte> bytes;
61   size_t blockSize;
62 };
63 
64 class MockAsyncInputStream: public AsyncInputStream {
65 public:
MockAsyncInputStream(kj::ArrayPtr<const byte> bytes,size_t blockSize)66   MockAsyncInputStream(kj::ArrayPtr<const byte> bytes, size_t blockSize)
67       : bytes(bytes), blockSize(blockSize) {}
68 
tryRead(void * buffer,size_t minBytes,size_t maxBytes)69   Promise<size_t> tryRead(void* buffer, size_t minBytes, size_t maxBytes) override {
70     // Clamp max read to blockSize.
71     size_t n = kj::min(blockSize, maxBytes);
72 
73     // Unless that's less than minBytes -- in which case, use minBytes.
74     n = kj::max(n, minBytes);
75 
76     // But also don't read more data than we have.
77     n = kj::min(n, bytes.size());
78 
79     memcpy(buffer, bytes.begin(), n);
80     bytes = bytes.slice(n, bytes.size());
81     return n;
82   }
83 
84 private:
85   kj::ArrayPtr<const byte> bytes;
86   size_t blockSize;
87 };
88 
89 class MockOutputStream: public OutputStream {
90 public:
91   kj::Vector<byte> bytes;
92 
decompress()93   kj::String decompress() {
94     MockInputStream rawInput(bytes, kj::maxValue);
95     GzipInputStream gzip(rawInput);
96     return gzip.readAllText();
97   }
98 
write(const void * buffer,size_t size)99   void write(const void* buffer, size_t size) override {
100     bytes.addAll(arrayPtr(reinterpret_cast<const byte*>(buffer), size));
101   }
write(ArrayPtr<const ArrayPtr<const byte>> pieces)102   void write(ArrayPtr<const ArrayPtr<const byte>> pieces) override {
103     for (auto& piece: pieces) {
104       bytes.addAll(piece);
105     }
106   }
107 };
108 
109 class MockAsyncOutputStream: public AsyncOutputStream {
110 public:
111   kj::Vector<byte> bytes;
112 
decompress(WaitScope & ws)113   kj::String decompress(WaitScope& ws) {
114     MockAsyncInputStream rawInput(bytes, kj::maxValue);
115     GzipAsyncInputStream gzip(rawInput);
116     return gzip.readAllText().wait(ws);
117   }
118 
write(const void * buffer,size_t size)119   Promise<void> write(const void* buffer, size_t size) override {
120     bytes.addAll(arrayPtr(reinterpret_cast<const byte*>(buffer), size));
121     return kj::READY_NOW;
122   }
write(ArrayPtr<const ArrayPtr<const byte>> pieces)123   Promise<void> write(ArrayPtr<const ArrayPtr<const byte>> pieces) override {
124     for (auto& piece: pieces) {
125       bytes.addAll(piece);
126     }
127     return kj::READY_NOW;
128   }
129 
whenWriteDisconnected()130   Promise<void> whenWriteDisconnected() override { KJ_UNIMPLEMENTED("not used"); }
131 };
132 
133 KJ_TEST("gzip decompression") {
134   // Normal read.
135   {
136     MockInputStream rawInput(FOOBAR_GZIP, kj::maxValue);
137     GzipInputStream gzip(rawInput);
138     KJ_EXPECT(gzip.readAllText() == "foobar");
139   }
140 
141   // Force read one byte at a time.
142   {
143     MockInputStream rawInput(FOOBAR_GZIP, 1);
144     GzipInputStream gzip(rawInput);
145     KJ_EXPECT(gzip.readAllText() == "foobar");
146   }
147 
148   // Read truncated input.
149   {
150     MockInputStream rawInput(kj::arrayPtr(FOOBAR_GZIP, sizeof(FOOBAR_GZIP) / 2), kj::maxValue);
151     GzipInputStream gzip(rawInput);
152 
153     char text[16];
154     size_t n = gzip.tryRead(text, 1, sizeof(text));
155     text[n] = '\0';
156     KJ_EXPECT(StringPtr(text, n) == "fo");
157 
158     KJ_EXPECT_THROW_MESSAGE("gzip compressed stream ended prematurely",
159         gzip.tryRead(text, 1, sizeof(text)));
160   }
161 
162   // Read concatenated input.
163   {
164     Vector<byte> bytes;
165     bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP));
166     bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP));
167     MockInputStream rawInput(bytes, kj::maxValue);
168     GzipInputStream gzip(rawInput);
169 
170     KJ_EXPECT(gzip.readAllText() == "foobarfoobar");
171   }
172 }
173 
174 KJ_TEST("async gzip decompression") {
175   auto io = setupAsyncIo();
176 
177   // Normal read.
178   {
179     MockAsyncInputStream rawInput(FOOBAR_GZIP, kj::maxValue);
180     GzipAsyncInputStream gzip(rawInput);
181     KJ_EXPECT(gzip.readAllText().wait(io.waitScope) == "foobar");
182   }
183 
184   // Force read one byte at a time.
185   {
186     MockAsyncInputStream rawInput(FOOBAR_GZIP, 1);
187     GzipAsyncInputStream gzip(rawInput);
188     KJ_EXPECT(gzip.readAllText().wait(io.waitScope) == "foobar");
189   }
190 
191   // Read truncated input.
192   {
193     MockAsyncInputStream rawInput(kj::arrayPtr(FOOBAR_GZIP, sizeof(FOOBAR_GZIP) / 2), kj::maxValue);
194     GzipAsyncInputStream gzip(rawInput);
195 
196     char text[16];
197     size_t n = gzip.tryRead(text, 1, sizeof(text)).wait(io.waitScope);
198     text[n] = '\0';
199     KJ_EXPECT(StringPtr(text, n) == "fo");
200 
201     KJ_EXPECT_THROW_MESSAGE("gzip compressed stream ended prematurely",
202         gzip.tryRead(text, 1, sizeof(text)).wait(io.waitScope));
203   }
204 
205   // Read concatenated input.
206   {
207     Vector<byte> bytes;
208     bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP));
209     bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP));
210     MockAsyncInputStream rawInput(bytes, kj::maxValue);
211     GzipAsyncInputStream gzip(rawInput);
212 
213     KJ_EXPECT(gzip.readAllText().wait(io.waitScope) == "foobarfoobar");
214   }
215 
216   // Decompress using an output stream.
217   {
218     MockAsyncOutputStream rawOutput;
219     GzipAsyncOutputStream gzip(rawOutput, GzipAsyncOutputStream::DECOMPRESS);
220 
221     auto mid = sizeof(FOOBAR_GZIP) / 2;
222     gzip.write(FOOBAR_GZIP, mid).wait(io.waitScope);
223     auto str1 = kj::heapString(rawOutput.bytes.asPtr().asChars());
224     KJ_EXPECT(str1 == "fo", str1);
225 
226     gzip.write(FOOBAR_GZIP + mid, sizeof(FOOBAR_GZIP) - mid).wait(io.waitScope);
227     auto str2 = kj::heapString(rawOutput.bytes.asPtr().asChars());
228     KJ_EXPECT(str2 == "foobar", str2);
229 
230     gzip.end().wait(io.waitScope);
231   }
232 }
233 
234 KJ_TEST("gzip compression") {
235   // Normal write.
236   {
237     MockOutputStream rawOutput;
238     {
239       GzipOutputStream gzip(rawOutput);
240       gzip.write("foobar", 6);
241     }
242 
243     KJ_EXPECT(rawOutput.decompress() == "foobar");
244   }
245 
246   // Multi-part write.
247   {
248     MockOutputStream rawOutput;
249     {
250       GzipOutputStream gzip(rawOutput);
251       gzip.write("foo", 3);
252       gzip.write("bar", 3);
253     }
254 
255     KJ_EXPECT(rawOutput.decompress() == "foobar");
256   }
257 
258   // Array-of-arrays write.
259   {
260     MockOutputStream rawOutput;
261 
262     {
263       GzipOutputStream gzip(rawOutput);
264 
265       ArrayPtr<const byte> pieces[] = {
266         kj::StringPtr("foo").asBytes(),
267         kj::StringPtr("bar").asBytes(),
268       };
269       gzip.write(pieces);
270     }
271 
272     KJ_EXPECT(rawOutput.decompress() == "foobar");
273   }
274 }
275 
276 KJ_TEST("gzip huge round trip") {
277   auto bytes = heapArray<byte>(65536);
278   for (auto& b: bytes) {
279     b = rand();
280   }
281 
282   MockOutputStream rawOutput;
283   {
284     GzipOutputStream gzipOut(rawOutput);
285     gzipOut.write(bytes.begin(), bytes.size());
286   }
287 
288   MockInputStream rawInput(rawOutput.bytes, kj::maxValue);
289   GzipInputStream gzipIn(rawInput);
290   auto decompressed = gzipIn.readAllBytes();
291 
292   KJ_ASSERT(decompressed.size() == bytes.size());
293   KJ_ASSERT(memcmp(bytes.begin(), decompressed.begin(), bytes.size()) == 0);
294 }
295 
296 KJ_TEST("async gzip compression") {
297   auto io = setupAsyncIo();
298 
299   // Normal write.
300   {
301     MockAsyncOutputStream rawOutput;
302     GzipAsyncOutputStream gzip(rawOutput);
303     gzip.write("foobar", 6).wait(io.waitScope);
304     gzip.end().wait(io.waitScope);
305 
306     KJ_EXPECT(rawOutput.decompress(io.waitScope) == "foobar");
307   }
308 
309   // Multi-part write.
310   {
311     MockAsyncOutputStream rawOutput;
312     GzipAsyncOutputStream gzip(rawOutput);
313 
314     gzip.write("foo", 3).wait(io.waitScope);
315     auto prevSize = rawOutput.bytes.size();
316 
317     gzip.write("bar", 3).wait(io.waitScope);
318     auto curSize = rawOutput.bytes.size();
319     KJ_EXPECT(prevSize == curSize, prevSize, curSize);
320 
321     gzip.flush().wait(io.waitScope);
322     curSize = rawOutput.bytes.size();
323     KJ_EXPECT(prevSize < curSize, prevSize, curSize);
324 
325     gzip.end().wait(io.waitScope);
326 
327     KJ_EXPECT(rawOutput.decompress(io.waitScope) == "foobar");
328   }
329 
330   // Array-of-arrays write.
331   {
332     MockAsyncOutputStream rawOutput;
333     GzipAsyncOutputStream gzip(rawOutput);
334 
335     ArrayPtr<const byte> pieces[] = {
336       kj::StringPtr("foo").asBytes(),
337       kj::StringPtr("bar").asBytes(),
338     };
339     gzip.write(pieces).wait(io.waitScope);
340     gzip.end().wait(io.waitScope);
341 
342     KJ_EXPECT(rawOutput.decompress(io.waitScope) == "foobar");
343   }
344 }
345 
346 KJ_TEST("async gzip huge round trip") {
347   auto io = setupAsyncIo();
348 
349   auto bytes = heapArray<byte>(65536);
350   for (auto& b: bytes) {
351     b = rand();
352   }
353 
354   MockAsyncOutputStream rawOutput;
355   GzipAsyncOutputStream gzipOut(rawOutput);
356   gzipOut.write(bytes.begin(), bytes.size()).wait(io.waitScope);
357   gzipOut.end().wait(io.waitScope);
358 
359   MockAsyncInputStream rawInput(rawOutput.bytes, kj::maxValue);
360   GzipAsyncInputStream gzipIn(rawInput);
361   auto decompressed = gzipIn.readAllBytes().wait(io.waitScope);
362 
363   KJ_ASSERT(decompressed.size() == bytes.size());
364   KJ_ASSERT(memcmp(bytes.begin(), decompressed.begin(), bytes.size()) == 0);
365 }
366 
367 }  // namespace
368 }  // namespace kj
369 
370 #endif  // KJ_HAS_ZLIB
371