1 // Copyright (c) 2017 Cloudflare, Inc. and contributors 2 // Licensed under the MIT License: 3 // 4 // Permission is hereby granted, free of charge, to any person obtaining a copy 5 // of this software and associated documentation files (the "Software"), to deal 6 // in the Software without restriction, including without limitation the rights 7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 // copies of the Software, and to permit persons to whom the Software is 9 // furnished to do so, subject to the following conditions: 10 // 11 // The above copyright notice and this permission notice shall be included in 12 // all copies or substantial portions of the Software. 13 // 14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 // THE SOFTWARE. 21 22 #if KJ_HAS_ZLIB 23 24 #include "gzip.h" 25 #include <kj/test.h> 26 #include <kj/debug.h> 27 #include <stdlib.h> 28 29 namespace kj { 30 namespace { 31 32 static const byte FOOBAR_GZIP[] = { 33 0x1F, 0x8B, 0x08, 0x00, 0xF9, 0x05, 0xB7, 0x59, 34 0x00, 0x03, 0x4B, 0xCB, 0xCF, 0x4F, 0x4A, 0x2C, 35 0x02, 0x00, 0x95, 0x1F, 0xF6, 0x9E, 0x06, 0x00, 36 0x00, 0x00, 37 }; 38 39 class MockInputStream: public InputStream { 40 public: MockInputStream(kj::ArrayPtr<const byte> bytes,size_t blockSize)41 MockInputStream(kj::ArrayPtr<const byte> bytes, size_t blockSize) 42 : bytes(bytes), blockSize(blockSize) {} 43 tryRead(void * buffer,size_t minBytes,size_t maxBytes)44 size_t tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { 45 // Clamp max read to blockSize. 46 size_t n = kj::min(blockSize, maxBytes); 47 48 // Unless that's less than minBytes -- in which case, use minBytes. 49 n = kj::max(n, minBytes); 50 51 // But also don't read more data than we have. 52 n = kj::min(n, bytes.size()); 53 54 memcpy(buffer, bytes.begin(), n); 55 bytes = bytes.slice(n, bytes.size()); 56 return n; 57 } 58 59 private: 60 kj::ArrayPtr<const byte> bytes; 61 size_t blockSize; 62 }; 63 64 class MockAsyncInputStream: public AsyncInputStream { 65 public: MockAsyncInputStream(kj::ArrayPtr<const byte> bytes,size_t blockSize)66 MockAsyncInputStream(kj::ArrayPtr<const byte> bytes, size_t blockSize) 67 : bytes(bytes), blockSize(blockSize) {} 68 tryRead(void * buffer,size_t minBytes,size_t maxBytes)69 Promise<size_t> tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { 70 // Clamp max read to blockSize. 71 size_t n = kj::min(blockSize, maxBytes); 72 73 // Unless that's less than minBytes -- in which case, use minBytes. 74 n = kj::max(n, minBytes); 75 76 // But also don't read more data than we have. 77 n = kj::min(n, bytes.size()); 78 79 memcpy(buffer, bytes.begin(), n); 80 bytes = bytes.slice(n, bytes.size()); 81 return n; 82 } 83 84 private: 85 kj::ArrayPtr<const byte> bytes; 86 size_t blockSize; 87 }; 88 89 class MockOutputStream: public OutputStream { 90 public: 91 kj::Vector<byte> bytes; 92 decompress()93 kj::String decompress() { 94 MockInputStream rawInput(bytes, kj::maxValue); 95 GzipInputStream gzip(rawInput); 96 return gzip.readAllText(); 97 } 98 write(const void * buffer,size_t size)99 void write(const void* buffer, size_t size) override { 100 bytes.addAll(arrayPtr(reinterpret_cast<const byte*>(buffer), size)); 101 } write(ArrayPtr<const ArrayPtr<const byte>> pieces)102 void write(ArrayPtr<const ArrayPtr<const byte>> pieces) override { 103 for (auto& piece: pieces) { 104 bytes.addAll(piece); 105 } 106 } 107 }; 108 109 class MockAsyncOutputStream: public AsyncOutputStream { 110 public: 111 kj::Vector<byte> bytes; 112 decompress(WaitScope & ws)113 kj::String decompress(WaitScope& ws) { 114 MockAsyncInputStream rawInput(bytes, kj::maxValue); 115 GzipAsyncInputStream gzip(rawInput); 116 return gzip.readAllText().wait(ws); 117 } 118 write(const void * buffer,size_t size)119 Promise<void> write(const void* buffer, size_t size) override { 120 bytes.addAll(arrayPtr(reinterpret_cast<const byte*>(buffer), size)); 121 return kj::READY_NOW; 122 } write(ArrayPtr<const ArrayPtr<const byte>> pieces)123 Promise<void> write(ArrayPtr<const ArrayPtr<const byte>> pieces) override { 124 for (auto& piece: pieces) { 125 bytes.addAll(piece); 126 } 127 return kj::READY_NOW; 128 } 129 whenWriteDisconnected()130 Promise<void> whenWriteDisconnected() override { KJ_UNIMPLEMENTED("not used"); } 131 }; 132 133 KJ_TEST("gzip decompression") { 134 // Normal read. 135 { 136 MockInputStream rawInput(FOOBAR_GZIP, kj::maxValue); 137 GzipInputStream gzip(rawInput); 138 KJ_EXPECT(gzip.readAllText() == "foobar"); 139 } 140 141 // Force read one byte at a time. 142 { 143 MockInputStream rawInput(FOOBAR_GZIP, 1); 144 GzipInputStream gzip(rawInput); 145 KJ_EXPECT(gzip.readAllText() == "foobar"); 146 } 147 148 // Read truncated input. 149 { 150 MockInputStream rawInput(kj::arrayPtr(FOOBAR_GZIP, sizeof(FOOBAR_GZIP) / 2), kj::maxValue); 151 GzipInputStream gzip(rawInput); 152 153 char text[16]; 154 size_t n = gzip.tryRead(text, 1, sizeof(text)); 155 text[n] = '\0'; 156 KJ_EXPECT(StringPtr(text, n) == "fo"); 157 158 KJ_EXPECT_THROW_MESSAGE("gzip compressed stream ended prematurely", 159 gzip.tryRead(text, 1, sizeof(text))); 160 } 161 162 // Read concatenated input. 163 { 164 Vector<byte> bytes; 165 bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP)); 166 bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP)); 167 MockInputStream rawInput(bytes, kj::maxValue); 168 GzipInputStream gzip(rawInput); 169 170 KJ_EXPECT(gzip.readAllText() == "foobarfoobar"); 171 } 172 } 173 174 KJ_TEST("async gzip decompression") { 175 auto io = setupAsyncIo(); 176 177 // Normal read. 178 { 179 MockAsyncInputStream rawInput(FOOBAR_GZIP, kj::maxValue); 180 GzipAsyncInputStream gzip(rawInput); 181 KJ_EXPECT(gzip.readAllText().wait(io.waitScope) == "foobar"); 182 } 183 184 // Force read one byte at a time. 185 { 186 MockAsyncInputStream rawInput(FOOBAR_GZIP, 1); 187 GzipAsyncInputStream gzip(rawInput); 188 KJ_EXPECT(gzip.readAllText().wait(io.waitScope) == "foobar"); 189 } 190 191 // Read truncated input. 192 { 193 MockAsyncInputStream rawInput(kj::arrayPtr(FOOBAR_GZIP, sizeof(FOOBAR_GZIP) / 2), kj::maxValue); 194 GzipAsyncInputStream gzip(rawInput); 195 196 char text[16]; 197 size_t n = gzip.tryRead(text, 1, sizeof(text)).wait(io.waitScope); 198 text[n] = '\0'; 199 KJ_EXPECT(StringPtr(text, n) == "fo"); 200 201 KJ_EXPECT_THROW_MESSAGE("gzip compressed stream ended prematurely", 202 gzip.tryRead(text, 1, sizeof(text)).wait(io.waitScope)); 203 } 204 205 // Read concatenated input. 206 { 207 Vector<byte> bytes; 208 bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP)); 209 bytes.addAll(ArrayPtr<const byte>(FOOBAR_GZIP)); 210 MockAsyncInputStream rawInput(bytes, kj::maxValue); 211 GzipAsyncInputStream gzip(rawInput); 212 213 KJ_EXPECT(gzip.readAllText().wait(io.waitScope) == "foobarfoobar"); 214 } 215 216 // Decompress using an output stream. 217 { 218 MockAsyncOutputStream rawOutput; 219 GzipAsyncOutputStream gzip(rawOutput, GzipAsyncOutputStream::DECOMPRESS); 220 221 auto mid = sizeof(FOOBAR_GZIP) / 2; 222 gzip.write(FOOBAR_GZIP, mid).wait(io.waitScope); 223 auto str1 = kj::heapString(rawOutput.bytes.asPtr().asChars()); 224 KJ_EXPECT(str1 == "fo", str1); 225 226 gzip.write(FOOBAR_GZIP + mid, sizeof(FOOBAR_GZIP) - mid).wait(io.waitScope); 227 auto str2 = kj::heapString(rawOutput.bytes.asPtr().asChars()); 228 KJ_EXPECT(str2 == "foobar", str2); 229 230 gzip.end().wait(io.waitScope); 231 } 232 } 233 234 KJ_TEST("gzip compression") { 235 // Normal write. 236 { 237 MockOutputStream rawOutput; 238 { 239 GzipOutputStream gzip(rawOutput); 240 gzip.write("foobar", 6); 241 } 242 243 KJ_EXPECT(rawOutput.decompress() == "foobar"); 244 } 245 246 // Multi-part write. 247 { 248 MockOutputStream rawOutput; 249 { 250 GzipOutputStream gzip(rawOutput); 251 gzip.write("foo", 3); 252 gzip.write("bar", 3); 253 } 254 255 KJ_EXPECT(rawOutput.decompress() == "foobar"); 256 } 257 258 // Array-of-arrays write. 259 { 260 MockOutputStream rawOutput; 261 262 { 263 GzipOutputStream gzip(rawOutput); 264 265 ArrayPtr<const byte> pieces[] = { 266 kj::StringPtr("foo").asBytes(), 267 kj::StringPtr("bar").asBytes(), 268 }; 269 gzip.write(pieces); 270 } 271 272 KJ_EXPECT(rawOutput.decompress() == "foobar"); 273 } 274 } 275 276 KJ_TEST("gzip huge round trip") { 277 auto bytes = heapArray<byte>(65536); 278 for (auto& b: bytes) { 279 b = rand(); 280 } 281 282 MockOutputStream rawOutput; 283 { 284 GzipOutputStream gzipOut(rawOutput); 285 gzipOut.write(bytes.begin(), bytes.size()); 286 } 287 288 MockInputStream rawInput(rawOutput.bytes, kj::maxValue); 289 GzipInputStream gzipIn(rawInput); 290 auto decompressed = gzipIn.readAllBytes(); 291 292 KJ_ASSERT(decompressed.size() == bytes.size()); 293 KJ_ASSERT(memcmp(bytes.begin(), decompressed.begin(), bytes.size()) == 0); 294 } 295 296 KJ_TEST("async gzip compression") { 297 auto io = setupAsyncIo(); 298 299 // Normal write. 300 { 301 MockAsyncOutputStream rawOutput; 302 GzipAsyncOutputStream gzip(rawOutput); 303 gzip.write("foobar", 6).wait(io.waitScope); 304 gzip.end().wait(io.waitScope); 305 306 KJ_EXPECT(rawOutput.decompress(io.waitScope) == "foobar"); 307 } 308 309 // Multi-part write. 310 { 311 MockAsyncOutputStream rawOutput; 312 GzipAsyncOutputStream gzip(rawOutput); 313 314 gzip.write("foo", 3).wait(io.waitScope); 315 auto prevSize = rawOutput.bytes.size(); 316 317 gzip.write("bar", 3).wait(io.waitScope); 318 auto curSize = rawOutput.bytes.size(); 319 KJ_EXPECT(prevSize == curSize, prevSize, curSize); 320 321 gzip.flush().wait(io.waitScope); 322 curSize = rawOutput.bytes.size(); 323 KJ_EXPECT(prevSize < curSize, prevSize, curSize); 324 325 gzip.end().wait(io.waitScope); 326 327 KJ_EXPECT(rawOutput.decompress(io.waitScope) == "foobar"); 328 } 329 330 // Array-of-arrays write. 331 { 332 MockAsyncOutputStream rawOutput; 333 GzipAsyncOutputStream gzip(rawOutput); 334 335 ArrayPtr<const byte> pieces[] = { 336 kj::StringPtr("foo").asBytes(), 337 kj::StringPtr("bar").asBytes(), 338 }; 339 gzip.write(pieces).wait(io.waitScope); 340 gzip.end().wait(io.waitScope); 341 342 KJ_EXPECT(rawOutput.decompress(io.waitScope) == "foobar"); 343 } 344 } 345 346 KJ_TEST("async gzip huge round trip") { 347 auto io = setupAsyncIo(); 348 349 auto bytes = heapArray<byte>(65536); 350 for (auto& b: bytes) { 351 b = rand(); 352 } 353 354 MockAsyncOutputStream rawOutput; 355 GzipAsyncOutputStream gzipOut(rawOutput); 356 gzipOut.write(bytes.begin(), bytes.size()).wait(io.waitScope); 357 gzipOut.end().wait(io.waitScope); 358 359 MockAsyncInputStream rawInput(rawOutput.bytes, kj::maxValue); 360 GzipAsyncInputStream gzipIn(rawInput); 361 auto decompressed = gzipIn.readAllBytes().wait(io.waitScope); 362 363 KJ_ASSERT(decompressed.size() == bytes.size()); 364 KJ_ASSERT(memcmp(bytes.begin(), decompressed.begin(), bytes.size()) == 0); 365 } 366 367 } // namespace 368 } // namespace kj 369 370 #endif // KJ_HAS_ZLIB 371