1 /* 2 * Created: Mar 14, 2013 3 */ 4 package org.xerial.snappy; 5 6 import static org.junit.Assert.assertArrayEquals; 7 import static org.junit.Assert.assertEquals; 8 import static org.junit.Assert.fail; 9 import static org.xerial.snappy.SnappyFramed.COMPRESSED_DATA_FLAG; 10 import static org.xerial.snappy.SnappyFramed.HEADER_BYTES; 11 import static org.xerial.snappy.SnappyFramed.UNCOMPRESSED_DATA_FLAG; 12 import static org.xerial.snappy.SnappyFramed.maskedCrc32c; 13 14 import java.io.ByteArrayInputStream; 15 import java.io.ByteArrayOutputStream; 16 import java.io.EOFException; 17 import java.io.IOException; 18 import java.io.InputStream; 19 import java.io.OutputStream; 20 import java.nio.channels.Channels; 21 import java.nio.channels.WritableByteChannel; 22 import java.util.Arrays; 23 24 import org.junit.Test; 25 26 /** 27 * Tests the functionality of {@link org.xerial.snappy.SnappyFramedInputStream} 28 * and {@link org.xerial.snappy.SnappyFramedOutputStream}. 29 * 30 * @author Brett Okken 31 */ 32 public class SnappyFramedStreamTest 33 { 34 35 /** 36 * @throws IOException 37 */ createOutputStream(OutputStream target)38 protected OutputStream createOutputStream(OutputStream target) 39 throws IOException 40 { 41 return new SnappyFramedOutputStream(target); 42 } 43 44 /** 45 * {@inheritDoc} 46 * 47 * @throws IOException 48 */ createInputStream(InputStream source, boolean verifyCheckSums)49 protected InputStream createInputStream(InputStream source, 50 boolean verifyCheckSums) 51 throws IOException 52 { 53 return new SnappyFramedInputStream(source, verifyCheckSums); 54 } 55 getMarkerFrame()56 protected byte[] getMarkerFrame() 57 { 58 return HEADER_BYTES; 59 } 60 61 @Test testSimple()62 public void testSimple() 63 throws Exception 64 { 65 byte[] original = "aaaaaaaaaaaabbbbbbbaaaaaa".getBytes("utf-8"); 66 67 byte[] compressed = compress(original); 68 byte[] uncompressed = uncompress(compressed); 69 70 assertArrayEquals(uncompressed, original); 71 // 10 byte stream header, 4 byte block header, 4 byte crc, 19 bytes 72 assertEquals(compressed.length, 37); 73 74 // stream header 75 assertArrayEquals(Arrays.copyOf(compressed, 10), HEADER_BYTES); 76 77 // flag: compressed 78 assertEquals(toInt(compressed[10]), COMPRESSED_DATA_FLAG); 79 80 // length: 23 = 0x000017 81 assertEquals(toInt(compressed[11]), 0x17); 82 assertEquals(toInt(compressed[12]), 0x00); 83 assertEquals(toInt(compressed[13]), 0x00); 84 85 // crc32c: 0x9274cda8 86 assertEquals(toInt(compressed[17]), 0x92); 87 assertEquals(toInt(compressed[16]), 0x74); 88 assertEquals(toInt(compressed[15]), 0xCD); 89 assertEquals(toInt(compressed[14]), 0xA8); 90 } 91 92 @Test testUncompressable()93 public void testUncompressable() 94 throws Exception 95 { 96 byte[] random = getRandom(1, 5000); 97 int crc32c = maskedCrc32c(random); 98 99 byte[] compressed = compress(random); 100 byte[] uncompressed = uncompress(compressed); 101 102 assertArrayEquals(uncompressed, random); 103 assertEquals(compressed.length, random.length + 10 + 4 + 4); 104 105 // flag: uncompressed 106 assertEquals(toInt(compressed[10]), UNCOMPRESSED_DATA_FLAG); 107 108 // length: 5004 = 0x138c 109 assertEquals(toInt(compressed[13]), 0x00); 110 assertEquals(toInt(compressed[12]), 0x13); 111 assertEquals(toInt(compressed[11]), 0x8c); 112 } 113 114 @Test testEmptyCompression()115 public void testEmptyCompression() 116 throws Exception 117 { 118 byte[] empty = new byte[0]; 119 assertArrayEquals(compress(empty), HEADER_BYTES); 120 assertArrayEquals(uncompress(HEADER_BYTES), empty); 121 } 122 123 @Test(expected = EOFException.class) testShortBlockHeader()124 public void testShortBlockHeader() 125 throws Exception 126 { 127 uncompressBlock(new byte[] {0}); 128 } 129 130 @Test(expected = EOFException.class) testShortBlockData()131 public void testShortBlockData() 132 throws Exception 133 { 134 // flag = 0, size = 8, crc32c = 0, block data= [x, x] 135 uncompressBlock(new byte[] {1, 8, 0, 0, 0, 0, 0, 0, 'x', 'x'}); 136 } 137 138 @Test testUnskippableChunkFlags()139 public void testUnskippableChunkFlags() 140 throws Exception 141 { 142 for (int i = 2; i <= 0x7f; i++) { 143 try { 144 uncompressBlock(new byte[] {(byte) i, 5, 0, 0, 0, 0, 0, 0, 0}); 145 fail("no exception thrown with flag: " + Integer.toHexString(i)); 146 } 147 catch (IOException e) { 148 149 } 150 } 151 } 152 153 @Test testSkippableChunkFlags()154 public void testSkippableChunkFlags() 155 throws Exception 156 { 157 for (int i = 0x80; i <= 0xfe; i++) { 158 try { 159 uncompressBlock(new byte[] {(byte) i, 5, 0, 0, 0, 0, 0, 0, 0}); 160 } 161 catch (IOException e) { 162 fail("exception thrown with flag: " + Integer.toHexString(i)); 163 } 164 } 165 } 166 167 @Test(expected = IOException.class) testInvalidBlockSizeZero()168 public void testInvalidBlockSizeZero() 169 throws Exception 170 { 171 // flag = '0', block size = 4, crc32c = 0 172 uncompressBlock(new byte[] {1, 4, 0, 0, 0, 0, 0, 0}); 173 } 174 175 @Test(expected = IOException.class) testInvalidChecksum()176 public void testInvalidChecksum() 177 throws Exception 178 { 179 // flag = 0, size = 5, crc32c = 0, block data = [a] 180 uncompressBlock(new byte[] {1, 5, 0, 0, 0, 0, 0, 0, 'a'}); 181 } 182 183 @Test testInvalidChecksumIgnoredWhenVerificationDisabled()184 public void testInvalidChecksumIgnoredWhenVerificationDisabled() 185 throws Exception 186 { 187 // flag = 0, size = 4, crc32c = 0, block data = [a] 188 byte[] block = {1, 5, 0, 0, 0, 0, 0, 0, 'a'}; 189 ByteArrayInputStream inputData = new ByteArrayInputStream( 190 blockToStream(block)); 191 assertArrayEquals(toByteArray(createInputStream(inputData, false)), 192 new byte[] {'a'}); 193 } 194 195 @Test testTransferFrom_InputStream()196 public void testTransferFrom_InputStream() 197 throws IOException 198 { 199 final byte[] random = getRandom(0.5, 100000); 200 201 final ByteArrayOutputStream baos = new ByteArrayOutputStream( 202 random.length); 203 final SnappyFramedOutputStream sfos = new SnappyFramedOutputStream(baos); 204 205 sfos.transferFrom(new ByteArrayInputStream(random)); 206 207 sfos.close(); 208 209 final byte[] uncompressed = uncompress(baos.toByteArray()); 210 211 assertArrayEquals(random, uncompressed); 212 } 213 214 @Test testTransferFrom_ReadableByteChannel()215 public void testTransferFrom_ReadableByteChannel() 216 throws IOException 217 { 218 final byte[] random = getRandom(0.5, 100000); 219 220 final ByteArrayOutputStream baos = new ByteArrayOutputStream( 221 random.length); 222 final SnappyFramedOutputStream sfos = new SnappyFramedOutputStream(baos); 223 224 sfos.transferFrom(Channels.newChannel(new ByteArrayInputStream(random))); 225 226 sfos.close(); 227 228 final byte[] uncompressed = uncompress(baos.toByteArray()); 229 230 assertArrayEquals(random, uncompressed); 231 } 232 233 @Test testTransferTo_OutputStream()234 public void testTransferTo_OutputStream() 235 throws IOException 236 { 237 final byte[] random = getRandom(0.5, 100000); 238 239 final byte[] compressed = compress(random); 240 final SnappyFramedInputStream sfis = new SnappyFramedInputStream( 241 new ByteArrayInputStream(compressed)); 242 243 final ByteArrayOutputStream baos = new ByteArrayOutputStream( 244 random.length); 245 sfis.transferTo(baos); 246 247 assertArrayEquals(random, baos.toByteArray()); 248 } 249 250 @Test testTransferTo_WritableByteChannel()251 public void testTransferTo_WritableByteChannel() 252 throws IOException 253 { 254 final byte[] random = getRandom(0.5, 100000); 255 256 final byte[] compressed = compress(random); 257 final SnappyFramedInputStream sfis = new SnappyFramedInputStream( 258 new ByteArrayInputStream(compressed)); 259 260 final ByteArrayOutputStream baos = new ByteArrayOutputStream( 261 random.length); 262 final WritableByteChannel wbc = Channels.newChannel(baos); 263 sfis.transferTo(wbc); 264 wbc.close(); 265 266 assertArrayEquals(random, baos.toByteArray()); 267 } 268 269 @Test testLargerFrames_raw_()270 public void testLargerFrames_raw_() 271 throws IOException 272 { 273 final byte[] random = getRandom(0.5, 100000); 274 275 final byte[] stream = new byte[HEADER_BYTES.length + 8 + random.length]; 276 System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); 277 278 stream[10] = UNCOMPRESSED_DATA_FLAG; 279 280 int length = random.length + 4; 281 stream[11] = (byte) length; 282 stream[12] = (byte) (length >>> 8); 283 stream[13] = (byte) (length >>> 16); 284 285 int crc32c = maskedCrc32c(random); 286 stream[14] = (byte) crc32c; 287 stream[15] = (byte) (crc32c >>> 8); 288 stream[16] = (byte) (crc32c >>> 16); 289 stream[17] = (byte) (crc32c >>> 24); 290 291 System.arraycopy(random, 0, stream, 18, random.length); 292 293 final byte[] uncompressed = uncompress(stream); 294 295 assertArrayEquals(random, uncompressed); 296 } 297 298 @Test testLargerFrames_compressed_()299 public void testLargerFrames_compressed_() 300 throws IOException 301 { 302 final byte[] random = getRandom(0.5, 500000); 303 304 final byte[] compressed = Snappy.compress(random); 305 306 final byte[] stream = new byte[HEADER_BYTES.length + 8 + compressed.length]; 307 System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); 308 309 stream[10] = COMPRESSED_DATA_FLAG; 310 311 int length = compressed.length + 4; 312 stream[11] = (byte) length; 313 stream[12] = (byte) (length >>> 8); 314 stream[13] = (byte) (length >>> 16); 315 316 int crc32c = maskedCrc32c(random); 317 stream[14] = (byte) crc32c; 318 stream[15] = (byte) (crc32c >>> 8); 319 stream[16] = (byte) (crc32c >>> 16); 320 stream[17] = (byte) (crc32c >>> 24); 321 322 System.arraycopy(compressed, 0, stream, 18, compressed.length); 323 324 final byte[] uncompressed = uncompress(stream); 325 326 assertArrayEquals(random, uncompressed); 327 } 328 329 @Test testLargerFrames_compressed_smaller_raw_larger()330 public void testLargerFrames_compressed_smaller_raw_larger() 331 throws IOException 332 { 333 final byte[] random = getRandom(0.5, 100000); 334 335 final byte[] compressed = Snappy.compress(random); 336 337 final byte[] stream = new byte[HEADER_BYTES.length + 8 338 + compressed.length]; 339 System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); 340 341 stream[10] = COMPRESSED_DATA_FLAG; 342 343 int length = compressed.length + 4; 344 stream[11] = (byte) length; 345 stream[12] = (byte) (length >>> 8); 346 stream[13] = (byte) (length >>> 16); 347 348 int crc32c = maskedCrc32c(random); 349 stream[14] = (byte) crc32c; 350 stream[15] = (byte) (crc32c >>> 8); 351 stream[16] = (byte) (crc32c >>> 16); 352 stream[17] = (byte) (crc32c >>> 24); 353 354 System.arraycopy(compressed, 0, stream, 18, compressed.length); 355 356 final byte[] uncompressed = uncompress(stream); 357 358 assertArrayEquals(random, uncompressed); 359 } 360 uncompressBlock(byte[] block)361 private byte[] uncompressBlock(byte[] block) 362 throws IOException 363 { 364 return uncompress(blockToStream(block)); 365 } 366 blockToStream(byte[] block)367 private static byte[] blockToStream(byte[] block) 368 { 369 byte[] stream = new byte[HEADER_BYTES.length + block.length]; 370 System.arraycopy(HEADER_BYTES, 0, stream, 0, HEADER_BYTES.length); 371 System.arraycopy(block, 0, stream, HEADER_BYTES.length, block.length); 372 return stream; 373 } 374 compress(byte[] original)375 protected byte[] compress(byte[] original) 376 throws IOException 377 { 378 ByteArrayOutputStream out = new ByteArrayOutputStream(); 379 OutputStream snappyOut = createOutputStream(out); 380 snappyOut.write(original); 381 snappyOut.close(); 382 return out.toByteArray(); 383 } 384 uncompress(byte[] compressed)385 protected byte[] uncompress(byte[] compressed) 386 throws IOException 387 { 388 return toByteArray(createInputStream(new ByteArrayInputStream( 389 compressed), true)); 390 } 391 toByteArray(InputStream createInputStream)392 private static byte[] toByteArray(InputStream createInputStream) 393 throws IOException 394 { 395 final ByteArrayOutputStream baos = new ByteArrayOutputStream(64 * 1024); 396 397 final byte[] buffer = new byte[8 * 1024]; 398 399 int read; 400 while ((read = createInputStream.read(buffer)) > 0) { 401 baos.write(buffer, 0, read); 402 } 403 404 return baos.toByteArray(); 405 } 406 toInt(byte value)407 static int toInt(byte value) 408 { 409 return value & 0xFF; 410 } 411 getRandom(double compressionRatio, int length)412 private byte[] getRandom(double compressionRatio, int length) 413 { 414 RandomGenerator gen = new RandomGenerator( 415 compressionRatio); 416 gen.getNextPosition(length); 417 byte[] random = Arrays.copyOf(gen.data, length); 418 assertEquals(random.length, length); 419 return random; 420 } 421 } 422