1 /*--------------------------------------------------------------------------
2  *  Copyright 2011 Taro L. Saito
3  *
4  *  Licensed under the Apache License, Version 2.0 (the "License");
5  *  you may not use this file except in compliance with the License.
6  *  You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  *  Unless required by applicable law or agreed to in writing, software
11  *  distributed under the License is distributed on an "AS IS" BASIS,
12  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  *  See the License for the specific language governing permissions and
14  *  limitations under the License.
15  *--------------------------------------------------------------------------*/
16 //--------------------------------------
17 // XerialJ
18 //
19 // SnappyOutputStreamTest.java
20 // Since: 2011/03/31 18:26:31
21 //
22 // $URL$
23 // $Author$
24 //--------------------------------------
25 package org.xerial.snappy;
26 
27 import static org.junit.Assert.*;
28 
29 import java.io.BufferedInputStream;
30 import java.io.ByteArrayInputStream;
31 import java.io.ByteArrayOutputStream;
32 import java.io.IOException;
33 import java.lang.ref.WeakReference;
34 import java.nio.ByteOrder;
35 
36 import org.junit.Test;
37 import org.xerial.snappy.buffer.BufferAllocatorFactory;
38 import org.xerial.snappy.buffer.CachedBufferAllocator;
39 import org.xerial.snappy.buffer.DefaultBufferAllocator;
40 import org.xerial.util.FileResource;
41 import org.xerial.util.log.Logger;
42 
43 public class SnappyOutputStreamTest
44 {
45     private static Logger _logger = Logger.getLogger(SnappyOutputStreamTest.class);
46 
47     @Test
test()48     public void test()
49             throws Exception
50     {
51         ByteArrayOutputStream buf = new ByteArrayOutputStream();
52         SnappyOutputStream sout = new SnappyOutputStream(buf);
53 
54         BufferedInputStream input = new BufferedInputStream(FileResource.find(SnappyOutputStreamTest.class,
55                 "alice29.txt").openStream());
56         assertNotNull(input);
57 
58         ByteArrayOutputStream orig = new ByteArrayOutputStream();
59         byte[] tmp = new byte[1024];
60         for (int readBytes = 0; (readBytes = input.read(tmp)) != -1; ) {
61             sout.write(tmp, 0, readBytes);
62             orig.write(tmp, 0, readBytes); // preserve the original data
63         }
64         input.close();
65         sout.flush();
66         orig.flush();
67 
68         int compressedSize = buf.size();
69         _logger.debug("compressed size: " + compressedSize);
70 
71         ByteArrayOutputStream decompressed = new ByteArrayOutputStream();
72         byte[] compressed = buf.toByteArray();
73         // decompress
74         for (int cursor = SnappyCodec.headerSize(); cursor < compressed.length; ) {
75             int chunkSize = SnappyOutputStream.readInt(compressed, cursor);
76             cursor += 4;
77             byte[] tmpOut = new byte[Snappy.uncompressedLength(compressed, cursor, chunkSize)];
78             int decompressedSize = Snappy.uncompress(compressed, cursor, chunkSize, tmpOut, 0);
79             cursor += chunkSize;
80 
81             decompressed.write(tmpOut);
82         }
83         decompressed.flush();
84         assertEquals(orig.size(), decompressed.size());
85         assertArrayEquals(orig.toByteArray(), decompressed.toByteArray());
86     }
87 
88     @Test
bufferSize()89     public void bufferSize()
90             throws Exception
91     {
92         ByteArrayOutputStream b = new ByteArrayOutputStream();
93         SnappyOutputStream os = new SnappyOutputStream(b, 1500);
94         final int bytesToWrite = 5000;
95         byte[] orig = new byte[bytesToWrite];
96         for (int i = 0; i < 5000; ++i) {
97             byte v = (byte) (i % 128);
98             orig[i] = v;
99             os.write(v);
100         }
101         os.close();
102         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
103         byte[] buf = new byte[bytesToWrite / 101];
104         while (is.read(buf) != -1) {
105         }
106         is.close();
107     }
108 
109     @Test
smallWrites()110     public void smallWrites()
111             throws Exception
112     {
113 
114         byte[] orig = CalgaryTest.readFile("alice29.txt");
115         ByteArrayOutputStream b = new ByteArrayOutputStream();
116         SnappyOutputStream out = new SnappyOutputStream(b);
117 
118         for (byte c : orig) {
119             out.write(c);
120         }
121         out.close();
122 
123         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
124         byte[] decompressed = new byte[orig.length];
125         int cursor = 0;
126         int readLen = 0;
127         for (int i = 0; i < decompressed.length && (readLen = is.read(decompressed, i, decompressed.length - i)) != -1; ) {
128             i += readLen;
129         }
130         is.close();
131         assertArrayEquals(orig, decompressed);
132     }
133 
134     /**
135      * Compress the input array by passing it chunk-by-chunk to a SnappyOutputStream.
136      *
137      * @param orig the data to compress
138      * @param maxChunkSize the maximum chunk size, in bytes.
139      * @return the compressed bytes
140      */
compressAsChunks(byte[] orig, int maxChunkSize)141     private static byte[] compressAsChunks(byte[] orig, int maxChunkSize)
142             throws Exception
143     {
144         ByteArrayOutputStream b = new ByteArrayOutputStream();
145         SnappyOutputStream out = new SnappyOutputStream(b);
146 
147         int remaining = orig.length;
148         for (int start = 0; start < orig.length; start += maxChunkSize) {
149             out.write(orig, start, remaining < maxChunkSize ? remaining : maxChunkSize);
150             remaining -= maxChunkSize;
151         }
152         out.close();
153         return b.toByteArray();
154     }
155 
156     @Test
batchingOfWritesShouldNotAffectCompressedDataSize()157     public void batchingOfWritesShouldNotAffectCompressedDataSize()
158             throws Exception
159     {
160         // Regression test for issue #100, a bug where the size of compressed data could be affected
161         // by the batching of writes to the SnappyOutputStream rather than the total amount of data
162         // written to the stream.
163         byte[] orig = CalgaryTest.readFile("alice29.txt");
164         // Compress the data once so that we know the expected size:
165         byte[] expectedCompressedData = compressAsChunks(orig, Integer.MAX_VALUE);
166         // Hardcoding an expected compressed size here will catch regressions that lower the
167         // compression quality:
168         if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN)
169             assertEquals(90992, expectedCompressedData.length);
170         else
171             assertEquals(91080, expectedCompressedData.length);
172         // The chunk size should not affect the size of the compressed output:
173         int[] chunkSizes = new int[] {1, 100, 1023, 1024, 10000};
174         for (int chunkSize : chunkSizes) {
175             byte[] compressedData = compressAsChunks(orig, chunkSize);
176             assertEquals(String.format("when chunk size = %,d", chunkSize), expectedCompressedData.length, compressedData.length);
177             assertArrayEquals(expectedCompressedData, compressedData);
178         }
179     }
180 
181     @Test
closeShouldBeIdempotent()182     public void closeShouldBeIdempotent()
183             throws Exception
184     {
185         // Regression test for issue #107, a bug where close() was non-idempotent and would release
186         // its buffers to the allocator multiple times, which could cause scenarios where two open
187         // SnappyOutputStreams could share the same buffers, leading to stream corruption issues.
188         final BufferAllocatorFactory bufferAllocatorFactory = CachedBufferAllocator.getBufferAllocatorFactory();
189         final int BLOCK_SIZE = 4096;
190         // Create a stream, use it, then close it once:
191         ByteArrayOutputStream ba1 = new ByteArrayOutputStream();
192         SnappyOutputStream os1 = new SnappyOutputStream(ba1, BLOCK_SIZE, bufferAllocatorFactory);
193         os1.write(42);
194         os1.close();
195         // Create a new output stream, which should end up re-using the first stream's freed buffers
196         ByteArrayOutputStream ba2 = new ByteArrayOutputStream();
197         SnappyOutputStream os2 = new SnappyOutputStream(ba2, BLOCK_SIZE, bufferAllocatorFactory);
198         // Close the first stream a second time, which is supposed to be safe due to idempotency:
199         os1.close();
200         // Allocate a third output stream, which is supposed to get its own fresh set of buffers:
201         ByteArrayOutputStream ba3 = new ByteArrayOutputStream();
202         SnappyOutputStream os3 = new SnappyOutputStream(ba3, BLOCK_SIZE, bufferAllocatorFactory);
203         // Since the second and third streams should have distinct sets of buffers, writes to these
204         // streams should not interfere with one another:
205         os2.write(2);
206         os3.write(3);
207         os2.close();
208         os3.close();
209         SnappyInputStream in2 = new SnappyInputStream(new ByteArrayInputStream(ba2.toByteArray()));
210         assertEquals(2, in2.read());
211         in2.close();
212         SnappyInputStream in3 = new SnappyInputStream(new ByteArrayInputStream(ba3.toByteArray()));
213         assertEquals(3, in3.read());
214         in3.close();
215     }
216 
217     @Test
writingToClosedStreamShouldThrowIOException()218     public void writingToClosedStreamShouldThrowIOException()
219             throws IOException
220     {
221         ByteArrayOutputStream b = new ByteArrayOutputStream();
222         SnappyOutputStream os = new SnappyOutputStream(b);
223         os.close();
224         try {
225             os.write(4);
226             fail("Expected write() to throw IOException");
227         }
228         catch (IOException e) {
229             // Expected exception
230         }
231         try {
232             os.write(new int[] {1, 2, 3, 4});
233             fail("Expected write() to throw IOException");
234         }
235         catch (IOException e) {
236             // Expected exception
237         }
238     }
239 
240     @Test
flushingClosedStreamShouldThrowIOException()241     public void flushingClosedStreamShouldThrowIOException()
242             throws IOException
243     {
244         ByteArrayOutputStream b = new ByteArrayOutputStream();
245         SnappyOutputStream os = new SnappyOutputStream(b);
246         os.close();
247         try {
248             os.flush();
249         }
250         catch (IOException e) {
251             // Expected exception
252         }
253     }
254 
255     @Test
closingStreamShouldMakeBuffersEligibleForGarbageCollection()256     public void closingStreamShouldMakeBuffersEligibleForGarbageCollection()
257             throws IOException
258     {
259         ByteArrayOutputStream b = new ByteArrayOutputStream();
260         SnappyOutputStream os = new SnappyOutputStream(b, 4095, DefaultBufferAllocator.factory);
261         WeakReference<byte[]> inputBuffer = new WeakReference<byte[]>(os.inputBuffer);
262         WeakReference<byte[]> outputBuffer = new WeakReference<byte[]>(os.inputBuffer);
263         os.close();
264         System.gc();
265         assertNull(inputBuffer.get());
266         assertNull(outputBuffer.get());
267     }
268 
269     @Test
longArrayCompress()270     public void longArrayCompress()
271             throws Exception
272     {
273         long[] l = new long[10];
274         for (int i = 0; i < l.length; ++i) {
275             l[i] = i % 3 + i * 11;
276         }
277 
278         ByteArrayOutputStream b = new ByteArrayOutputStream();
279         SnappyOutputStream os = new SnappyOutputStream(b);
280 
281         os.write(l);
282         os.close();
283         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
284         long[] l2 = new long[10];
285         int readBytes = is.read(l2);
286         is.close();
287 
288         assertEquals(10 * 8, readBytes);
289         assertArrayEquals(l, l2);
290     }
291 
292     @Test
writeDoubleArray()293     public void writeDoubleArray()
294             throws Exception
295     {
296         ByteArrayOutputStream b = new ByteArrayOutputStream();
297         SnappyOutputStream os = new SnappyOutputStream(b);
298 
299         double[] orig = new double[] {1.0, 2.0, 1.4, 0.00343430014, -4.4, 4e-20};
300         os.write(orig);
301         os.close();
302 
303         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
304         double[] uncompressed = new double[orig.length];
305         is.read(uncompressed);
306         is.close();
307 
308         assertArrayEquals(orig, uncompressed, 0.0);
309     }
310 
311     @Test
writeFloatArray()312     public void writeFloatArray()
313             throws Exception
314     {
315         ByteArrayOutputStream b = new ByteArrayOutputStream();
316         SnappyOutputStream os = new SnappyOutputStream(b);
317 
318         float[] orig = new float[] {1.0f, 2.0f, 1.4f, 0.00343430014f, -4.4f, 4e-20f};
319         os.write(orig);
320         os.close();
321 
322         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
323         float[] uncompressed = new float[orig.length];
324         is.read(uncompressed);
325         is.close();
326 
327         assertArrayEquals(orig, uncompressed, 0.0f);
328     }
329 
330     @Test
writeIntArray()331     public void writeIntArray()
332             throws Exception
333     {
334         ByteArrayOutputStream b = new ByteArrayOutputStream();
335         SnappyOutputStream os = new SnappyOutputStream(b);
336 
337         int[] orig = new int[] {0, -1, -34, 43, 234, 34324, -234};
338         os.write(orig);
339         os.close();
340 
341         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
342         int[] uncompressed = new int[orig.length];
343         is.read(uncompressed);
344         is.close();
345 
346         assertArrayEquals(orig, uncompressed);
347     }
348 
349     @Test
writeShortArray()350     public void writeShortArray()
351             throws Exception
352     {
353         ByteArrayOutputStream b = new ByteArrayOutputStream();
354         SnappyOutputStream os = new SnappyOutputStream(b);
355 
356         short[] orig = new short[] {0, -1, -34, 43, 234, 324, -234};
357         os.write(orig);
358         os.close();
359 
360         SnappyInputStream is = new SnappyInputStream(new ByteArrayInputStream(b.toByteArray()));
361         short[] uncompressed = new short[orig.length];
362         is.read(uncompressed);
363         is.close();
364 
365         assertArrayEquals(orig, uncompressed);
366     }
367 }
368