1package snappystream 2 3import ( 4 "bytes" 5 "crypto/rand" 6 "io" 7 "io/ioutil" 8 "testing" 9) 10 11const TestFileSize = 10 << 20 // 10MB 12 13// dummyBytesReader returns an io.Reader that avoids buffering optimizations 14// in io.Copy. This can be considered a 'worst-case' io.Reader as far as writer 15// frame alignment goes. 16// 17// Note: io.Copy uses a 32KB buffer internally as of Go 1.3, but that isn't 18// part of its public API (undocumented). 19func dummyBytesReader(p []byte) io.Reader { 20 return ioutil.NopCloser(bytes.NewReader(p)) 21} 22 23func testWriteThenRead(t *testing.T, name string, bs []byte) { 24 var buf bytes.Buffer 25 w := NewWriter(&buf) 26 n, err := io.Copy(w, dummyBytesReader(bs)) 27 if err != nil { 28 t.Errorf("write %v: %v", name, err) 29 return 30 } 31 if n != int64(len(bs)) { 32 t.Errorf("write %v: wrote %d bytes (!= %d)", name, n, len(bs)) 33 return 34 } 35 36 enclen := buf.Len() 37 38 r := NewReader(&buf, true) 39 gotbs, err := ioutil.ReadAll(r) 40 if err != nil { 41 t.Errorf("read %v: %v", name, err) 42 return 43 } 44 n = int64(len(gotbs)) 45 if n != int64(len(bs)) { 46 t.Errorf("read %v: read %d bytes (!= %d)", name, n, len(bs)) 47 return 48 } 49 50 if !bytes.Equal(gotbs, bs) { 51 t.Errorf("%v: unequal decompressed content", name) 52 return 53 } 54 55 c := float64(len(bs)) / float64(enclen) 56 t.Logf("%v compression ratio %.03g (%d byte reduction)", name, c, len(bs)-enclen) 57} 58 59func testBufferedWriteThenRead(t *testing.T, name string, bs []byte) { 60 var buf bytes.Buffer 61 w := NewBufferedWriter(&buf) 62 n, err := io.Copy(w, dummyBytesReader(bs)) 63 if err != nil { 64 t.Errorf("write %v: %v", name, err) 65 return 66 } 67 if n != int64(len(bs)) { 68 t.Errorf("write %v: wrote %d bytes (!= %d)", name, n, len(bs)) 69 return 70 } 71 err = w.Close() 72 if err != nil { 73 t.Errorf("close %v: %v", name, err) 74 return 75 } 76 77 enclen := buf.Len() 78 79 r := NewReader(&buf, true) 80 gotbs, err := ioutil.ReadAll(r) 81 if err != nil { 82 t.Errorf("read %v: %v", name, err) 83 return 84 } 85 n = int64(len(gotbs)) 86 if n != int64(len(bs)) { 87 t.Errorf("read %v: read %d bytes (!= %d)", name, n, len(bs)) 88 return 89 } 90 91 if !bytes.Equal(gotbs, bs) { 92 t.Errorf("%v: unequal decompressed content", name) 93 return 94 } 95 96 c := float64(len(bs)) / float64(enclen) 97 t.Logf("%v compression ratio %.03g (%d byte reduction)", name, c, len(bs)-enclen) 98} 99 100func TestWriterReader(t *testing.T) { 101 testWriteThenRead(t, "simple", []byte("test")) 102 testWriteThenRead(t, "manpage", testDataMan) 103 testWriteThenRead(t, "json", testDataJSON) 104 105 p := make([]byte, TestFileSize) 106 testWriteThenRead(t, "constant", p) 107 108 _, err := rand.Read(p) 109 if err != nil { 110 t.Fatal(err) 111 } 112 testWriteThenRead(t, "random", p) 113 114} 115 116func TestBufferedWriterReader(t *testing.T) { 117 testBufferedWriteThenRead(t, "simple", []byte("test")) 118 testBufferedWriteThenRead(t, "manpage", testDataMan) 119 testBufferedWriteThenRead(t, "json", testDataJSON) 120 121 p := make([]byte, TestFileSize) 122 testBufferedWriteThenRead(t, "constant", p) 123 124 _, err := rand.Read(p) 125 if err != nil { 126 t.Fatal(err) 127 } 128 testBufferedWriteThenRead(t, "random", p) 129 130} 131 132func TestWriterChunk(t *testing.T) { 133 var buf bytes.Buffer 134 135 in := make([]byte, 128000) 136 137 w := NewWriter(&buf) 138 r := NewReader(&buf, VerifyChecksum) 139 140 n, err := w.Write(in) 141 if err != nil { 142 t.Fatalf(err.Error()) 143 } 144 if n != len(in) { 145 t.Fatalf("wrote wrong amount %d != %d", n, len(in)) 146 } 147 148 out := make([]byte, len(in)) 149 n, err = io.ReadFull(r, out) 150 if err != nil { 151 t.Fatal(err) 152 } 153 if n != len(in) { 154 t.Fatalf("read wrong amount %d != %d", n, len(in)) 155 } 156 157 if !bytes.Equal(out, in) { 158 t.Fatalf("bytes not equal %v != %v", out, in) 159 } 160} 161 162func BenchmarkWriterManpage(b *testing.B) { 163 benchmarkWriterBytes(b, testDataMan) 164} 165func BenchmarkBufferedWriterManpage(b *testing.B) { 166 benchmarkBufferedWriterBytes(b, testDataMan) 167} 168func BenchmarkBufferedWriterManpageNoCopy(b *testing.B) { 169 benchmarkBufferedWriterBytesNoCopy(b, testDataMan) 170} 171 172func BenchmarkWriterJSON(b *testing.B) { 173 benchmarkWriterBytes(b, testDataJSON) 174} 175func BenchmarkBufferedWriterJSON(b *testing.B) { 176 benchmarkBufferedWriterBytes(b, testDataJSON) 177} 178func BenchmarkBufferedWriterJSONNoCopy(b *testing.B) { 179 benchmarkBufferedWriterBytesNoCopy(b, testDataJSON) 180} 181 182// BenchmarkWriterRandom tests performance encoding effectively uncompressable 183// data. 184func BenchmarkWriterRandom(b *testing.B) { 185 benchmarkWriterBytes(b, randBytes(b, TestFileSize)) 186} 187func BenchmarkBufferedWriterRandom(b *testing.B) { 188 benchmarkBufferedWriterBytes(b, randBytes(b, TestFileSize)) 189} 190func BenchmarkBufferedWriterRandomNoCopy(b *testing.B) { 191 benchmarkBufferedWriterBytesNoCopy(b, randBytes(b, TestFileSize)) 192} 193 194// BenchmarkWriterConstant tests performance encoding maximally compressible 195// data. 196func BenchmarkWriterConstant(b *testing.B) { 197 benchmarkWriterBytes(b, make([]byte, TestFileSize)) 198} 199func BenchmarkBufferedWriterConstant(b *testing.B) { 200 benchmarkBufferedWriterBytes(b, make([]byte, TestFileSize)) 201} 202func BenchmarkBufferedWriterConstantNoCopy(b *testing.B) { 203 benchmarkBufferedWriterBytesNoCopy(b, make([]byte, TestFileSize)) 204} 205 206func benchmarkWriterBytes(b *testing.B, p []byte) { 207 enc := func() io.WriteCloser { 208 // wrap the normal writer so that it has a noop Close method. writer 209 // does not implement ReaderFrom so this does not impact performance. 210 return &nopWriteCloser{NewWriter(ioutil.Discard)} 211 } 212 benchmarkEncode(b, enc, p) 213} 214func benchmarkBufferedWriterBytes(b *testing.B, p []byte) { 215 enc := func() io.WriteCloser { 216 // the writer's ReaderFrom implemention will be used in the benchmark. 217 return NewBufferedWriter(ioutil.Discard) 218 } 219 benchmarkEncode(b, enc, p) 220} 221func benchmarkBufferedWriterBytesNoCopy(b *testing.B, p []byte) { 222 enc := func() io.WriteCloser { 223 // the writer is wrapped as to hide it's ReaderFrom implemention. 224 return &writeCloserNoCopy{NewBufferedWriter(ioutil.Discard)} 225 } 226 benchmarkEncode(b, enc, p) 227} 228 229// benchmarkEncode benchmarks the speed at which bytes can be copied from 230// bs into writers created by enc. 231func benchmarkEncode(b *testing.B, enc func() io.WriteCloser, bs []byte) { 232 size := int64(len(bs)) 233 b.SetBytes(size) 234 b.StartTimer() 235 for i := 0; i < b.N; i++ { 236 w := enc() 237 n, err := io.Copy(w, dummyBytesReader(bs)) 238 if err != nil { 239 b.Fatal(err) 240 } 241 if n != size { 242 b.Fatalf("wrote wrong amount %d != %d", n, size) 243 } 244 err = w.Close() 245 if err != nil { 246 b.Fatalf("close: %v", err) 247 } 248 } 249 b.StopTimer() 250} 251 252func BenchmarkReaderManpage(b *testing.B) { 253 encodeAndBenchmarkReader(b, testDataMan) 254} 255func BenchmarkReaderManpage_buffered(b *testing.B) { 256 encodeAndBenchmarkReader_buffered(b, testDataMan) 257} 258func BenchmarkReaderManpageNoCopy(b *testing.B) { 259 encodeAndBenchmarkReaderNoCopy(b, testDataMan) 260} 261 262func BenchmarkReaderJSON(b *testing.B) { 263 encodeAndBenchmarkReader(b, testDataJSON) 264} 265func BenchmarkReaderJSON_buffered(b *testing.B) { 266 encodeAndBenchmarkReader_buffered(b, testDataJSON) 267} 268func BenchmarkReaderJSONNoCopy(b *testing.B) { 269 encodeAndBenchmarkReaderNoCopy(b, testDataJSON) 270} 271 272// BenchmarkReaderRandom tests decoding of effectively uncompressable data. 273func BenchmarkReaderRandom(b *testing.B) { 274 encodeAndBenchmarkReader(b, randBytes(b, TestFileSize)) 275} 276func BenchmarkReaderRandom_buffered(b *testing.B) { 277 encodeAndBenchmarkReader_buffered(b, randBytes(b, TestFileSize)) 278} 279func BenchmarkReaderRandomNoCopy(b *testing.B) { 280 encodeAndBenchmarkReaderNoCopy(b, randBytes(b, TestFileSize)) 281} 282 283// BenchmarkReaderConstant tests decoding of maximally compressible data. 284func BenchmarkReaderConstant(b *testing.B) { 285 encodeAndBenchmarkReader(b, make([]byte, TestFileSize)) 286} 287func BenchmarkReaderConstant_buffered(b *testing.B) { 288 encodeAndBenchmarkReader_buffered(b, make([]byte, TestFileSize)) 289} 290func BenchmarkReaderConstantNoCopy(b *testing.B) { 291 encodeAndBenchmarkReaderNoCopy(b, make([]byte, TestFileSize)) 292} 293 294// encodeAndBenchmarkReader is a helper that benchmarks the package 295// reader's performance given p encoded as a snappy framed stream. 296// 297// encodeAndBenchmarkReader benchmarks decoding of streams containing 298// (multiple) short frames. 299func encodeAndBenchmarkReader(b *testing.B, p []byte) { 300 enc, err := encodeStreamBytes(p, false) 301 if err != nil { 302 b.Fatalf("pre-benchmark compression: %v", err) 303 } 304 dec := func(r io.Reader) io.Reader { 305 return NewReader(r, VerifyChecksum) 306 } 307 benchmarkDecode(b, dec, int64(len(p)), enc) 308} 309 310// encodeAndBenchmarkReader_buffered is a helper that benchmarks the 311// package reader's performance given p encoded as a snappy framed stream. 312// 313// encodeAndBenchmarkReader_buffered benchmarks decoding of streams that 314// contain at most one short frame (at the end). 315func encodeAndBenchmarkReader_buffered(b *testing.B, p []byte) { 316 enc, err := encodeStreamBytes(p, true) 317 if err != nil { 318 b.Fatalf("pre-benchmark compression: %v", err) 319 } 320 dec := func(r io.Reader) io.Reader { 321 return NewReader(r, VerifyChecksum) 322 } 323 benchmarkDecode(b, dec, int64(len(p)), enc) 324} 325 326// encodeAndBenchmarkReaderNoCopy is a helper that benchmarks the 327// package reader's performance given p encoded as a snappy framed stream. 328// encodeAndBenchmarReaderNoCopy avoids use of the reader's io.WriterTo 329// interface. 330// 331// encodeAndBenchmarkReaderNoCopy benchmarks decoding of streams that 332// contain at most one short frame (at the end). 333func encodeAndBenchmarkReaderNoCopy(b *testing.B, p []byte) { 334 enc, err := encodeStreamBytes(p, true) 335 if err != nil { 336 b.Fatalf("pre-benchmark compression: %v", err) 337 } 338 dec := func(r io.Reader) io.Reader { 339 return ioutil.NopCloser(NewReader(r, VerifyChecksum)) 340 } 341 benchmarkDecode(b, dec, int64(len(p)), enc) 342} 343 344// benchmarkDecode runs a benchmark that repeatedly decoded snappy 345// framed bytes enc. The length of the decoded result in each iteration must 346// equal size. 347func benchmarkDecode(b *testing.B, dec func(io.Reader) io.Reader, size int64, enc []byte) { 348 b.SetBytes(int64(len(enc))) // BUG this is probably wrong 349 b.ResetTimer() 350 for i := 0; i < b.N; i++ { 351 r := dec(bytes.NewReader(enc)) 352 n, err := io.Copy(ioutil.Discard, r) 353 if err != nil { 354 b.Fatalf(err.Error()) 355 } 356 if n != size { 357 b.Fatalf("read wrong amount %d != %d", n, size) 358 } 359 } 360 b.StopTimer() 361} 362 363// encodeStreamBytes is like encodeStream but operates on a byte slice. 364// encodeStreamBytes ensures that long streams are not maximally compressed if 365// buffer is false. 366func encodeStreamBytes(b []byte, buffer bool) ([]byte, error) { 367 return encodeStream(dummyBytesReader(b), buffer) 368} 369 370// encodeStream encodes data read from r as a snappy framed stream and returns 371// the result as a byte slice. if buffer is true the bytes from r are buffered 372// to improve the resulting slice's compression ratio. 373func encodeStream(r io.Reader, buffer bool) ([]byte, error) { 374 var buf bytes.Buffer 375 if !buffer { 376 w := NewWriter(&buf) 377 _, err := io.Copy(w, r) 378 if err != nil { 379 return nil, err 380 } 381 return buf.Bytes(), nil 382 } 383 384 w := NewBufferedWriter(&buf) 385 _, err := io.Copy(w, r) 386 if err != nil { 387 return nil, err 388 } 389 err = w.Close() 390 if err != nil { 391 return nil, err 392 } 393 return buf.Bytes(), nil 394} 395 396// randBytes reads size bytes from the computer's cryptographic random source. 397// the resulting bytes have approximately maximal entropy and are effectively 398// uncompressible with any algorithm. 399func randBytes(b *testing.B, size int) []byte { 400 randp := make([]byte, size) 401 _, err := io.ReadFull(rand.Reader, randp) 402 if err != nil { 403 b.Fatal(err) 404 } 405 return randp 406} 407 408// writeCloserNoCopy is an io.WriteCloser that simply wraps another 409// io.WriteCloser. This is useful for masking implementations for interfaces 410// like ReaderFrom which may be opted into use inside functions like io.Copy. 411type writeCloserNoCopy struct { 412 io.WriteCloser 413} 414 415// nopWriteCloser is an io.WriteCloser that has a noop Close method. This type 416// has the effect of masking the underlying writer's Close implementation if it 417// has one, or satisfying interface implementations for writers that do not 418// need to be closing. 419type nopWriteCloser struct { 420 io.Writer 421} 422 423func (w *nopWriteCloser) Close() error { 424 return nil 425} 426