1// Copyright 2019+ Klaus Post. All rights reserved. 2// License information can be found in the LICENSE file. 3// Based on work by Yann Collet, released under BSD License. 4 5package zstd 6 7import ( 8 "bufio" 9 "bytes" 10 "encoding/binary" 11 "encoding/hex" 12 "errors" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "log" 17 "math/rand" 18 "os" 19 "path/filepath" 20 "reflect" 21 "runtime" 22 "strings" 23 "sync" 24 "testing" 25 "time" 26 27 // "github.com/DataDog/zstd" 28 // zstd "github.com/valyala/gozstd" 29 30 "github.com/klauspost/compress/zip" 31 "github.com/klauspost/compress/zstd/internal/xxhash" 32) 33 34func TestNewReaderMismatch(t *testing.T) { 35 // To identify a potential decoding error, do the following steps: 36 // 1) Place the compressed file in testdata, eg 'testdata/backup.bin.zst' 37 // 2) Decompress the file to using zstd, so it will be named 'testdata/backup.bin' 38 // 3) Run the test. A hash file will be generated 'testdata/backup.bin.hash' 39 // 4) The decoder will also run and decode the file. It will stop as soon as a mismatch is found. 40 // The hash file will be reused between runs if present. 41 const baseFile = "testdata/backup.bin" 42 const blockSize = 1024 43 hashes, err := ioutil.ReadFile(baseFile + ".hash") 44 if os.IsNotExist(err) { 45 // Create the hash file. 46 f, err := os.Open(baseFile) 47 if os.IsNotExist(err) { 48 t.Skip("no decompressed file found") 49 return 50 } 51 defer f.Close() 52 br := bufio.NewReader(f) 53 var tmp [8]byte 54 xx := xxhash.New() 55 for { 56 xx.Reset() 57 buf := make([]byte, blockSize) 58 n, err := io.ReadFull(br, buf) 59 if err != nil { 60 if err != io.EOF && err != io.ErrUnexpectedEOF { 61 t.Fatal(err) 62 } 63 } 64 if n > 0 { 65 _, _ = xx.Write(buf[:n]) 66 binary.LittleEndian.PutUint64(tmp[:], xx.Sum64()) 67 hashes = append(hashes, tmp[4:]...) 68 } 69 if n != blockSize { 70 break 71 } 72 } 73 err = ioutil.WriteFile(baseFile+".hash", hashes, os.ModePerm) 74 if err != nil { 75 // We can continue for now 76 t.Error(err) 77 } 78 t.Log("Saved", len(hashes)/4, "hashes as", baseFile+".hash") 79 } 80 81 f, err := os.Open(baseFile + ".zst") 82 if os.IsNotExist(err) { 83 t.Skip("no compressed file found") 84 return 85 } 86 defer f.Close() 87 dec, err := NewReader(f, WithDecoderConcurrency(1)) 88 if err != nil { 89 t.Fatal(err) 90 } 91 defer dec.Close() 92 var tmp [8]byte 93 xx := xxhash.New() 94 var cHash int 95 for { 96 xx.Reset() 97 buf := make([]byte, blockSize) 98 n, err := io.ReadFull(dec, buf) 99 if err != nil { 100 if err != io.EOF && err != io.ErrUnexpectedEOF { 101 t.Fatal("block", cHash, "err:", err) 102 } 103 } 104 if n > 0 { 105 if cHash+4 > len(hashes) { 106 extra, _ := io.Copy(ioutil.Discard, dec) 107 t.Fatal("not enough hashes (length mismatch). Only have", len(hashes)/4, "hashes. Got block of", n, "bytes and", extra, "bytes still on stream.") 108 } 109 _, _ = xx.Write(buf[:n]) 110 binary.LittleEndian.PutUint64(tmp[:], xx.Sum64()) 111 want, got := hashes[cHash:cHash+4], tmp[4:] 112 if !bytes.Equal(want, got) { 113 org, err := os.Open(baseFile) 114 if err == nil { 115 const sizeBack = 8 << 20 116 defer org.Close() 117 start := int64(cHash)/4*blockSize - sizeBack 118 if start < 0 { 119 start = 0 120 } 121 _, err = org.Seek(start, io.SeekStart) 122 if err != nil { 123 t.Fatal(err) 124 } 125 buf2 := make([]byte, sizeBack+1<<20) 126 n, _ := io.ReadFull(org, buf2) 127 if n > 0 { 128 err = ioutil.WriteFile(baseFile+".section", buf2[:n], os.ModePerm) 129 if err == nil { 130 t.Log("Wrote problematic section to", baseFile+".section") 131 } 132 } 133 } 134 135 t.Fatal("block", cHash/4, "offset", cHash/4*blockSize, "hash mismatch, want:", hex.EncodeToString(want), "got:", hex.EncodeToString(got)) 136 } 137 cHash += 4 138 } 139 if n != blockSize { 140 break 141 } 142 } 143 t.Log("Output matched") 144} 145 146type errorReader struct { 147 err error 148} 149 150func (r *errorReader) Read(p []byte) (int, error) { 151 return 0, r.err 152} 153 154func TestErrorReader(t *testing.T) { 155 wantErr := fmt.Errorf("i'm a failure") 156 zr, err := NewReader(&errorReader{err: wantErr}) 157 if err != nil { 158 t.Fatal(err) 159 } 160 defer zr.Close() 161 162 _, err = ioutil.ReadAll(zr) 163 if !errors.Is(err, wantErr) { 164 t.Errorf("want error %v, got %v", wantErr, err) 165 } 166} 167 168type failingWriter struct { 169 err error 170} 171 172func (f failingWriter) Write(_ []byte) (n int, err error) { 173 return 0, f.err 174} 175 176func TestErrorWriter(t *testing.T) { 177 input := make([]byte, 100) 178 cmp := bytes.Buffer{} 179 w, err := NewWriter(&cmp) 180 if err != nil { 181 t.Fatal(err) 182 } 183 _, _ = rand.Read(input) 184 _, err = w.Write(input) 185 if err != nil { 186 t.Fatal(err) 187 } 188 err = w.Close() 189 if err != nil { 190 t.Fatal(err) 191 } 192 wantErr := fmt.Errorf("i'm a failure") 193 zr, err := NewReader(&cmp) 194 defer zr.Close() 195 out := failingWriter{err: wantErr} 196 _, err = zr.WriteTo(out) 197 if !errors.Is(err, wantErr) { 198 t.Errorf("error: wanted: %v, got: %v", wantErr, err) 199 } 200} 201 202func TestNewDecoder(t *testing.T) { 203 defer timeout(60 * time.Second)() 204 testDecoderFile(t, "testdata/decoder.zip") 205 dec, err := NewReader(nil) 206 if err != nil { 207 t.Fatal(err) 208 } 209 testDecoderDecodeAll(t, "testdata/decoder.zip", dec) 210} 211 212func TestNewDecoderMemory(t *testing.T) { 213 defer timeout(60 * time.Second)() 214 var testdata bytes.Buffer 215 enc, err := NewWriter(&testdata, WithWindowSize(32<<10), WithSingleSegment(false)) 216 if err != nil { 217 t.Fatal(err) 218 } 219 // Write 256KB 220 for i := 0; i < 256; i++ { 221 tmp := strings.Repeat(string([]byte{byte(i)}), 1024) 222 _, err := enc.Write([]byte(tmp)) 223 if err != nil { 224 t.Fatal(err) 225 } 226 } 227 err = enc.Close() 228 if err != nil { 229 t.Fatal(err) 230 } 231 232 var n = 5000 233 if testing.Short() { 234 n = 200 235 } 236 237 // 16K buffer 238 var tmp [16 << 10]byte 239 240 var before, after runtime.MemStats 241 runtime.GC() 242 runtime.ReadMemStats(&before) 243 244 var decs = make([]*Decoder, n) 245 for i := range decs { 246 // Wrap in NopCloser to avoid shortcut. 247 input := ioutil.NopCloser(bytes.NewBuffer(testdata.Bytes())) 248 decs[i], err = NewReader(input, WithDecoderConcurrency(1), WithDecoderLowmem(true)) 249 if err != nil { 250 t.Fatal(err) 251 } 252 } 253 254 for i := range decs { 255 _, err := io.ReadFull(decs[i], tmp[:]) 256 if err != nil { 257 t.Fatal(err) 258 } 259 } 260 261 runtime.GC() 262 runtime.ReadMemStats(&after) 263 size := (after.HeapInuse - before.HeapInuse) / uint64(n) / 1024 264 265 const expect = 124 266 t.Log(size, "KiB per decoder") 267 // This is not exact science, but fail if we suddenly get more than 2x what we expect. 268 if size > expect*2 && !testing.Short() { 269 t.Errorf("expected < %dKB per decoder, got %d", expect, size) 270 } 271 272 for _, dec := range decs { 273 dec.Close() 274 } 275} 276 277func TestNewDecoderMemoryHighMem(t *testing.T) { 278 defer timeout(60 * time.Second)() 279 var testdata bytes.Buffer 280 enc, err := NewWriter(&testdata, WithWindowSize(32<<10), WithSingleSegment(false)) 281 if err != nil { 282 t.Fatal(err) 283 } 284 // Write 256KB 285 for i := 0; i < 256; i++ { 286 tmp := strings.Repeat(string([]byte{byte(i)}), 1024) 287 _, err := enc.Write([]byte(tmp)) 288 if err != nil { 289 t.Fatal(err) 290 } 291 } 292 err = enc.Close() 293 if err != nil { 294 t.Fatal(err) 295 } 296 297 var n = 50 298 if testing.Short() { 299 n = 10 300 } 301 302 // 16K buffer 303 var tmp [16 << 10]byte 304 305 var before, after runtime.MemStats 306 runtime.GC() 307 runtime.ReadMemStats(&before) 308 309 var decs = make([]*Decoder, n) 310 for i := range decs { 311 // Wrap in NopCloser to avoid shortcut. 312 input := ioutil.NopCloser(bytes.NewBuffer(testdata.Bytes())) 313 decs[i], err = NewReader(input, WithDecoderConcurrency(1), WithDecoderLowmem(false)) 314 if err != nil { 315 t.Fatal(err) 316 } 317 } 318 319 for i := range decs { 320 _, err := io.ReadFull(decs[i], tmp[:]) 321 if err != nil { 322 t.Fatal(err) 323 } 324 } 325 326 runtime.GC() 327 runtime.ReadMemStats(&after) 328 size := (after.HeapInuse - before.HeapInuse) / uint64(n) / 1024 329 330 const expect = 3915 331 t.Log(size, "KiB per decoder") 332 // This is not exact science, but fail if we suddenly get more than 2x what we expect. 333 if size > expect*2 && !testing.Short() { 334 t.Errorf("expected < %dKB per decoder, got %d", expect, size) 335 } 336 337 for _, dec := range decs { 338 dec.Close() 339 } 340} 341 342func TestNewDecoderFrameSize(t *testing.T) { 343 defer timeout(60 * time.Second)() 344 var testdata bytes.Buffer 345 enc, err := NewWriter(&testdata, WithWindowSize(64<<10)) 346 if err != nil { 347 t.Fatal(err) 348 } 349 // Write 256KB 350 for i := 0; i < 256; i++ { 351 tmp := strings.Repeat(string([]byte{byte(i)}), 1024) 352 _, err := enc.Write([]byte(tmp)) 353 if err != nil { 354 t.Fatal(err) 355 } 356 } 357 err = enc.Close() 358 if err != nil { 359 t.Fatal(err) 360 } 361 // Must fail 362 dec, err := NewReader(bytes.NewReader(testdata.Bytes()), WithDecoderMaxWindow(32<<10)) 363 if err != nil { 364 t.Fatal(err) 365 } 366 _, err = io.Copy(ioutil.Discard, dec) 367 if err == nil { 368 dec.Close() 369 t.Fatal("Wanted error, got none") 370 } 371 dec.Close() 372 373 // Must succeed. 374 dec, err = NewReader(bytes.NewReader(testdata.Bytes()), WithDecoderMaxWindow(64<<10)) 375 if err != nil { 376 t.Fatal(err) 377 } 378 _, err = io.Copy(ioutil.Discard, dec) 379 if err != nil { 380 dec.Close() 381 t.Fatalf("Wanted no error, got %+v", err) 382 } 383 dec.Close() 384} 385 386func TestNewDecoderGood(t *testing.T) { 387 defer timeout(30 * time.Second)() 388 testDecoderFile(t, "testdata/good.zip") 389 dec, err := NewReader(nil) 390 if err != nil { 391 t.Fatal(err) 392 } 393 testDecoderDecodeAll(t, "testdata/good.zip", dec) 394} 395 396func TestNewDecoderBad(t *testing.T) { 397 defer timeout(10 * time.Second)() 398 dec, err := NewReader(nil) 399 if err != nil { 400 t.Fatal(err) 401 } 402 testDecoderDecodeAllError(t, "testdata/bad.zip", dec) 403} 404 405func TestNewDecoderLarge(t *testing.T) { 406 testDecoderFile(t, "testdata/large.zip") 407 dec, err := NewReader(nil) 408 if err != nil { 409 t.Fatal(err) 410 } 411 testDecoderDecodeAll(t, "testdata/large.zip", dec) 412} 413 414func TestNewReaderRead(t *testing.T) { 415 dec, err := NewReader(nil) 416 if err != nil { 417 t.Fatal(err) 418 } 419 defer dec.Close() 420 _, err = dec.Read([]byte{0}) 421 if err == nil { 422 t.Fatal("Wanted error on uninitialized read, got nil") 423 } 424 t.Log("correctly got error", err) 425} 426 427func TestNewDecoderBig(t *testing.T) { 428 if testing.Short() { 429 t.SkipNow() 430 } 431 file := "testdata/zstd-10kfiles.zip" 432 if _, err := os.Stat(file); os.IsNotExist(err) { 433 t.Skip("To run extended tests, download https://files.klauspost.com/compress/zstd-10kfiles.zip \n" + 434 "and place it in " + file + "\n" + "Running it requires about 5GB of RAM") 435 } 436 testDecoderFile(t, file) 437 dec, err := NewReader(nil) 438 if err != nil { 439 t.Fatal(err) 440 } 441 testDecoderDecodeAll(t, file, dec) 442} 443 444func TestNewDecoderBigFile(t *testing.T) { 445 if testing.Short() { 446 t.SkipNow() 447 } 448 file := "testdata/enwik9.zst" 449 const wantSize = 1000000000 450 if _, err := os.Stat(file); os.IsNotExist(err) { 451 t.Skip("To run extended tests, download http://mattmahoney.net/dc/enwik9.zip unzip it \n" + 452 "compress it with 'zstd -15 -T0 enwik9' and place it in " + file) 453 } 454 f, err := os.Open(file) 455 if err != nil { 456 t.Fatal(err) 457 } 458 defer f.Close() 459 start := time.Now() 460 dec, err := NewReader(f) 461 if err != nil { 462 t.Fatal(err) 463 } 464 n, err := io.Copy(ioutil.Discard, dec) 465 if err != nil { 466 t.Fatal(err) 467 } 468 if n != wantSize { 469 t.Errorf("want size %d, got size %d", wantSize, n) 470 } 471 elapsed := time.Since(start) 472 mbpersec := (float64(n) / (1024 * 1024)) / (float64(elapsed) / (float64(time.Second))) 473 t.Logf("Decoded %d bytes with %f.2 MB/s", n, mbpersec) 474} 475 476func TestNewDecoderSmallFile(t *testing.T) { 477 if testing.Short() { 478 t.SkipNow() 479 } 480 file := "testdata/z000028.zst" 481 const wantSize = 39807 482 f, err := os.Open(file) 483 if err != nil { 484 t.Fatal(err) 485 } 486 defer f.Close() 487 start := time.Now() 488 dec, err := NewReader(f) 489 if err != nil { 490 t.Fatal(err) 491 } 492 defer dec.Close() 493 n, err := io.Copy(ioutil.Discard, dec) 494 if err != nil { 495 t.Fatal(err) 496 } 497 if n != wantSize { 498 t.Errorf("want size %d, got size %d", wantSize, n) 499 } 500 mbpersec := (float64(n) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second))) 501 t.Logf("Decoded %d bytes with %f.2 MB/s", n, mbpersec) 502} 503 504type readAndBlock struct { 505 buf []byte 506 unblock chan struct{} 507} 508 509func (r *readAndBlock) Read(p []byte) (int, error) { 510 n := copy(p, r.buf) 511 if n == 0 { 512 <-r.unblock 513 return 0, io.EOF 514 } 515 r.buf = r.buf[n:] 516 return n, nil 517} 518 519func TestNewDecoderFlushed(t *testing.T) { 520 if testing.Short() { 521 t.SkipNow() 522 } 523 file := "testdata/z000028.zst" 524 payload, err := ioutil.ReadFile(file) 525 if err != nil { 526 t.Fatal(err) 527 } 528 payload = append(payload, payload...) //2x 529 payload = append(payload, payload...) //4x 530 payload = append(payload, payload...) //8x 531 rng := rand.New(rand.NewSource(0x1337)) 532 runs := 100 533 if testing.Short() { 534 runs = 5 535 } 536 enc, err := NewWriter(nil, WithWindowSize(128<<10)) 537 if err != nil { 538 t.Fatal(err) 539 } 540 defer enc.Close() 541 for i := 0; i < runs; i++ { 542 wantSize := rng.Intn(len(payload)-1) + 1 543 t.Run(fmt.Sprint("size-", wantSize), func(t *testing.T) { 544 var encoded bytes.Buffer 545 enc.Reset(&encoded) 546 _, err := enc.Write(payload[:wantSize]) 547 if err != nil { 548 t.Fatal(err) 549 } 550 err = enc.Flush() 551 if err != nil { 552 t.Fatal(err) 553 } 554 555 // We must be able to read back up until the flush... 556 r := readAndBlock{ 557 buf: encoded.Bytes(), 558 unblock: make(chan struct{}), 559 } 560 defer timeout(5 * time.Second)() 561 dec, err := NewReader(&r) 562 if err != nil { 563 t.Fatal(err) 564 } 565 defer dec.Close() 566 defer close(r.unblock) 567 readBack := 0 568 dst := make([]byte, 1024) 569 for readBack < wantSize { 570 // Read until we have enough. 571 n, err := dec.Read(dst) 572 if err != nil { 573 t.Fatal(err) 574 } 575 readBack += n 576 } 577 }) 578 } 579} 580 581func TestDecoderRegression(t *testing.T) { 582 defer timeout(160 * time.Second)() 583 data, err := ioutil.ReadFile("testdata/regression.zip") 584 if err != nil { 585 t.Fatal(err) 586 } 587 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 588 if err != nil { 589 t.Fatal(err) 590 } 591 dec, err := NewReader(nil, WithDecoderConcurrency(1), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<20)) 592 if err != nil { 593 t.Error(err) 594 return 595 } 596 defer dec.Close() 597 for i, tt := range zr.File { 598 if !strings.HasSuffix(tt.Name, "artifact (5)") || (testing.Short() && i > 10) { 599 continue 600 } 601 t.Run("Reader-"+tt.Name, func(t *testing.T) { 602 r, err := tt.Open() 603 if err != nil { 604 t.Error(err) 605 return 606 } 607 err = dec.Reset(r) 608 if err != nil { 609 t.Error(err) 610 return 611 } 612 got, gotErr := ioutil.ReadAll(dec) 613 t.Log("Received:", len(got), gotErr) 614 615 // Check a fresh instance 616 r, err = tt.Open() 617 if err != nil { 618 t.Error(err) 619 return 620 } 621 decL, err := NewReader(r, WithDecoderConcurrency(1), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<20)) 622 if err != nil { 623 t.Error(err) 624 return 625 } 626 defer decL.Close() 627 got2, gotErr2 := ioutil.ReadAll(decL) 628 t.Log("Fresh Reader received:", len(got2), gotErr2) 629 if gotErr != gotErr2 { 630 if gotErr != nil && gotErr2 != nil && gotErr.Error() != gotErr2.Error() { 631 t.Error(gotErr, "!=", gotErr2) 632 } 633 if (gotErr == nil) != (gotErr2 == nil) { 634 t.Error(gotErr, "!=", gotErr2) 635 } 636 } 637 if !bytes.Equal(got2, got) { 638 if gotErr != nil { 639 t.Log("Buffer mismatch without Reset") 640 } else { 641 t.Error("Buffer mismatch without Reset") 642 } 643 } 644 }) 645 t.Run("DecodeAll-"+tt.Name, func(t *testing.T) { 646 r, err := tt.Open() 647 if err != nil { 648 t.Error(err) 649 return 650 } 651 in, err := ioutil.ReadAll(r) 652 if err != nil { 653 t.Error(err) 654 } 655 got, gotErr := dec.DecodeAll(in, nil) 656 t.Log("Received:", len(got), gotErr) 657 658 // Check if we got the same: 659 decL, err := NewReader(nil, WithDecoderConcurrency(1), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<20)) 660 if err != nil { 661 t.Error(err) 662 return 663 } 664 defer decL.Close() 665 got2, gotErr2 := decL.DecodeAll(in, nil) 666 t.Log("Fresh Reader received:", len(got2), gotErr2) 667 if gotErr != gotErr2 { 668 if gotErr != nil && gotErr2 != nil && gotErr.Error() != gotErr2.Error() { 669 t.Error(gotErr, "!=", gotErr2) 670 } 671 if (gotErr == nil) != (gotErr2 == nil) { 672 t.Error(gotErr, "!=", gotErr2) 673 } 674 } 675 if !bytes.Equal(got2, got) { 676 if gotErr != nil { 677 t.Log("Buffer mismatch without Reset") 678 } else { 679 t.Error("Buffer mismatch without Reset") 680 } 681 } 682 }) 683 t.Run("Match-"+tt.Name, func(t *testing.T) { 684 r, err := tt.Open() 685 if err != nil { 686 t.Error(err) 687 return 688 } 689 in, err := ioutil.ReadAll(r) 690 if err != nil { 691 t.Error(err) 692 } 693 got, gotErr := dec.DecodeAll(in, nil) 694 t.Log("Received:", len(got), gotErr) 695 696 // Check a fresh instance 697 decL, err := NewReader(bytes.NewBuffer(in), WithDecoderConcurrency(1), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<20)) 698 if err != nil { 699 t.Error(err) 700 return 701 } 702 defer decL.Close() 703 got2, gotErr2 := ioutil.ReadAll(decL) 704 t.Log("Reader Reader received:", len(got2), gotErr2) 705 if gotErr != gotErr2 { 706 if gotErr != nil && gotErr2 != nil && gotErr.Error() != gotErr2.Error() { 707 t.Error(gotErr, "!=", gotErr2) 708 } 709 if (gotErr == nil) != (gotErr2 == nil) { 710 t.Error(gotErr, "!=", gotErr2) 711 } 712 } 713 if !bytes.Equal(got2, got) { 714 if gotErr != nil { 715 t.Log("Buffer mismatch") 716 } else { 717 t.Error("Buffer mismatch") 718 } 719 } 720 }) 721 } 722} 723 724func TestShort(t *testing.T) { 725 for _, in := range []string{"f", "fo", "foo"} { 726 inb := []byte(in) 727 dec, err := NewReader(nil) 728 if err != nil { 729 t.Fatal(err) 730 } 731 defer dec.Close() 732 733 t.Run(fmt.Sprintf("DecodeAll-%d", len(in)), func(t *testing.T) { 734 _, err := dec.DecodeAll(inb, nil) 735 if err == nil { 736 t.Error("want error, got nil") 737 } 738 }) 739 t.Run(fmt.Sprintf("Reader-%d", len(in)), func(t *testing.T) { 740 dec.Reset(bytes.NewReader(inb)) 741 _, err := io.Copy(ioutil.Discard, dec) 742 if err == nil { 743 t.Error("want error, got nil") 744 } 745 }) 746 } 747} 748 749func TestDecoder_Reset(t *testing.T) { 750 in, err := ioutil.ReadFile("testdata/z000028") 751 if err != nil { 752 t.Fatal(err) 753 } 754 in = append(in, in...) 755 var e Encoder 756 start := time.Now() 757 dst := e.EncodeAll(in, nil) 758 t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst)) 759 mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second))) 760 t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec) 761 762 dec, err := NewReader(nil) 763 if err != nil { 764 t.Fatal(err) 765 } 766 defer dec.Close() 767 decoded, err := dec.DecodeAll(dst, nil) 768 if err != nil { 769 t.Error(err, len(decoded)) 770 } 771 if !bytes.Equal(decoded, in) { 772 t.Fatal("Decoded does not match") 773 } 774 t.Log("Encoded content matched") 775 776 // Decode using reset+copy 777 for i := 0; i < 3; i++ { 778 err = dec.Reset(bytes.NewBuffer(dst)) 779 if err != nil { 780 t.Fatal(err) 781 } 782 var dBuf bytes.Buffer 783 n, err := io.Copy(&dBuf, dec) 784 if err != nil { 785 t.Fatal(err) 786 } 787 decoded = dBuf.Bytes() 788 if int(n) != len(decoded) { 789 t.Fatalf("decoded reported length mismatch %d != %d", n, len(decoded)) 790 } 791 if !bytes.Equal(decoded, in) { 792 ioutil.WriteFile("testdata/"+t.Name()+"-z000028.got", decoded, os.ModePerm) 793 ioutil.WriteFile("testdata/"+t.Name()+"-z000028.want", in, os.ModePerm) 794 t.Fatal("Decoded does not match") 795 } 796 } 797 // Test without WriterTo interface support. 798 for i := 0; i < 3; i++ { 799 err = dec.Reset(bytes.NewBuffer(dst)) 800 if err != nil { 801 t.Fatal(err) 802 } 803 decoded, err := ioutil.ReadAll(ioutil.NopCloser(dec)) 804 if err != nil { 805 t.Fatal(err) 806 } 807 if !bytes.Equal(decoded, in) { 808 ioutil.WriteFile("testdata/"+t.Name()+"-z000028.got", decoded, os.ModePerm) 809 ioutil.WriteFile("testdata/"+t.Name()+"-z000028.want", in, os.ModePerm) 810 t.Fatal("Decoded does not match") 811 } 812 } 813} 814 815func TestDecoderMultiFrame(t *testing.T) { 816 fn := "testdata/benchdecoder.zip" 817 data, err := ioutil.ReadFile(fn) 818 if err != nil { 819 t.Fatal(err) 820 } 821 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 822 if err != nil { 823 t.Fatal(err) 824 } 825 dec, err := NewReader(nil) 826 if err != nil { 827 t.Fatal(err) 828 return 829 } 830 defer dec.Close() 831 for _, tt := range zr.File { 832 if !strings.HasSuffix(tt.Name, ".zst") { 833 continue 834 } 835 t.Run(tt.Name, func(t *testing.T) { 836 r, err := tt.Open() 837 if err != nil { 838 t.Fatal(err) 839 } 840 defer r.Close() 841 in, err := ioutil.ReadAll(r) 842 if err != nil { 843 t.Fatal(err) 844 } 845 // 2x 846 in = append(in, in...) 847 if !testing.Short() { 848 // 4x 849 in = append(in, in...) 850 // 8x 851 in = append(in, in...) 852 } 853 err = dec.Reset(bytes.NewBuffer(in)) 854 if err != nil { 855 t.Fatal(err) 856 } 857 got, err := ioutil.ReadAll(dec) 858 if err != nil { 859 t.Fatal(err) 860 } 861 err = dec.Reset(bytes.NewBuffer(in)) 862 if err != nil { 863 t.Fatal(err) 864 } 865 got2, err := ioutil.ReadAll(dec) 866 if err != nil { 867 t.Fatal(err) 868 } 869 if !bytes.Equal(got, got2) { 870 t.Error("results mismatch") 871 } 872 }) 873 } 874} 875 876func TestDecoderMultiFrameReset(t *testing.T) { 877 fn := "testdata/benchdecoder.zip" 878 data, err := ioutil.ReadFile(fn) 879 if err != nil { 880 t.Fatal(err) 881 } 882 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 883 if err != nil { 884 t.Fatal(err) 885 } 886 dec, err := NewReader(nil) 887 if err != nil { 888 t.Fatal(err) 889 return 890 } 891 rng := rand.New(rand.NewSource(1337)) 892 defer dec.Close() 893 for _, tt := range zr.File { 894 if !strings.HasSuffix(tt.Name, ".zst") { 895 continue 896 } 897 t.Run(tt.Name, func(t *testing.T) { 898 r, err := tt.Open() 899 if err != nil { 900 t.Fatal(err) 901 } 902 defer r.Close() 903 in, err := ioutil.ReadAll(r) 904 if err != nil { 905 t.Fatal(err) 906 } 907 // 2x 908 in = append(in, in...) 909 if !testing.Short() { 910 // 4x 911 in = append(in, in...) 912 // 8x 913 in = append(in, in...) 914 } 915 err = dec.Reset(bytes.NewBuffer(in)) 916 if err != nil { 917 t.Fatal(err) 918 } 919 got, err := ioutil.ReadAll(dec) 920 if err != nil { 921 t.Fatal(err) 922 } 923 err = dec.Reset(bytes.NewBuffer(in)) 924 if err != nil { 925 t.Fatal(err) 926 } 927 // Read a random number of bytes 928 tmp := make([]byte, rng.Intn(len(got))) 929 _, err = io.ReadAtLeast(dec, tmp, len(tmp)) 930 if err != nil { 931 t.Fatal(err) 932 } 933 err = dec.Reset(bytes.NewBuffer(in)) 934 if err != nil { 935 t.Fatal(err) 936 } 937 got2, err := ioutil.ReadAll(dec) 938 if err != nil { 939 t.Fatal(err) 940 } 941 if !bytes.Equal(got, got2) { 942 t.Error("results mismatch") 943 } 944 }) 945 } 946} 947 948func testDecoderFile(t *testing.T, fn string) { 949 data, err := ioutil.ReadFile(fn) 950 if err != nil { 951 t.Fatal(err) 952 } 953 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 954 if err != nil { 955 t.Fatal(err) 956 } 957 var want = make(map[string][]byte) 958 for _, tt := range zr.File { 959 if strings.HasSuffix(tt.Name, ".zst") { 960 continue 961 } 962 r, err := tt.Open() 963 if err != nil { 964 t.Fatal(err) 965 return 966 } 967 want[tt.Name+".zst"], _ = ioutil.ReadAll(r) 968 } 969 970 dec, err := NewReader(nil) 971 if err != nil { 972 t.Error(err) 973 return 974 } 975 defer dec.Close() 976 for i, tt := range zr.File { 977 if !strings.HasSuffix(tt.Name, ".zst") || (testing.Short() && i > 20) { 978 continue 979 } 980 t.Run("Reader-"+tt.Name, func(t *testing.T) { 981 r, err := tt.Open() 982 if err != nil { 983 t.Error(err) 984 return 985 } 986 defer r.Close() 987 err = dec.Reset(r) 988 if err != nil { 989 t.Error(err) 990 return 991 } 992 got, err := ioutil.ReadAll(dec) 993 if err != nil { 994 t.Error(err) 995 if err != ErrCRCMismatch { 996 return 997 } 998 } 999 wantB := want[tt.Name] 1000 if !bytes.Equal(wantB, got) { 1001 if len(wantB)+len(got) < 1000 { 1002 t.Logf(" got: %v\nwant: %v", got, wantB) 1003 } else { 1004 fileName, _ := filepath.Abs(filepath.Join("testdata", t.Name()+"-want.bin")) 1005 _ = os.MkdirAll(filepath.Dir(fileName), os.ModePerm) 1006 err := ioutil.WriteFile(fileName, wantB, os.ModePerm) 1007 t.Log("Wrote file", fileName, err) 1008 1009 fileName, _ = filepath.Abs(filepath.Join("testdata", t.Name()+"-got.bin")) 1010 _ = os.MkdirAll(filepath.Dir(fileName), os.ModePerm) 1011 err = ioutil.WriteFile(fileName, got, os.ModePerm) 1012 t.Log("Wrote file", fileName, err) 1013 } 1014 t.Logf("Length, want: %d, got: %d", len(wantB), len(got)) 1015 t.Error("Output mismatch") 1016 return 1017 } 1018 t.Log(len(got), "bytes returned, matches input, ok!") 1019 }) 1020 } 1021} 1022 1023func BenchmarkDecoder_DecoderSmall(b *testing.B) { 1024 fn := "testdata/benchdecoder.zip" 1025 data, err := ioutil.ReadFile(fn) 1026 if err != nil { 1027 b.Fatal(err) 1028 } 1029 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1030 if err != nil { 1031 b.Fatal(err) 1032 } 1033 dec, err := NewReader(nil) 1034 if err != nil { 1035 b.Fatal(err) 1036 return 1037 } 1038 defer dec.Close() 1039 for _, tt := range zr.File { 1040 if !strings.HasSuffix(tt.Name, ".zst") { 1041 continue 1042 } 1043 b.Run(tt.Name, func(b *testing.B) { 1044 r, err := tt.Open() 1045 if err != nil { 1046 b.Fatal(err) 1047 } 1048 defer r.Close() 1049 in, err := ioutil.ReadAll(r) 1050 if err != nil { 1051 b.Fatal(err) 1052 } 1053 // 2x 1054 in = append(in, in...) 1055 // 4x 1056 in = append(in, in...) 1057 // 8x 1058 in = append(in, in...) 1059 err = dec.Reset(bytes.NewBuffer(in)) 1060 if err != nil { 1061 b.Fatal(err) 1062 } 1063 got, err := ioutil.ReadAll(dec) 1064 if err != nil { 1065 b.Fatal(err) 1066 } 1067 b.SetBytes(int64(len(got))) 1068 b.ReportAllocs() 1069 b.ResetTimer() 1070 for i := 0; i < b.N; i++ { 1071 err = dec.Reset(bytes.NewBuffer(in)) 1072 if err != nil { 1073 b.Fatal(err) 1074 } 1075 _, err := io.Copy(ioutil.Discard, dec) 1076 if err != nil { 1077 b.Fatal(err) 1078 } 1079 } 1080 }) 1081 } 1082} 1083 1084func BenchmarkDecoder_DecodeAll(b *testing.B) { 1085 fn := "testdata/benchdecoder.zip" 1086 data, err := ioutil.ReadFile(fn) 1087 if err != nil { 1088 b.Fatal(err) 1089 } 1090 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1091 if err != nil { 1092 b.Fatal(err) 1093 } 1094 dec, err := NewReader(nil, WithDecoderConcurrency(1)) 1095 if err != nil { 1096 b.Fatal(err) 1097 return 1098 } 1099 defer dec.Close() 1100 for _, tt := range zr.File { 1101 if !strings.HasSuffix(tt.Name, ".zst") { 1102 continue 1103 } 1104 b.Run(tt.Name, func(b *testing.B) { 1105 r, err := tt.Open() 1106 if err != nil { 1107 b.Fatal(err) 1108 } 1109 defer r.Close() 1110 in, err := ioutil.ReadAll(r) 1111 if err != nil { 1112 b.Fatal(err) 1113 } 1114 got, err := dec.DecodeAll(in, nil) 1115 if err != nil { 1116 b.Fatal(err) 1117 } 1118 b.SetBytes(int64(len(got))) 1119 b.ReportAllocs() 1120 b.ResetTimer() 1121 for i := 0; i < b.N; i++ { 1122 _, err = dec.DecodeAll(in, got[:0]) 1123 if err != nil { 1124 b.Fatal(err) 1125 } 1126 } 1127 }) 1128 } 1129} 1130 1131func BenchmarkDecoder_DecodeAllParallel(b *testing.B) { 1132 fn := "testdata/benchdecoder.zip" 1133 data, err := ioutil.ReadFile(fn) 1134 if err != nil { 1135 b.Fatal(err) 1136 } 1137 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1138 if err != nil { 1139 b.Fatal(err) 1140 } 1141 dec, err := NewReader(nil) 1142 if err != nil { 1143 b.Fatal(err) 1144 return 1145 } 1146 defer dec.Close() 1147 for _, tt := range zr.File { 1148 if !strings.HasSuffix(tt.Name, ".zst") { 1149 continue 1150 } 1151 b.Run(tt.Name, func(b *testing.B) { 1152 r, err := tt.Open() 1153 if err != nil { 1154 b.Fatal(err) 1155 } 1156 defer r.Close() 1157 in, err := ioutil.ReadAll(r) 1158 if err != nil { 1159 b.Fatal(err) 1160 } 1161 got, err := dec.DecodeAll(in, nil) 1162 if err != nil { 1163 b.Fatal(err) 1164 } 1165 b.SetBytes(int64(len(got))) 1166 b.ReportAllocs() 1167 b.ResetTimer() 1168 b.RunParallel(func(pb *testing.PB) { 1169 got := make([]byte, len(got)) 1170 for pb.Next() { 1171 _, err = dec.DecodeAll(in, got[:0]) 1172 if err != nil { 1173 b.Fatal(err) 1174 } 1175 } 1176 }) 1177 }) 1178 } 1179} 1180 1181/* 1182func BenchmarkDecoder_DecodeAllCgo(b *testing.B) { 1183 fn := "testdata/benchdecoder.zip" 1184 data, err := ioutil.ReadFile(fn) 1185 if err != nil { 1186 b.Fatal(err) 1187 } 1188 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1189 if err != nil { 1190 b.Fatal(err) 1191 } 1192 for _, tt := range zr.File { 1193 if !strings.HasSuffix(tt.Name, ".zst") { 1194 continue 1195 } 1196 b.Run(tt.Name, func(b *testing.B) { 1197 tt := tt 1198 r, err := tt.Open() 1199 if err != nil { 1200 b.Fatal(err) 1201 } 1202 defer r.Close() 1203 in, err := ioutil.ReadAll(r) 1204 if err != nil { 1205 b.Fatal(err) 1206 } 1207 got, err := zstd.Decompress(nil, in) 1208 if err != nil { 1209 b.Fatal(err) 1210 } 1211 b.SetBytes(int64(len(got))) 1212 b.ReportAllocs() 1213 b.ResetTimer() 1214 for i := 0; i < b.N; i++ { 1215 got, err = zstd.Decompress(got, in) 1216 if err != nil { 1217 b.Fatal(err) 1218 } 1219 } 1220 }) 1221 } 1222} 1223 1224func BenchmarkDecoder_DecodeAllParallelCgo(b *testing.B) { 1225 fn := "testdata/benchdecoder.zip" 1226 data, err := ioutil.ReadFile(fn) 1227 if err != nil { 1228 b.Fatal(err) 1229 } 1230 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1231 if err != nil { 1232 b.Fatal(err) 1233 } 1234 for _, tt := range zr.File { 1235 if !strings.HasSuffix(tt.Name, ".zst") { 1236 continue 1237 } 1238 b.Run(tt.Name, func(b *testing.B) { 1239 r, err := tt.Open() 1240 if err != nil { 1241 b.Fatal(err) 1242 } 1243 defer r.Close() 1244 in, err := ioutil.ReadAll(r) 1245 if err != nil { 1246 b.Fatal(err) 1247 } 1248 got, err := zstd.Decompress(nil, in) 1249 if err != nil { 1250 b.Fatal(err) 1251 } 1252 b.SetBytes(int64(len(got))) 1253 b.ReportAllocs() 1254 b.ResetTimer() 1255 b.RunParallel(func(pb *testing.PB) { 1256 got := make([]byte, len(got)) 1257 for pb.Next() { 1258 got, err = zstd.Decompress(got, in) 1259 if err != nil { 1260 b.Fatal(err) 1261 } 1262 } 1263 }) 1264 }) 1265 } 1266} 1267 1268func BenchmarkDecoderSilesiaCgo(b *testing.B) { 1269 fn := "testdata/silesia.tar.zst" 1270 data, err := ioutil.ReadFile(fn) 1271 if err != nil { 1272 if os.IsNotExist(err) { 1273 b.Skip("Missing testdata/silesia.tar.zst") 1274 return 1275 } 1276 b.Fatal(err) 1277 } 1278 dec := zstd.NewReader(bytes.NewBuffer(data)) 1279 n, err := io.Copy(ioutil.Discard, dec) 1280 if err != nil { 1281 b.Fatal(err) 1282 } 1283 1284 b.SetBytes(n) 1285 b.ReportAllocs() 1286 b.ResetTimer() 1287 for i := 0; i < b.N; i++ { 1288 dec := zstd.NewReader(bytes.NewBuffer(data)) 1289 _, err := io.CopyN(ioutil.Discard, dec, n) 1290 if err != nil { 1291 b.Fatal(err) 1292 } 1293 } 1294} 1295func BenchmarkDecoderEnwik9Cgo(b *testing.B) { 1296 fn := "testdata/enwik9-1.zst" 1297 data, err := ioutil.ReadFile(fn) 1298 if err != nil { 1299 if os.IsNotExist(err) { 1300 b.Skip("Missing " + fn) 1301 return 1302 } 1303 b.Fatal(err) 1304 } 1305 dec := zstd.NewReader(bytes.NewBuffer(data)) 1306 n, err := io.Copy(ioutil.Discard, dec) 1307 if err != nil { 1308 b.Fatal(err) 1309 } 1310 1311 b.SetBytes(n) 1312 b.ReportAllocs() 1313 b.ResetTimer() 1314 for i := 0; i < b.N; i++ { 1315 dec := zstd.NewReader(bytes.NewBuffer(data)) 1316 _, err := io.CopyN(ioutil.Discard, dec, n) 1317 if err != nil { 1318 b.Fatal(err) 1319 } 1320 } 1321} 1322 1323*/ 1324 1325func BenchmarkDecoderSilesia(b *testing.B) { 1326 fn := "testdata/silesia.tar.zst" 1327 data, err := ioutil.ReadFile(fn) 1328 if err != nil { 1329 if os.IsNotExist(err) { 1330 b.Skip("Missing testdata/silesia.tar.zst") 1331 return 1332 } 1333 b.Fatal(err) 1334 } 1335 dec, err := NewReader(nil, WithDecoderLowmem(false)) 1336 if err != nil { 1337 b.Fatal(err) 1338 } 1339 defer dec.Close() 1340 err = dec.Reset(bytes.NewBuffer(data)) 1341 if err != nil { 1342 b.Fatal(err) 1343 } 1344 n, err := io.Copy(ioutil.Discard, dec) 1345 if err != nil { 1346 b.Fatal(err) 1347 } 1348 1349 b.SetBytes(n) 1350 b.ReportAllocs() 1351 b.ResetTimer() 1352 for i := 0; i < b.N; i++ { 1353 err = dec.Reset(bytes.NewBuffer(data)) 1354 if err != nil { 1355 b.Fatal(err) 1356 } 1357 _, err := io.CopyN(ioutil.Discard, dec, n) 1358 if err != nil { 1359 b.Fatal(err) 1360 } 1361 } 1362} 1363 1364func BenchmarkDecoderEnwik9(b *testing.B) { 1365 fn := "testdata/enwik9-1.zst" 1366 data, err := ioutil.ReadFile(fn) 1367 if err != nil { 1368 if os.IsNotExist(err) { 1369 b.Skip("Missing " + fn) 1370 return 1371 } 1372 b.Fatal(err) 1373 } 1374 dec, err := NewReader(nil, WithDecoderLowmem(false)) 1375 if err != nil { 1376 b.Fatal(err) 1377 } 1378 defer dec.Close() 1379 err = dec.Reset(bytes.NewBuffer(data)) 1380 if err != nil { 1381 b.Fatal(err) 1382 } 1383 n, err := io.Copy(ioutil.Discard, dec) 1384 if err != nil { 1385 b.Fatal(err) 1386 } 1387 1388 b.SetBytes(n) 1389 b.ReportAllocs() 1390 b.ResetTimer() 1391 for i := 0; i < b.N; i++ { 1392 err = dec.Reset(bytes.NewBuffer(data)) 1393 if err != nil { 1394 b.Fatal(err) 1395 } 1396 _, err := io.CopyN(ioutil.Discard, dec, n) 1397 if err != nil { 1398 b.Fatal(err) 1399 } 1400 } 1401} 1402 1403func testDecoderDecodeAll(t *testing.T, fn string, dec *Decoder) { 1404 data, err := ioutil.ReadFile(fn) 1405 if err != nil { 1406 t.Fatal(err) 1407 } 1408 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1409 if err != nil { 1410 t.Fatal(err) 1411 } 1412 var want = make(map[string][]byte) 1413 for _, tt := range zr.File { 1414 if strings.HasSuffix(tt.Name, ".zst") { 1415 continue 1416 } 1417 r, err := tt.Open() 1418 if err != nil { 1419 t.Fatal(err) 1420 return 1421 } 1422 want[tt.Name+".zst"], _ = ioutil.ReadAll(r) 1423 } 1424 var wg sync.WaitGroup 1425 for i, tt := range zr.File { 1426 tt := tt 1427 if !strings.HasSuffix(tt.Name, ".zst") || (testing.Short() && i > 20) { 1428 continue 1429 } 1430 wg.Add(1) 1431 t.Run("DecodeAll-"+tt.Name, func(t *testing.T) { 1432 defer wg.Done() 1433 t.Parallel() 1434 r, err := tt.Open() 1435 if err != nil { 1436 t.Fatal(err) 1437 } 1438 in, err := ioutil.ReadAll(r) 1439 if err != nil { 1440 t.Fatal(err) 1441 } 1442 wantB := want[tt.Name] 1443 // make a buffer that is too small. 1444 got, err := dec.DecodeAll(in, make([]byte, 10, 200)) 1445 if err != nil { 1446 t.Error(err) 1447 } 1448 if len(got) < 10 { 1449 t.Fatal("didn't get input back") 1450 } 1451 got = got[10:] 1452 if !bytes.Equal(wantB, got) { 1453 if len(wantB)+len(got) < 1000 { 1454 t.Logf(" got: %v\nwant: %v", got, wantB) 1455 } else { 1456 fileName, _ := filepath.Abs(filepath.Join("testdata", t.Name()+"-want.bin")) 1457 _ = os.MkdirAll(filepath.Dir(fileName), os.ModePerm) 1458 err := ioutil.WriteFile(fileName, wantB, os.ModePerm) 1459 t.Log("Wrote file", fileName, err) 1460 1461 fileName, _ = filepath.Abs(filepath.Join("testdata", t.Name()+"-got.bin")) 1462 _ = os.MkdirAll(filepath.Dir(fileName), os.ModePerm) 1463 err = ioutil.WriteFile(fileName, got, os.ModePerm) 1464 t.Log("Wrote file", fileName, err) 1465 } 1466 t.Logf("Length, want: %d, got: %d", len(wantB), len(got)) 1467 t.Error("Output mismatch") 1468 return 1469 } 1470 t.Log(len(got), "bytes returned, matches input, ok!") 1471 }) 1472 } 1473 go func() { 1474 wg.Wait() 1475 dec.Close() 1476 }() 1477} 1478 1479func testDecoderDecodeAllError(t *testing.T, fn string, dec *Decoder) { 1480 data, err := ioutil.ReadFile(fn) 1481 if err != nil { 1482 t.Fatal(err) 1483 } 1484 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 1485 if err != nil { 1486 t.Fatal(err) 1487 } 1488 1489 var wg sync.WaitGroup 1490 for _, tt := range zr.File { 1491 tt := tt 1492 if !strings.HasSuffix(tt.Name, ".zst") { 1493 continue 1494 } 1495 wg.Add(1) 1496 t.Run("DecodeAll-"+tt.Name, func(t *testing.T) { 1497 defer wg.Done() 1498 t.Parallel() 1499 r, err := tt.Open() 1500 if err != nil { 1501 t.Fatal(err) 1502 } 1503 in, err := ioutil.ReadAll(r) 1504 if err != nil { 1505 t.Fatal(err) 1506 } 1507 // make a buffer that is too small. 1508 _, err = dec.DecodeAll(in, make([]byte, 0, 200)) 1509 if err == nil { 1510 t.Error("Did not get expected error") 1511 } 1512 }) 1513 } 1514 go func() { 1515 wg.Wait() 1516 dec.Close() 1517 }() 1518} 1519 1520// Test our predefined tables are correct. 1521// We don't predefine them, since this also tests our transformations. 1522// Reference from here: https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L234 1523func TestPredefTables(t *testing.T) { 1524 initPredefined() 1525 x := func(nextState uint16, nbAddBits, nbBits uint8, baseVal uint32) decSymbol { 1526 return newDecSymbol(nbBits, nbAddBits, nextState, baseVal) 1527 } 1528 for i := range fsePredef[:] { 1529 var want []decSymbol 1530 switch tableIndex(i) { 1531 case tableLiteralLengths: 1532 want = []decSymbol{ 1533 /* nextState, nbAddBits, nbBits, baseVal */ 1534 x(0, 0, 4, 0), x(16, 0, 4, 0), 1535 x(32, 0, 5, 1), x(0, 0, 5, 3), 1536 x(0, 0, 5, 4), x(0, 0, 5, 6), 1537 x(0, 0, 5, 7), x(0, 0, 5, 9), 1538 x(0, 0, 5, 10), x(0, 0, 5, 12), 1539 x(0, 0, 6, 14), x(0, 1, 5, 16), 1540 x(0, 1, 5, 20), x(0, 1, 5, 22), 1541 x(0, 2, 5, 28), x(0, 3, 5, 32), 1542 x(0, 4, 5, 48), x(32, 6, 5, 64), 1543 x(0, 7, 5, 128), x(0, 8, 6, 256), 1544 x(0, 10, 6, 1024), x(0, 12, 6, 4096), 1545 x(32, 0, 4, 0), x(0, 0, 4, 1), 1546 x(0, 0, 5, 2), x(32, 0, 5, 4), 1547 x(0, 0, 5, 5), x(32, 0, 5, 7), 1548 x(0, 0, 5, 8), x(32, 0, 5, 10), 1549 x(0, 0, 5, 11), x(0, 0, 6, 13), 1550 x(32, 1, 5, 16), x(0, 1, 5, 18), 1551 x(32, 1, 5, 22), x(0, 2, 5, 24), 1552 x(32, 3, 5, 32), x(0, 3, 5, 40), 1553 x(0, 6, 4, 64), x(16, 6, 4, 64), 1554 x(32, 7, 5, 128), x(0, 9, 6, 512), 1555 x(0, 11, 6, 2048), x(48, 0, 4, 0), 1556 x(16, 0, 4, 1), x(32, 0, 5, 2), 1557 x(32, 0, 5, 3), x(32, 0, 5, 5), 1558 x(32, 0, 5, 6), x(32, 0, 5, 8), 1559 x(32, 0, 5, 9), x(32, 0, 5, 11), 1560 x(32, 0, 5, 12), x(0, 0, 6, 15), 1561 x(32, 1, 5, 18), x(32, 1, 5, 20), 1562 x(32, 2, 5, 24), x(32, 2, 5, 28), 1563 x(32, 3, 5, 40), x(32, 4, 5, 48), 1564 x(0, 16, 6, 65536), x(0, 15, 6, 32768), 1565 x(0, 14, 6, 16384), x(0, 13, 6, 8192), 1566 } 1567 case tableOffsets: 1568 want = []decSymbol{ 1569 /* nextState, nbAddBits, nbBits, baseVal */ 1570 x(0, 0, 5, 0), x(0, 6, 4, 61), 1571 x(0, 9, 5, 509), x(0, 15, 5, 32765), 1572 x(0, 21, 5, 2097149), x(0, 3, 5, 5), 1573 x(0, 7, 4, 125), x(0, 12, 5, 4093), 1574 x(0, 18, 5, 262141), x(0, 23, 5, 8388605), 1575 x(0, 5, 5, 29), x(0, 8, 4, 253), 1576 x(0, 14, 5, 16381), x(0, 20, 5, 1048573), 1577 x(0, 2, 5, 1), x(16, 7, 4, 125), 1578 x(0, 11, 5, 2045), x(0, 17, 5, 131069), 1579 x(0, 22, 5, 4194301), x(0, 4, 5, 13), 1580 x(16, 8, 4, 253), x(0, 13, 5, 8189), 1581 x(0, 19, 5, 524285), x(0, 1, 5, 1), 1582 x(16, 6, 4, 61), x(0, 10, 5, 1021), 1583 x(0, 16, 5, 65533), x(0, 28, 5, 268435453), 1584 x(0, 27, 5, 134217725), x(0, 26, 5, 67108861), 1585 x(0, 25, 5, 33554429), x(0, 24, 5, 16777213), 1586 } 1587 case tableMatchLengths: 1588 want = []decSymbol{ 1589 /* nextState, nbAddBits, nbBits, baseVal */ 1590 x(0, 0, 6, 3), x(0, 0, 4, 4), 1591 x(32, 0, 5, 5), x(0, 0, 5, 6), 1592 x(0, 0, 5, 8), x(0, 0, 5, 9), 1593 x(0, 0, 5, 11), x(0, 0, 6, 13), 1594 x(0, 0, 6, 16), x(0, 0, 6, 19), 1595 x(0, 0, 6, 22), x(0, 0, 6, 25), 1596 x(0, 0, 6, 28), x(0, 0, 6, 31), 1597 x(0, 0, 6, 34), x(0, 1, 6, 37), 1598 x(0, 1, 6, 41), x(0, 2, 6, 47), 1599 x(0, 3, 6, 59), x(0, 4, 6, 83), 1600 x(0, 7, 6, 131), x(0, 9, 6, 515), 1601 x(16, 0, 4, 4), x(0, 0, 4, 5), 1602 x(32, 0, 5, 6), x(0, 0, 5, 7), 1603 x(32, 0, 5, 9), x(0, 0, 5, 10), 1604 x(0, 0, 6, 12), x(0, 0, 6, 15), 1605 x(0, 0, 6, 18), x(0, 0, 6, 21), 1606 x(0, 0, 6, 24), x(0, 0, 6, 27), 1607 x(0, 0, 6, 30), x(0, 0, 6, 33), 1608 x(0, 1, 6, 35), x(0, 1, 6, 39), 1609 x(0, 2, 6, 43), x(0, 3, 6, 51), 1610 x(0, 4, 6, 67), x(0, 5, 6, 99), 1611 x(0, 8, 6, 259), x(32, 0, 4, 4), 1612 x(48, 0, 4, 4), x(16, 0, 4, 5), 1613 x(32, 0, 5, 7), x(32, 0, 5, 8), 1614 x(32, 0, 5, 10), x(32, 0, 5, 11), 1615 x(0, 0, 6, 14), x(0, 0, 6, 17), 1616 x(0, 0, 6, 20), x(0, 0, 6, 23), 1617 x(0, 0, 6, 26), x(0, 0, 6, 29), 1618 x(0, 0, 6, 32), x(0, 16, 6, 65539), 1619 x(0, 15, 6, 32771), x(0, 14, 6, 16387), 1620 x(0, 13, 6, 8195), x(0, 12, 6, 4099), 1621 x(0, 11, 6, 2051), x(0, 10, 6, 1027), 1622 } 1623 } 1624 pre := fsePredef[i] 1625 got := pre.dt[:1<<pre.actualTableLog] 1626 if !reflect.DeepEqual(got, want) { 1627 t.Logf("want: %v", want) 1628 t.Logf("got : %v", got) 1629 t.Errorf("Predefined table %d incorrect, len(got) = %d, len(want) = %d", i, len(got), len(want)) 1630 } 1631 } 1632} 1633 1634func TestResetNil(t *testing.T) { 1635 dec, err := NewReader(nil) 1636 if err != nil { 1637 t.Fatal(err) 1638 } 1639 defer dec.Close() 1640 1641 _, err = ioutil.ReadAll(dec) 1642 if err != ErrDecoderNilInput { 1643 t.Fatalf("Expected ErrDecoderNilInput when decoding from a nil reader, got %v", err) 1644 } 1645 1646 emptyZstdBlob := []byte{40, 181, 47, 253, 32, 0, 1, 0, 0} 1647 1648 dec.Reset(bytes.NewBuffer(emptyZstdBlob)) 1649 1650 result, err := ioutil.ReadAll(dec) 1651 if err != nil && err != io.EOF { 1652 t.Fatal(err) 1653 } 1654 if len(result) != 0 { 1655 t.Fatalf("Expected to read 0 bytes, actually read %d", len(result)) 1656 } 1657 1658 dec.Reset(nil) 1659 1660 _, err = ioutil.ReadAll(dec) 1661 if err != ErrDecoderNilInput { 1662 t.Fatalf("Expected ErrDecoderNilInput when decoding from a nil reader, got %v", err) 1663 } 1664 1665 dec.Reset(bytes.NewBuffer(emptyZstdBlob)) 1666 1667 result, err = ioutil.ReadAll(dec) 1668 if err != nil && err != io.EOF { 1669 t.Fatal(err) 1670 } 1671 if len(result) != 0 { 1672 t.Fatalf("Expected to read 0 bytes, actually read %d", len(result)) 1673 } 1674} 1675 1676func timeout(after time.Duration) (cancel func()) { 1677 c := time.After(after) 1678 cc := make(chan struct{}) 1679 go func() { 1680 select { 1681 case <-cc: 1682 return 1683 case <-c: 1684 buf := make([]byte, 1<<20) 1685 stacklen := runtime.Stack(buf, true) 1686 log.Printf("=== Timeout, assuming deadlock ===\n*** goroutine dump...\n%s\n*** end\n", string(buf[:stacklen])) 1687 os.Exit(2) 1688 } 1689 }() 1690 return func() { 1691 close(cc) 1692 } 1693} 1694