1package zstd 2 3import ( 4 "bytes" 5 "io" 6 "io/ioutil" 7 "os" 8 "testing" 9 10 "github.com/klauspost/compress/snappy" 11) 12 13func TestSnappy_ConvertSimple(t *testing.T) { 14 in, err := ioutil.ReadFile("testdata/z000028") 15 if err != nil { 16 t.Fatal(err) 17 } 18 19 var comp bytes.Buffer 20 w := snappy.NewBufferedWriter(&comp) 21 _, err = io.Copy(w, bytes.NewBuffer(in)) 22 if err != nil { 23 t.Fatal(err) 24 } 25 err = w.Close() 26 if err != nil { 27 t.Fatal(err) 28 } 29 snapLen := comp.Len() 30 s := SnappyConverter{} 31 var dst bytes.Buffer 32 n, err := s.Convert(&comp, &dst) 33 if err != io.EOF { 34 t.Fatal(err) 35 } 36 if n != int64(dst.Len()) { 37 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n) 38 } 39 t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len()) 40 41 dec, err := NewReader(nil) 42 if err != nil { 43 t.Fatal(err) 44 } 45 defer dec.Close() 46 decoded, err := dec.DecodeAll(dst.Bytes(), nil) 47 if err != nil { 48 t.Error(err, len(decoded)) 49 } 50 if !bytes.Equal(decoded, in) { 51 ioutil.WriteFile("testdata/z000028.got", decoded, os.ModePerm) 52 t.Fatal("Decoded does not match") 53 } 54 t.Log("Encoded content matched") 55} 56 57func TestSnappy_ConvertXML(t *testing.T) { 58 f, err := os.Open("testdata/xml.zst") 59 if err != nil { 60 t.Fatal(err) 61 } 62 dec, err := NewReader(f) 63 if err != nil { 64 t.Fatal(err) 65 } 66 defer dec.Close() 67 in, err := ioutil.ReadAll(dec) 68 if err != nil { 69 t.Fatal(err) 70 } 71 var comp bytes.Buffer 72 w := snappy.NewBufferedWriter(&comp) 73 _, err = io.Copy(w, bytes.NewBuffer(in)) 74 if err != nil { 75 t.Fatal(err) 76 } 77 err = w.Close() 78 if err != nil { 79 t.Fatal(err) 80 } 81 snapLen := comp.Len() 82 s := SnappyConverter{} 83 var dst bytes.Buffer 84 n, err := s.Convert(&comp, &dst) 85 if err != io.EOF { 86 t.Fatal(err) 87 } 88 if n != int64(dst.Len()) { 89 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n) 90 } 91 t.Log("Snappy len", snapLen, "-> zstd len", dst.Len()) 92 93 decoded, err := dec.DecodeAll(dst.Bytes(), nil) 94 if err != nil { 95 t.Error(err, len(decoded)) 96 } 97 if !bytes.Equal(decoded, in) { 98 ioutil.WriteFile("testdata/xml.got", decoded, os.ModePerm) 99 t.Fatal("Decoded does not match") 100 } 101 t.Log("Encoded content matched") 102} 103 104func TestSnappy_ConvertSilesia(t *testing.T) { 105 if testing.Short() { 106 t.SkipNow() 107 } 108 in, err := ioutil.ReadFile("testdata/silesia.tar") 109 if err != nil { 110 if os.IsNotExist(err) { 111 t.Skip("Missing testdata/silesia.tar") 112 return 113 } 114 t.Fatal(err) 115 } 116 117 var comp bytes.Buffer 118 w := snappy.NewBufferedWriter(&comp) 119 _, err = io.Copy(w, bytes.NewBuffer(in)) 120 if err != nil { 121 t.Fatal(err) 122 } 123 err = w.Close() 124 if err != nil { 125 t.Fatal(err) 126 } 127 snapLen := comp.Len() 128 s := SnappyConverter{} 129 var dst bytes.Buffer 130 n, err := s.Convert(&comp, &dst) 131 if err != io.EOF { 132 t.Fatal(err) 133 } 134 if n != int64(dst.Len()) { 135 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n) 136 } 137 t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len()) 138 139 dec, err := NewReader(nil) 140 if err != nil { 141 t.Fatal(err) 142 } 143 defer dec.Close() 144 decoded, err := dec.DecodeAll(dst.Bytes(), nil) 145 if err != nil { 146 t.Error(err, len(decoded)) 147 } 148 if !bytes.Equal(decoded, in) { 149 ioutil.WriteFile("testdata/silesia.tar.got", decoded, os.ModePerm) 150 t.Fatal("Decoded does not match") 151 } 152 t.Log("Encoded content matched") 153} 154 155func TestSnappy_ConvertEnwik9(t *testing.T) { 156 if testing.Short() { 157 t.SkipNow() 158 } 159 file := "testdata/enwik9.zst" 160 f, err := os.Open(file) 161 if err != nil { 162 if os.IsNotExist(err) { 163 t.Skip("To run extended tests, download http://mattmahoney.net/dc/enwik9.zip unzip it \n" + 164 "compress it with 'zstd -15 -T0 enwik9' and place it in " + file) 165 } 166 } 167 dec, err := NewReader(f) 168 if err != nil { 169 t.Fatal(err) 170 } 171 defer dec.Close() 172 in, err := ioutil.ReadAll(dec) 173 if err != nil { 174 t.Fatal(err) 175 } 176 177 var comp bytes.Buffer 178 w := snappy.NewBufferedWriter(&comp) 179 _, err = io.Copy(w, bytes.NewBuffer(in)) 180 if err != nil { 181 t.Fatal(err) 182 } 183 err = w.Close() 184 if err != nil { 185 t.Fatal(err) 186 } 187 snapLen := comp.Len() 188 s := SnappyConverter{} 189 var dst bytes.Buffer 190 n, err := s.Convert(&comp, &dst) 191 if err != io.EOF { 192 t.Fatal(err) 193 } 194 if n != int64(dst.Len()) { 195 t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n) 196 } 197 t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len()) 198 199 decoded, err := dec.DecodeAll(dst.Bytes(), nil) 200 if err != nil { 201 t.Error(err, len(decoded)) 202 } 203 if !bytes.Equal(decoded, in) { 204 ioutil.WriteFile("testdata/enwik9.got", decoded, os.ModePerm) 205 t.Fatal("Decoded does not match") 206 } 207 t.Log("Encoded content matched") 208} 209 210func BenchmarkSnappy_ConvertXML(b *testing.B) { 211 f, err := os.Open("testdata/xml.zst") 212 if err != nil { 213 b.Fatal(err) 214 } 215 dec, err := NewReader(f) 216 if err != nil { 217 b.Fatal(err) 218 } 219 defer dec.Close() 220 in, err := ioutil.ReadAll(dec) 221 if err != nil { 222 b.Fatal(err) 223 } 224 225 var comp bytes.Buffer 226 w := snappy.NewBufferedWriter(&comp) 227 _, err = io.Copy(w, bytes.NewBuffer(in)) 228 if err != nil { 229 b.Fatal(err) 230 } 231 err = w.Close() 232 if err != nil { 233 b.Fatal(err) 234 } 235 s := SnappyConverter{} 236 compBytes := comp.Bytes() 237 _, err = s.Convert(&comp, ioutil.Discard) 238 if err != io.EOF { 239 b.Fatal(err) 240 } 241 b.ResetTimer() 242 b.ReportAllocs() 243 b.SetBytes(int64(len(in))) 244 for i := 0; i < b.N; i++ { 245 _, err := s.Convert(bytes.NewBuffer(compBytes), ioutil.Discard) 246 if err != io.EOF { 247 b.Fatal(err) 248 } 249 } 250} 251 252func BenchmarkSnappy_Enwik9(b *testing.B) { 253 f, err := os.Open("testdata/enwik9.zst") 254 if err != nil { 255 b.Skip(err) 256 return 257 } 258 dec, err := NewReader(f) 259 if err != nil { 260 b.Fatal(err) 261 } 262 defer dec.Close() 263 in, err := ioutil.ReadAll(dec) 264 if err != nil { 265 b.Fatal(err) 266 } 267 defer dec.Close() 268 269 var comp bytes.Buffer 270 w := snappy.NewBufferedWriter(&comp) 271 _, err = io.Copy(w, bytes.NewBuffer(in)) 272 if err != nil { 273 b.Fatal(err) 274 } 275 err = w.Close() 276 if err != nil { 277 b.Fatal(err) 278 } 279 s := SnappyConverter{} 280 compBytes := comp.Bytes() 281 _, err = s.Convert(&comp, ioutil.Discard) 282 if err != io.EOF { 283 b.Fatal(err) 284 } 285 b.ResetTimer() 286 b.ReportAllocs() 287 b.SetBytes(int64(len(in))) 288 for i := 0; i < b.N; i++ { 289 _, err := s.Convert(bytes.NewBuffer(compBytes), ioutil.Discard) 290 if err != io.EOF { 291 b.Fatal(err) 292 } 293 } 294} 295 296func BenchmarkSnappy_ConvertSilesia(b *testing.B) { 297 in, err := ioutil.ReadFile("testdata/silesia.tar") 298 if err != nil { 299 if os.IsNotExist(err) { 300 b.Skip("Missing testdata/silesia.tar") 301 return 302 } 303 b.Fatal(err) 304 } 305 306 var comp bytes.Buffer 307 w := snappy.NewBufferedWriter(&comp) 308 _, err = io.Copy(w, bytes.NewBuffer(in)) 309 if err != nil { 310 b.Fatal(err) 311 } 312 err = w.Close() 313 if err != nil { 314 b.Fatal(err) 315 } 316 s := SnappyConverter{} 317 compBytes := comp.Bytes() 318 _, err = s.Convert(&comp, ioutil.Discard) 319 if err != io.EOF { 320 b.Fatal(err) 321 } 322 b.ResetTimer() 323 b.ReportAllocs() 324 b.SetBytes(int64(len(in))) 325 for i := 0; i < b.N; i++ { 326 _, err := s.Convert(bytes.NewBuffer(compBytes), ioutil.Discard) 327 if err != io.EOF { 328 b.Fatal(err) 329 } 330 } 331} 332