1package zstd
2
3import (
4	"bytes"
5	"io"
6	"io/ioutil"
7	"os"
8	"testing"
9
10	"github.com/klauspost/compress/snappy"
11)
12
13func TestSnappy_ConvertSimple(t *testing.T) {
14	in, err := ioutil.ReadFile("testdata/z000028")
15	if err != nil {
16		t.Fatal(err)
17	}
18
19	var comp bytes.Buffer
20	w := snappy.NewBufferedWriter(&comp)
21	_, err = io.Copy(w, bytes.NewBuffer(in))
22	if err != nil {
23		t.Fatal(err)
24	}
25	err = w.Close()
26	if err != nil {
27		t.Fatal(err)
28	}
29	snapLen := comp.Len()
30	s := SnappyConverter{}
31	var dst bytes.Buffer
32	n, err := s.Convert(&comp, &dst)
33	if err != io.EOF {
34		t.Fatal(err)
35	}
36	if n != int64(dst.Len()) {
37		t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
38	}
39	t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len())
40
41	dec, err := NewReader(nil)
42	if err != nil {
43		t.Fatal(err)
44	}
45	defer dec.Close()
46	decoded, err := dec.DecodeAll(dst.Bytes(), nil)
47	if err != nil {
48		t.Error(err, len(decoded))
49	}
50	if !bytes.Equal(decoded, in) {
51		ioutil.WriteFile("testdata/z000028.got", decoded, os.ModePerm)
52		t.Fatal("Decoded does not match")
53	}
54	t.Log("Encoded content matched")
55}
56
57func TestSnappy_ConvertXML(t *testing.T) {
58	f, err := os.Open("testdata/xml.zst")
59	if err != nil {
60		t.Fatal(err)
61	}
62	dec, err := NewReader(f)
63	if err != nil {
64		t.Fatal(err)
65	}
66	defer dec.Close()
67	in, err := ioutil.ReadAll(dec)
68	if err != nil {
69		t.Fatal(err)
70	}
71	var comp bytes.Buffer
72	w := snappy.NewBufferedWriter(&comp)
73	_, err = io.Copy(w, bytes.NewBuffer(in))
74	if err != nil {
75		t.Fatal(err)
76	}
77	err = w.Close()
78	if err != nil {
79		t.Fatal(err)
80	}
81	snapLen := comp.Len()
82	s := SnappyConverter{}
83	var dst bytes.Buffer
84	n, err := s.Convert(&comp, &dst)
85	if err != io.EOF {
86		t.Fatal(err)
87	}
88	if n != int64(dst.Len()) {
89		t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
90	}
91	t.Log("Snappy len", snapLen, "-> zstd len", dst.Len())
92
93	decoded, err := dec.DecodeAll(dst.Bytes(), nil)
94	if err != nil {
95		t.Error(err, len(decoded))
96	}
97	if !bytes.Equal(decoded, in) {
98		ioutil.WriteFile("testdata/xml.got", decoded, os.ModePerm)
99		t.Fatal("Decoded does not match")
100	}
101	t.Log("Encoded content matched")
102}
103
104func TestSnappy_ConvertSilesia(t *testing.T) {
105	if testing.Short() {
106		t.SkipNow()
107	}
108	in, err := ioutil.ReadFile("testdata/silesia.tar")
109	if err != nil {
110		if os.IsNotExist(err) {
111			t.Skip("Missing testdata/silesia.tar")
112			return
113		}
114		t.Fatal(err)
115	}
116
117	var comp bytes.Buffer
118	w := snappy.NewBufferedWriter(&comp)
119	_, err = io.Copy(w, bytes.NewBuffer(in))
120	if err != nil {
121		t.Fatal(err)
122	}
123	err = w.Close()
124	if err != nil {
125		t.Fatal(err)
126	}
127	snapLen := comp.Len()
128	s := SnappyConverter{}
129	var dst bytes.Buffer
130	n, err := s.Convert(&comp, &dst)
131	if err != io.EOF {
132		t.Fatal(err)
133	}
134	if n != int64(dst.Len()) {
135		t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
136	}
137	t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len())
138
139	dec, err := NewReader(nil)
140	if err != nil {
141		t.Fatal(err)
142	}
143	defer dec.Close()
144	decoded, err := dec.DecodeAll(dst.Bytes(), nil)
145	if err != nil {
146		t.Error(err, len(decoded))
147	}
148	if !bytes.Equal(decoded, in) {
149		ioutil.WriteFile("testdata/silesia.tar.got", decoded, os.ModePerm)
150		t.Fatal("Decoded does not match")
151	}
152	t.Log("Encoded content matched")
153}
154
155func TestSnappy_ConvertEnwik9(t *testing.T) {
156	if testing.Short() {
157		t.SkipNow()
158	}
159	file := "testdata/enwik9.zst"
160	f, err := os.Open(file)
161	if err != nil {
162		if os.IsNotExist(err) {
163			t.Skip("To run extended tests, download http://mattmahoney.net/dc/enwik9.zip unzip it \n" +
164				"compress it with 'zstd -15 -T0 enwik9' and place it in " + file)
165		}
166	}
167	dec, err := NewReader(f)
168	if err != nil {
169		t.Fatal(err)
170	}
171	defer dec.Close()
172	in, err := ioutil.ReadAll(dec)
173	if err != nil {
174		t.Fatal(err)
175	}
176
177	var comp bytes.Buffer
178	w := snappy.NewBufferedWriter(&comp)
179	_, err = io.Copy(w, bytes.NewBuffer(in))
180	if err != nil {
181		t.Fatal(err)
182	}
183	err = w.Close()
184	if err != nil {
185		t.Fatal(err)
186	}
187	snapLen := comp.Len()
188	s := SnappyConverter{}
189	var dst bytes.Buffer
190	n, err := s.Convert(&comp, &dst)
191	if err != io.EOF {
192		t.Fatal(err)
193	}
194	if n != int64(dst.Len()) {
195		t.Errorf("Dest was %d bytes, but said to have written %d bytes", dst.Len(), n)
196	}
197	t.Log("SnappyConverter len", snapLen, "-> zstd len", dst.Len())
198
199	decoded, err := dec.DecodeAll(dst.Bytes(), nil)
200	if err != nil {
201		t.Error(err, len(decoded))
202	}
203	if !bytes.Equal(decoded, in) {
204		ioutil.WriteFile("testdata/enwik9.got", decoded, os.ModePerm)
205		t.Fatal("Decoded does not match")
206	}
207	t.Log("Encoded content matched")
208}
209
210func BenchmarkSnappy_ConvertXML(b *testing.B) {
211	f, err := os.Open("testdata/xml.zst")
212	if err != nil {
213		b.Fatal(err)
214	}
215	dec, err := NewReader(f)
216	if err != nil {
217		b.Fatal(err)
218	}
219	defer dec.Close()
220	in, err := ioutil.ReadAll(dec)
221	if err != nil {
222		b.Fatal(err)
223	}
224
225	var comp bytes.Buffer
226	w := snappy.NewBufferedWriter(&comp)
227	_, err = io.Copy(w, bytes.NewBuffer(in))
228	if err != nil {
229		b.Fatal(err)
230	}
231	err = w.Close()
232	if err != nil {
233		b.Fatal(err)
234	}
235	s := SnappyConverter{}
236	compBytes := comp.Bytes()
237	_, err = s.Convert(&comp, ioutil.Discard)
238	if err != io.EOF {
239		b.Fatal(err)
240	}
241	b.ResetTimer()
242	b.ReportAllocs()
243	b.SetBytes(int64(len(in)))
244	for i := 0; i < b.N; i++ {
245		_, err := s.Convert(bytes.NewBuffer(compBytes), ioutil.Discard)
246		if err != io.EOF {
247			b.Fatal(err)
248		}
249	}
250}
251
252func BenchmarkSnappy_Enwik9(b *testing.B) {
253	f, err := os.Open("testdata/enwik9.zst")
254	if err != nil {
255		b.Skip(err)
256		return
257	}
258	dec, err := NewReader(f)
259	if err != nil {
260		b.Fatal(err)
261	}
262	defer dec.Close()
263	in, err := ioutil.ReadAll(dec)
264	if err != nil {
265		b.Fatal(err)
266	}
267	defer dec.Close()
268
269	var comp bytes.Buffer
270	w := snappy.NewBufferedWriter(&comp)
271	_, err = io.Copy(w, bytes.NewBuffer(in))
272	if err != nil {
273		b.Fatal(err)
274	}
275	err = w.Close()
276	if err != nil {
277		b.Fatal(err)
278	}
279	s := SnappyConverter{}
280	compBytes := comp.Bytes()
281	_, err = s.Convert(&comp, ioutil.Discard)
282	if err != io.EOF {
283		b.Fatal(err)
284	}
285	b.ResetTimer()
286	b.ReportAllocs()
287	b.SetBytes(int64(len(in)))
288	for i := 0; i < b.N; i++ {
289		_, err := s.Convert(bytes.NewBuffer(compBytes), ioutil.Discard)
290		if err != io.EOF {
291			b.Fatal(err)
292		}
293	}
294}
295
296func BenchmarkSnappy_ConvertSilesia(b *testing.B) {
297	in, err := ioutil.ReadFile("testdata/silesia.tar")
298	if err != nil {
299		if os.IsNotExist(err) {
300			b.Skip("Missing testdata/silesia.tar")
301			return
302		}
303		b.Fatal(err)
304	}
305
306	var comp bytes.Buffer
307	w := snappy.NewBufferedWriter(&comp)
308	_, err = io.Copy(w, bytes.NewBuffer(in))
309	if err != nil {
310		b.Fatal(err)
311	}
312	err = w.Close()
313	if err != nil {
314		b.Fatal(err)
315	}
316	s := SnappyConverter{}
317	compBytes := comp.Bytes()
318	_, err = s.Convert(&comp, ioutil.Discard)
319	if err != io.EOF {
320		b.Fatal(err)
321	}
322	b.ResetTimer()
323	b.ReportAllocs()
324	b.SetBytes(int64(len(in)))
325	for i := 0; i < b.N; i++ {
326		_, err := s.Convert(bytes.NewBuffer(compBytes), ioutil.Discard)
327		if err != io.EOF {
328			b.Fatal(err)
329		}
330	}
331}
332