1// Copyright 2017 The Prometheus Authors
2// Licensed under the Apache License, Version 2.0 (the "License");
3// you may not use this file except in compliance with the License.
4// You may obtain a copy of the License at
5//
6// http://www.apache.org/licenses/LICENSE-2.0
7//
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13
14package textparse
15
16import (
17	"bytes"
18	"compress/gzip"
19	"io"
20	"io/ioutil"
21	"os"
22	"testing"
23
24	"github.com/prometheus/common/expfmt"
25	"github.com/prometheus/common/model"
26	"github.com/stretchr/testify/require"
27
28	"github.com/prometheus/prometheus/pkg/labels"
29)
30
31func TestPromParse(t *testing.T) {
32	input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
33# 	TYPE go_gc_duration_seconds summary
34go_gc_duration_seconds{quantile="0"} 4.9351e-05
35go_gc_duration_seconds{quantile="0.25",} 7.424100000000001e-05
36go_gc_duration_seconds{quantile="0.5",a="b"} 8.3835e-05
37go_gc_duration_seconds{quantile="0.8", a="b"} 8.3835e-05
38go_gc_duration_seconds{ quantile="0.9", a="b"} 8.3835e-05
39# Hrandom comment starting with prefix of HELP
40#
41wind_speed{A="2",c="3"} 12345
42# comment with escaped \n newline
43# comment with escaped \ escape character
44# HELP nohelp1
45# HELP nohelp2
46go_gc_duration_seconds{ quantile="1.0", a="b" } 8.3835e-05
47go_gc_duration_seconds { quantile="1.0", a="b" } 8.3835e-05
48go_gc_duration_seconds { quantile= "1.0", a= "b", } 8.3835e-05
49go_gc_duration_seconds { quantile = "1.0", a = "b" } 8.3835e-05
50go_gc_duration_seconds_count 99
51some:aggregate:rate5m{a_b="c"}	1
52# HELP go_goroutines Number of goroutines that currently exist.
53# TYPE go_goroutines gauge
54go_goroutines 33  	123123
55_metric_starting_with_underscore 1
56testmetric{_label_starting_with_underscore="foo"} 1
57testmetric{label="\"bar\""} 1`
58	input += "\n# HELP metric foo\x00bar"
59	input += "\nnull_byte_metric{a=\"abc\x00\"} 1"
60
61	int64p := func(x int64) *int64 { return &x }
62
63	exp := []struct {
64		lset    labels.Labels
65		m       string
66		t       *int64
67		v       float64
68		typ     MetricType
69		help    string
70		comment string
71	}{
72		{
73			m:    "go_gc_duration_seconds",
74			help: "A summary of the GC invocation durations.",
75		}, {
76			m:   "go_gc_duration_seconds",
77			typ: MetricTypeSummary,
78		}, {
79			m:    `go_gc_duration_seconds{quantile="0"}`,
80			v:    4.9351e-05,
81			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0"),
82		}, {
83			m:    `go_gc_duration_seconds{quantile="0.25",}`,
84			v:    7.424100000000001e-05,
85			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.25"),
86		}, {
87			m:    `go_gc_duration_seconds{quantile="0.5",a="b"}`,
88			v:    8.3835e-05,
89			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.5", "a", "b"),
90		}, {
91			m:    `go_gc_duration_seconds{quantile="0.8", a="b"}`,
92			v:    8.3835e-05,
93			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.8", "a", "b"),
94		}, {
95			m:    `go_gc_duration_seconds{ quantile="0.9", a="b"}`,
96			v:    8.3835e-05,
97			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.9", "a", "b"),
98		}, {
99			comment: "# Hrandom comment starting with prefix of HELP",
100		}, {
101			comment: "#",
102		}, {
103			m:    `wind_speed{A="2",c="3"}`,
104			v:    12345,
105			lset: labels.FromStrings("A", "2", "__name__", "wind_speed", "c", "3"),
106		}, {
107			comment: "# comment with escaped \\n newline",
108		}, {
109			comment: "# comment with escaped \\ escape character",
110		}, {
111			m:    "nohelp1",
112			help: "",
113		}, {
114			m:    "nohelp2",
115			help: "",
116		}, {
117			m:    `go_gc_duration_seconds{ quantile="1.0", a="b" }`,
118			v:    8.3835e-05,
119			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"),
120		}, {
121			m:    `go_gc_duration_seconds { quantile="1.0", a="b" }`,
122			v:    8.3835e-05,
123			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"),
124		}, {
125			m:    `go_gc_duration_seconds { quantile= "1.0", a= "b", }`,
126			v:    8.3835e-05,
127			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"),
128		}, {
129			m:    `go_gc_duration_seconds { quantile = "1.0", a = "b" }`,
130			v:    8.3835e-05,
131			lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"),
132		}, {
133			m:    `go_gc_duration_seconds_count`,
134			v:    99,
135			lset: labels.FromStrings("__name__", "go_gc_duration_seconds_count"),
136		}, {
137			m:    `some:aggregate:rate5m{a_b="c"}`,
138			v:    1,
139			lset: labels.FromStrings("__name__", "some:aggregate:rate5m", "a_b", "c"),
140		}, {
141			m:    "go_goroutines",
142			help: "Number of goroutines that currently exist.",
143		}, {
144			m:   "go_goroutines",
145			typ: MetricTypeGauge,
146		}, {
147			m:    `go_goroutines`,
148			v:    33,
149			t:    int64p(123123),
150			lset: labels.FromStrings("__name__", "go_goroutines"),
151		}, {
152			m:    "_metric_starting_with_underscore",
153			v:    1,
154			lset: labels.FromStrings("__name__", "_metric_starting_with_underscore"),
155		}, {
156			m:    "testmetric{_label_starting_with_underscore=\"foo\"}",
157			v:    1,
158			lset: labels.FromStrings("__name__", "testmetric", "_label_starting_with_underscore", "foo"),
159		}, {
160			m:    "testmetric{label=\"\\\"bar\\\"\"}",
161			v:    1,
162			lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`),
163		}, {
164			m:    "metric",
165			help: "foo\x00bar",
166		}, {
167			m:    "null_byte_metric{a=\"abc\x00\"}",
168			v:    1,
169			lset: labels.FromStrings("__name__", "null_byte_metric", "a", "abc\x00"),
170		},
171	}
172
173	p := NewPromParser([]byte(input))
174	i := 0
175
176	var res labels.Labels
177
178	for {
179		et, err := p.Next()
180		if err == io.EOF {
181			break
182		}
183		require.NoError(t, err)
184
185		switch et {
186		case EntrySeries:
187			m, ts, v := p.Series()
188
189			p.Metric(&res)
190
191			require.Equal(t, exp[i].m, string(m))
192			require.Equal(t, exp[i].t, ts)
193			require.Equal(t, exp[i].v, v)
194			require.Equal(t, exp[i].lset, res)
195			res = res[:0]
196
197		case EntryType:
198			m, typ := p.Type()
199			require.Equal(t, exp[i].m, string(m))
200			require.Equal(t, exp[i].typ, typ)
201
202		case EntryHelp:
203			m, h := p.Help()
204			require.Equal(t, exp[i].m, string(m))
205			require.Equal(t, exp[i].help, string(h))
206
207		case EntryComment:
208			require.Equal(t, exp[i].comment, string(p.Comment()))
209		}
210
211		i++
212	}
213	require.Equal(t, len(exp), i)
214}
215
216func TestPromParseErrors(t *testing.T) {
217	cases := []struct {
218		input string
219		err   string
220	}{
221		{
222			input: "a",
223			err:   "expected value after metric, got \"MNAME\"",
224		},
225		{
226			input: "a{b='c'} 1\n",
227			err:   "expected label value, got \"INVALID\"",
228		},
229		{
230			input: "a{b=\n",
231			err:   "expected label value, got \"INVALID\"",
232		},
233		{
234			input: "a{\xff=\"foo\"} 1\n",
235			err:   "expected label name, got \"INVALID\"",
236		},
237		{
238			input: "a{b=\"\xff\"} 1\n",
239			err:   "invalid UTF-8 label value",
240		},
241		{
242			input: "a true\n",
243			err:   "strconv.ParseFloat: parsing \"true\": invalid syntax",
244		},
245		{
246			input: "something_weird{problem=\"",
247			err:   "expected label value, got \"INVALID\"",
248		},
249		{
250			input: "empty_label_name{=\"\"} 0",
251			err:   "expected label name, got \"EQUAL\"",
252		},
253		{
254			input: "foo 1_2\n",
255			err:   "unsupported character in float",
256		},
257		{
258			input: "foo 0x1p-3\n",
259			err:   "unsupported character in float",
260		},
261		{
262			input: "foo 0x1P-3\n",
263			err:   "unsupported character in float",
264		},
265		{
266			input: "foo 0 1_2\n",
267			err:   "expected next entry after timestamp, got \"MNAME\"",
268		},
269		{
270			input: `{a="ok"} 1`,
271			err:   `"INVALID" is not a valid start token`,
272		},
273	}
274
275	for i, c := range cases {
276		p := NewPromParser([]byte(c.input))
277		var err error
278		for err == nil {
279			_, err = p.Next()
280		}
281		require.Error(t, err)
282		require.Equal(t, c.err, err.Error(), "test %d", i)
283	}
284}
285
286func TestPromNullByteHandling(t *testing.T) {
287	cases := []struct {
288		input string
289		err   string
290	}{
291		{
292			input: "null_byte_metric{a=\"abc\x00\"} 1",
293			err:   "",
294		},
295		{
296			input: "a{b=\"\x00ss\"} 1\n",
297			err:   "",
298		},
299		{
300			input: "a{b=\"\x00\"} 1\n",
301			err:   "",
302		},
303		{
304			input: "a{b=\"\x00\"} 1\n",
305			err:   "",
306		},
307		{
308			input: "a{b=\x00\"ssss\"} 1\n",
309			err:   "expected label value, got \"INVALID\"",
310		},
311		{
312			input: "a{b=\"\x00",
313			err:   "expected label value, got \"INVALID\"",
314		},
315		{
316			input: "a{b\x00=\"hiih\"}	1",
317			err: "expected equal, got \"INVALID\"",
318		},
319		{
320			input: "a\x00{b=\"ddd\"} 1",
321			err:   "expected value after metric, got \"MNAME\"",
322		},
323	}
324
325	for i, c := range cases {
326		p := NewPromParser([]byte(c.input))
327		var err error
328		for err == nil {
329			_, err = p.Next()
330		}
331
332		if c.err == "" {
333			require.Equal(t, io.EOF, err, "test %d", i)
334			continue
335		}
336
337		require.Error(t, err)
338		require.Equal(t, c.err, err.Error(), "test %d", i)
339	}
340}
341
342const (
343	promtestdataSampleCount = 410
344)
345
346func BenchmarkParse(b *testing.B) {
347	for parserName, parser := range map[string]func([]byte) Parser{
348		"prometheus":  NewPromParser,
349		"openmetrics": NewOpenMetricsParser,
350	} {
351		for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} {
352			f, err := os.Open(fn)
353			require.NoError(b, err)
354			defer f.Close()
355
356			buf, err := ioutil.ReadAll(f)
357			require.NoError(b, err)
358
359			b.Run(parserName+"/no-decode-metric/"+fn, func(b *testing.B) {
360				total := 0
361
362				b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount)))
363				b.ReportAllocs()
364				b.ResetTimer()
365
366				for i := 0; i < b.N; i += promtestdataSampleCount {
367					p := parser(buf)
368
369				Outer:
370					for i < b.N {
371						t, err := p.Next()
372						switch t {
373						case EntryInvalid:
374							if err == io.EOF {
375								break Outer
376							}
377							b.Fatal(err)
378						case EntrySeries:
379							m, _, _ := p.Series()
380							total += len(m)
381							i++
382						}
383					}
384				}
385				_ = total
386			})
387			b.Run(parserName+"/decode-metric/"+fn, func(b *testing.B) {
388				total := 0
389
390				b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount)))
391				b.ReportAllocs()
392				b.ResetTimer()
393
394				for i := 0; i < b.N; i += promtestdataSampleCount {
395					p := parser(buf)
396
397				Outer:
398					for i < b.N {
399						t, err := p.Next()
400						switch t {
401						case EntryInvalid:
402							if err == io.EOF {
403								break Outer
404							}
405							b.Fatal(err)
406						case EntrySeries:
407							m, _, _ := p.Series()
408
409							res := make(labels.Labels, 0, 5)
410							p.Metric(&res)
411
412							total += len(m)
413							i++
414						}
415					}
416				}
417				_ = total
418			})
419			b.Run(parserName+"/decode-metric-reuse/"+fn, func(b *testing.B) {
420				total := 0
421				res := make(labels.Labels, 0, 5)
422
423				b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount)))
424				b.ReportAllocs()
425				b.ResetTimer()
426
427				for i := 0; i < b.N; i += promtestdataSampleCount {
428					p := parser(buf)
429
430				Outer:
431					for i < b.N {
432						t, err := p.Next()
433						switch t {
434						case EntryInvalid:
435							if err == io.EOF {
436								break Outer
437							}
438							b.Fatal(err)
439						case EntrySeries:
440							m, _, _ := p.Series()
441
442							p.Metric(&res)
443
444							total += len(m)
445							i++
446							res = res[:0]
447						}
448					}
449				}
450				_ = total
451			})
452			b.Run("expfmt-text/"+fn, func(b *testing.B) {
453				b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount)))
454				b.ReportAllocs()
455				b.ResetTimer()
456
457				total := 0
458
459				for i := 0; i < b.N; i += promtestdataSampleCount {
460					var (
461						decSamples = make(model.Vector, 0, 50)
462					)
463					sdec := expfmt.SampleDecoder{
464						Dec: expfmt.NewDecoder(bytes.NewReader(buf), expfmt.FmtText),
465						Opts: &expfmt.DecodeOptions{
466							Timestamp: model.TimeFromUnixNano(0),
467						},
468					}
469
470					for {
471						if err = sdec.Decode(&decSamples); err != nil {
472							break
473						}
474						total += len(decSamples)
475						decSamples = decSamples[:0]
476					}
477				}
478				_ = total
479			})
480		}
481	}
482}
483func BenchmarkGzip(b *testing.B) {
484	for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} {
485		b.Run(fn, func(b *testing.B) {
486			f, err := os.Open(fn)
487			require.NoError(b, err)
488			defer f.Close()
489
490			var buf bytes.Buffer
491			gw := gzip.NewWriter(&buf)
492
493			n, err := io.Copy(gw, f)
494			require.NoError(b, err)
495			require.NoError(b, gw.Close())
496
497			gbuf, err := ioutil.ReadAll(&buf)
498			require.NoError(b, err)
499
500			k := b.N / promtestdataSampleCount
501
502			b.ReportAllocs()
503			b.SetBytes(int64(k) * int64(n))
504			b.ResetTimer()
505
506			total := 0
507
508			for i := 0; i < k; i++ {
509				gr, err := gzip.NewReader(bytes.NewReader(gbuf))
510				require.NoError(b, err)
511
512				d, err := ioutil.ReadAll(gr)
513				require.NoError(b, err)
514				require.NoError(b, gr.Close())
515
516				total += len(d)
517			}
518			_ = total
519		})
520	}
521}
522