1// Copyright 2017 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package textparse 15 16import ( 17 "bytes" 18 "compress/gzip" 19 "io" 20 "io/ioutil" 21 "os" 22 "testing" 23 24 "github.com/prometheus/common/expfmt" 25 "github.com/prometheus/common/model" 26 "github.com/stretchr/testify/require" 27 28 "github.com/prometheus/prometheus/pkg/labels" 29) 30 31func TestPromParse(t *testing.T) { 32 input := `# HELP go_gc_duration_seconds A summary of the GC invocation durations. 33# TYPE go_gc_duration_seconds summary 34go_gc_duration_seconds{quantile="0"} 4.9351e-05 35go_gc_duration_seconds{quantile="0.25",} 7.424100000000001e-05 36go_gc_duration_seconds{quantile="0.5",a="b"} 8.3835e-05 37go_gc_duration_seconds{quantile="0.8", a="b"} 8.3835e-05 38go_gc_duration_seconds{ quantile="0.9", a="b"} 8.3835e-05 39# Hrandom comment starting with prefix of HELP 40# 41wind_speed{A="2",c="3"} 12345 42# comment with escaped \n newline 43# comment with escaped \ escape character 44# HELP nohelp1 45# HELP nohelp2 46go_gc_duration_seconds{ quantile="1.0", a="b" } 8.3835e-05 47go_gc_duration_seconds { quantile="1.0", a="b" } 8.3835e-05 48go_gc_duration_seconds { quantile= "1.0", a= "b", } 8.3835e-05 49go_gc_duration_seconds { quantile = "1.0", a = "b" } 8.3835e-05 50go_gc_duration_seconds_count 99 51some:aggregate:rate5m{a_b="c"} 1 52# HELP go_goroutines Number of goroutines that currently exist. 53# TYPE go_goroutines gauge 54go_goroutines 33 123123 55_metric_starting_with_underscore 1 56testmetric{_label_starting_with_underscore="foo"} 1 57testmetric{label="\"bar\""} 1` 58 input += "\n# HELP metric foo\x00bar" 59 input += "\nnull_byte_metric{a=\"abc\x00\"} 1" 60 61 int64p := func(x int64) *int64 { return &x } 62 63 exp := []struct { 64 lset labels.Labels 65 m string 66 t *int64 67 v float64 68 typ MetricType 69 help string 70 comment string 71 }{ 72 { 73 m: "go_gc_duration_seconds", 74 help: "A summary of the GC invocation durations.", 75 }, { 76 m: "go_gc_duration_seconds", 77 typ: MetricTypeSummary, 78 }, { 79 m: `go_gc_duration_seconds{quantile="0"}`, 80 v: 4.9351e-05, 81 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0"), 82 }, { 83 m: `go_gc_duration_seconds{quantile="0.25",}`, 84 v: 7.424100000000001e-05, 85 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.25"), 86 }, { 87 m: `go_gc_duration_seconds{quantile="0.5",a="b"}`, 88 v: 8.3835e-05, 89 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.5", "a", "b"), 90 }, { 91 m: `go_gc_duration_seconds{quantile="0.8", a="b"}`, 92 v: 8.3835e-05, 93 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.8", "a", "b"), 94 }, { 95 m: `go_gc_duration_seconds{ quantile="0.9", a="b"}`, 96 v: 8.3835e-05, 97 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "0.9", "a", "b"), 98 }, { 99 comment: "# Hrandom comment starting with prefix of HELP", 100 }, { 101 comment: "#", 102 }, { 103 m: `wind_speed{A="2",c="3"}`, 104 v: 12345, 105 lset: labels.FromStrings("A", "2", "__name__", "wind_speed", "c", "3"), 106 }, { 107 comment: "# comment with escaped \\n newline", 108 }, { 109 comment: "# comment with escaped \\ escape character", 110 }, { 111 m: "nohelp1", 112 help: "", 113 }, { 114 m: "nohelp2", 115 help: "", 116 }, { 117 m: `go_gc_duration_seconds{ quantile="1.0", a="b" }`, 118 v: 8.3835e-05, 119 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"), 120 }, { 121 m: `go_gc_duration_seconds { quantile="1.0", a="b" }`, 122 v: 8.3835e-05, 123 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"), 124 }, { 125 m: `go_gc_duration_seconds { quantile= "1.0", a= "b", }`, 126 v: 8.3835e-05, 127 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"), 128 }, { 129 m: `go_gc_duration_seconds { quantile = "1.0", a = "b" }`, 130 v: 8.3835e-05, 131 lset: labels.FromStrings("__name__", "go_gc_duration_seconds", "quantile", "1.0", "a", "b"), 132 }, { 133 m: `go_gc_duration_seconds_count`, 134 v: 99, 135 lset: labels.FromStrings("__name__", "go_gc_duration_seconds_count"), 136 }, { 137 m: `some:aggregate:rate5m{a_b="c"}`, 138 v: 1, 139 lset: labels.FromStrings("__name__", "some:aggregate:rate5m", "a_b", "c"), 140 }, { 141 m: "go_goroutines", 142 help: "Number of goroutines that currently exist.", 143 }, { 144 m: "go_goroutines", 145 typ: MetricTypeGauge, 146 }, { 147 m: `go_goroutines`, 148 v: 33, 149 t: int64p(123123), 150 lset: labels.FromStrings("__name__", "go_goroutines"), 151 }, { 152 m: "_metric_starting_with_underscore", 153 v: 1, 154 lset: labels.FromStrings("__name__", "_metric_starting_with_underscore"), 155 }, { 156 m: "testmetric{_label_starting_with_underscore=\"foo\"}", 157 v: 1, 158 lset: labels.FromStrings("__name__", "testmetric", "_label_starting_with_underscore", "foo"), 159 }, { 160 m: "testmetric{label=\"\\\"bar\\\"\"}", 161 v: 1, 162 lset: labels.FromStrings("__name__", "testmetric", "label", `"bar"`), 163 }, { 164 m: "metric", 165 help: "foo\x00bar", 166 }, { 167 m: "null_byte_metric{a=\"abc\x00\"}", 168 v: 1, 169 lset: labels.FromStrings("__name__", "null_byte_metric", "a", "abc\x00"), 170 }, 171 } 172 173 p := NewPromParser([]byte(input)) 174 i := 0 175 176 var res labels.Labels 177 178 for { 179 et, err := p.Next() 180 if err == io.EOF { 181 break 182 } 183 require.NoError(t, err) 184 185 switch et { 186 case EntrySeries: 187 m, ts, v := p.Series() 188 189 p.Metric(&res) 190 191 require.Equal(t, exp[i].m, string(m)) 192 require.Equal(t, exp[i].t, ts) 193 require.Equal(t, exp[i].v, v) 194 require.Equal(t, exp[i].lset, res) 195 res = res[:0] 196 197 case EntryType: 198 m, typ := p.Type() 199 require.Equal(t, exp[i].m, string(m)) 200 require.Equal(t, exp[i].typ, typ) 201 202 case EntryHelp: 203 m, h := p.Help() 204 require.Equal(t, exp[i].m, string(m)) 205 require.Equal(t, exp[i].help, string(h)) 206 207 case EntryComment: 208 require.Equal(t, exp[i].comment, string(p.Comment())) 209 } 210 211 i++ 212 } 213 require.Equal(t, len(exp), i) 214} 215 216func TestPromParseErrors(t *testing.T) { 217 cases := []struct { 218 input string 219 err string 220 }{ 221 { 222 input: "a", 223 err: "expected value after metric, got \"MNAME\"", 224 }, 225 { 226 input: "a{b='c'} 1\n", 227 err: "expected label value, got \"INVALID\"", 228 }, 229 { 230 input: "a{b=\n", 231 err: "expected label value, got \"INVALID\"", 232 }, 233 { 234 input: "a{\xff=\"foo\"} 1\n", 235 err: "expected label name, got \"INVALID\"", 236 }, 237 { 238 input: "a{b=\"\xff\"} 1\n", 239 err: "invalid UTF-8 label value", 240 }, 241 { 242 input: "a true\n", 243 err: "strconv.ParseFloat: parsing \"true\": invalid syntax", 244 }, 245 { 246 input: "something_weird{problem=\"", 247 err: "expected label value, got \"INVALID\"", 248 }, 249 { 250 input: "empty_label_name{=\"\"} 0", 251 err: "expected label name, got \"EQUAL\"", 252 }, 253 { 254 input: "foo 1_2\n", 255 err: "unsupported character in float", 256 }, 257 { 258 input: "foo 0x1p-3\n", 259 err: "unsupported character in float", 260 }, 261 { 262 input: "foo 0x1P-3\n", 263 err: "unsupported character in float", 264 }, 265 { 266 input: "foo 0 1_2\n", 267 err: "expected next entry after timestamp, got \"MNAME\"", 268 }, 269 { 270 input: `{a="ok"} 1`, 271 err: `"INVALID" is not a valid start token`, 272 }, 273 } 274 275 for i, c := range cases { 276 p := NewPromParser([]byte(c.input)) 277 var err error 278 for err == nil { 279 _, err = p.Next() 280 } 281 require.Error(t, err) 282 require.Equal(t, c.err, err.Error(), "test %d", i) 283 } 284} 285 286func TestPromNullByteHandling(t *testing.T) { 287 cases := []struct { 288 input string 289 err string 290 }{ 291 { 292 input: "null_byte_metric{a=\"abc\x00\"} 1", 293 err: "", 294 }, 295 { 296 input: "a{b=\"\x00ss\"} 1\n", 297 err: "", 298 }, 299 { 300 input: "a{b=\"\x00\"} 1\n", 301 err: "", 302 }, 303 { 304 input: "a{b=\"\x00\"} 1\n", 305 err: "", 306 }, 307 { 308 input: "a{b=\x00\"ssss\"} 1\n", 309 err: "expected label value, got \"INVALID\"", 310 }, 311 { 312 input: "a{b=\"\x00", 313 err: "expected label value, got \"INVALID\"", 314 }, 315 { 316 input: "a{b\x00=\"hiih\"} 1", 317 err: "expected equal, got \"INVALID\"", 318 }, 319 { 320 input: "a\x00{b=\"ddd\"} 1", 321 err: "expected value after metric, got \"MNAME\"", 322 }, 323 } 324 325 for i, c := range cases { 326 p := NewPromParser([]byte(c.input)) 327 var err error 328 for err == nil { 329 _, err = p.Next() 330 } 331 332 if c.err == "" { 333 require.Equal(t, io.EOF, err, "test %d", i) 334 continue 335 } 336 337 require.Error(t, err) 338 require.Equal(t, c.err, err.Error(), "test %d", i) 339 } 340} 341 342const ( 343 promtestdataSampleCount = 410 344) 345 346func BenchmarkParse(b *testing.B) { 347 for parserName, parser := range map[string]func([]byte) Parser{ 348 "prometheus": NewPromParser, 349 "openmetrics": NewOpenMetricsParser, 350 } { 351 for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} { 352 f, err := os.Open(fn) 353 require.NoError(b, err) 354 defer f.Close() 355 356 buf, err := ioutil.ReadAll(f) 357 require.NoError(b, err) 358 359 b.Run(parserName+"/no-decode-metric/"+fn, func(b *testing.B) { 360 total := 0 361 362 b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount))) 363 b.ReportAllocs() 364 b.ResetTimer() 365 366 for i := 0; i < b.N; i += promtestdataSampleCount { 367 p := parser(buf) 368 369 Outer: 370 for i < b.N { 371 t, err := p.Next() 372 switch t { 373 case EntryInvalid: 374 if err == io.EOF { 375 break Outer 376 } 377 b.Fatal(err) 378 case EntrySeries: 379 m, _, _ := p.Series() 380 total += len(m) 381 i++ 382 } 383 } 384 } 385 _ = total 386 }) 387 b.Run(parserName+"/decode-metric/"+fn, func(b *testing.B) { 388 total := 0 389 390 b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount))) 391 b.ReportAllocs() 392 b.ResetTimer() 393 394 for i := 0; i < b.N; i += promtestdataSampleCount { 395 p := parser(buf) 396 397 Outer: 398 for i < b.N { 399 t, err := p.Next() 400 switch t { 401 case EntryInvalid: 402 if err == io.EOF { 403 break Outer 404 } 405 b.Fatal(err) 406 case EntrySeries: 407 m, _, _ := p.Series() 408 409 res := make(labels.Labels, 0, 5) 410 p.Metric(&res) 411 412 total += len(m) 413 i++ 414 } 415 } 416 } 417 _ = total 418 }) 419 b.Run(parserName+"/decode-metric-reuse/"+fn, func(b *testing.B) { 420 total := 0 421 res := make(labels.Labels, 0, 5) 422 423 b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount))) 424 b.ReportAllocs() 425 b.ResetTimer() 426 427 for i := 0; i < b.N; i += promtestdataSampleCount { 428 p := parser(buf) 429 430 Outer: 431 for i < b.N { 432 t, err := p.Next() 433 switch t { 434 case EntryInvalid: 435 if err == io.EOF { 436 break Outer 437 } 438 b.Fatal(err) 439 case EntrySeries: 440 m, _, _ := p.Series() 441 442 p.Metric(&res) 443 444 total += len(m) 445 i++ 446 res = res[:0] 447 } 448 } 449 } 450 _ = total 451 }) 452 b.Run("expfmt-text/"+fn, func(b *testing.B) { 453 b.SetBytes(int64(len(buf) * (b.N / promtestdataSampleCount))) 454 b.ReportAllocs() 455 b.ResetTimer() 456 457 total := 0 458 459 for i := 0; i < b.N; i += promtestdataSampleCount { 460 var ( 461 decSamples = make(model.Vector, 0, 50) 462 ) 463 sdec := expfmt.SampleDecoder{ 464 Dec: expfmt.NewDecoder(bytes.NewReader(buf), expfmt.FmtText), 465 Opts: &expfmt.DecodeOptions{ 466 Timestamp: model.TimeFromUnixNano(0), 467 }, 468 } 469 470 for { 471 if err = sdec.Decode(&decSamples); err != nil { 472 break 473 } 474 total += len(decSamples) 475 decSamples = decSamples[:0] 476 } 477 } 478 _ = total 479 }) 480 } 481 } 482} 483func BenchmarkGzip(b *testing.B) { 484 for _, fn := range []string{"promtestdata.txt", "promtestdata.nometa.txt"} { 485 b.Run(fn, func(b *testing.B) { 486 f, err := os.Open(fn) 487 require.NoError(b, err) 488 defer f.Close() 489 490 var buf bytes.Buffer 491 gw := gzip.NewWriter(&buf) 492 493 n, err := io.Copy(gw, f) 494 require.NoError(b, err) 495 require.NoError(b, gw.Close()) 496 497 gbuf, err := ioutil.ReadAll(&buf) 498 require.NoError(b, err) 499 500 k := b.N / promtestdataSampleCount 501 502 b.ReportAllocs() 503 b.SetBytes(int64(k) * int64(n)) 504 b.ResetTimer() 505 506 total := 0 507 508 for i := 0; i < k; i++ { 509 gr, err := gzip.NewReader(bytes.NewReader(gbuf)) 510 require.NoError(b, err) 511 512 d, err := ioutil.ReadAll(gr) 513 require.NoError(b, err) 514 require.NoError(b, gr.Close()) 515 516 total += len(d) 517 } 518 _ = total 519 }) 520 } 521} 522