1// Copyright 2015 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package promql 15 16import ( 17 "fmt" 18 "io/ioutil" 19 "math" 20 "regexp" 21 "strconv" 22 "strings" 23 "time" 24 25 "github.com/prometheus/common/model" 26 "golang.org/x/net/context" 27 28 "github.com/prometheus/prometheus/storage" 29 "github.com/prometheus/prometheus/storage/local" 30 "github.com/prometheus/prometheus/util/testutil" 31) 32 33var ( 34 minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64. 35 36 patSpace = regexp.MustCompile("[\t ]+") 37 patLoad = regexp.MustCompile(`^load\s+(.+?)$`) 38 patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|ordered))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`) 39) 40 41const ( 42 testStartTime = model.Time(0) 43 epsilon = 0.000001 // Relative error allowed for sample values. 44) 45 46// Test is a sequence of read and write commands that are run 47// against a test storage. 48type Test struct { 49 testutil.T 50 51 cmds []testCommand 52 53 storage local.Storage 54 closeStorage func() 55 queryEngine *Engine 56 context context.Context 57 cancelCtx context.CancelFunc 58} 59 60// NewTest returns an initialized empty Test. 61func NewTest(t testutil.T, input string) (*Test, error) { 62 test := &Test{ 63 T: t, 64 cmds: []testCommand{}, 65 } 66 err := test.parse(input) 67 test.clear() 68 69 return test, err 70} 71 72func newTestFromFile(t testutil.T, filename string) (*Test, error) { 73 content, err := ioutil.ReadFile(filename) 74 if err != nil { 75 return nil, err 76 } 77 return NewTest(t, string(content)) 78} 79 80// QueryEngine returns the test's query engine. 81func (t *Test) QueryEngine() *Engine { 82 return t.queryEngine 83} 84 85// Context returns the test's context. 86func (t *Test) Context() context.Context { 87 return t.context 88} 89 90// Storage returns the test's storage. 91func (t *Test) Storage() local.Storage { 92 return t.storage 93} 94 95func raise(line int, format string, v ...interface{}) error { 96 return &ParseErr{ 97 Line: line + 1, 98 Err: fmt.Errorf(format, v...), 99 } 100} 101 102func (t *Test) parseLoad(lines []string, i int) (int, *loadCmd, error) { 103 if !patLoad.MatchString(lines[i]) { 104 return i, nil, raise(i, "invalid load command. (load <step:duration>)") 105 } 106 parts := patLoad.FindStringSubmatch(lines[i]) 107 108 gap, err := model.ParseDuration(parts[1]) 109 if err != nil { 110 return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err) 111 } 112 cmd := newLoadCmd(time.Duration(gap)) 113 for i+1 < len(lines) { 114 i++ 115 defLine := lines[i] 116 if len(defLine) == 0 { 117 i-- 118 break 119 } 120 metric, vals, err := parseSeriesDesc(defLine) 121 if err != nil { 122 if perr, ok := err.(*ParseErr); ok { 123 perr.Line = i + 1 124 } 125 return i, nil, err 126 } 127 cmd.set(metric, vals...) 128 } 129 return i, cmd, nil 130} 131 132func (t *Test) parseEval(lines []string, i int) (int, *evalCmd, error) { 133 if !patEvalInstant.MatchString(lines[i]) { 134 return i, nil, raise(i, "invalid evaluation command. (eval[_fail|_ordered] instant [at <offset:duration>] <query>") 135 } 136 parts := patEvalInstant.FindStringSubmatch(lines[i]) 137 var ( 138 mod = parts[1] 139 at = parts[2] 140 qry = parts[3] 141 ) 142 expr, err := ParseExpr(qry) 143 if err != nil { 144 if perr, ok := err.(*ParseErr); ok { 145 perr.Line = i + 1 146 perr.Pos += strings.Index(lines[i], qry) 147 } 148 return i, nil, err 149 } 150 151 offset, err := model.ParseDuration(at) 152 if err != nil { 153 return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err) 154 } 155 ts := testStartTime.Add(time.Duration(offset)) 156 157 cmd := newEvalCmd(expr, ts, ts, 0) 158 switch mod { 159 case "ordered": 160 cmd.ordered = true 161 case "fail": 162 cmd.fail = true 163 } 164 165 for j := 1; i+1 < len(lines); j++ { 166 i++ 167 defLine := lines[i] 168 if len(defLine) == 0 { 169 i-- 170 break 171 } 172 if f, err := parseNumber(defLine); err == nil { 173 cmd.expect(0, nil, sequenceValue{value: model.SampleValue(f)}) 174 break 175 } 176 metric, vals, err := parseSeriesDesc(defLine) 177 if err != nil { 178 if perr, ok := err.(*ParseErr); ok { 179 perr.Line = i + 1 180 } 181 return i, nil, err 182 } 183 184 // Currently, we are not expecting any matrices. 185 if len(vals) > 1 { 186 return i, nil, raise(i, "expecting multiple values in instant evaluation not allowed") 187 } 188 cmd.expect(j, metric, vals...) 189 } 190 return i, cmd, nil 191} 192 193// parse the given command sequence and appends it to the test. 194func (t *Test) parse(input string) error { 195 // Trim lines and remove comments. 196 lines := strings.Split(input, "\n") 197 for i, l := range lines { 198 l = strings.TrimSpace(l) 199 if strings.HasPrefix(l, "#") { 200 l = "" 201 } 202 lines[i] = l 203 } 204 var err error 205 206 // Scan for steps line by line. 207 for i := 0; i < len(lines); i++ { 208 l := lines[i] 209 if len(l) == 0 { 210 continue 211 } 212 var cmd testCommand 213 214 switch c := strings.ToLower(patSpace.Split(l, 2)[0]); { 215 case c == "clear": 216 cmd = &clearCmd{} 217 case c == "load": 218 i, cmd, err = t.parseLoad(lines, i) 219 case strings.HasPrefix(c, "eval"): 220 i, cmd, err = t.parseEval(lines, i) 221 default: 222 return raise(i, "invalid command %q", l) 223 } 224 if err != nil { 225 return err 226 } 227 t.cmds = append(t.cmds, cmd) 228 } 229 return nil 230} 231 232// testCommand is an interface that ensures that only the package internal 233// types can be a valid command for a test. 234type testCommand interface { 235 testCmd() 236} 237 238func (*clearCmd) testCmd() {} 239func (*loadCmd) testCmd() {} 240func (*evalCmd) testCmd() {} 241 242// loadCmd is a command that loads sequences of sample values for specific 243// metrics into the storage. 244type loadCmd struct { 245 gap time.Duration 246 metrics map[model.Fingerprint]model.Metric 247 defs map[model.Fingerprint][]model.SamplePair 248} 249 250func newLoadCmd(gap time.Duration) *loadCmd { 251 return &loadCmd{ 252 gap: gap, 253 metrics: map[model.Fingerprint]model.Metric{}, 254 defs: map[model.Fingerprint][]model.SamplePair{}, 255 } 256} 257 258func (cmd loadCmd) String() string { 259 return "load" 260} 261 262// set a sequence of sample values for the given metric. 263func (cmd *loadCmd) set(m model.Metric, vals ...sequenceValue) { 264 fp := m.Fingerprint() 265 266 samples := make([]model.SamplePair, 0, len(vals)) 267 ts := testStartTime 268 for _, v := range vals { 269 if !v.omitted { 270 samples = append(samples, model.SamplePair{ 271 Timestamp: ts, 272 Value: v.value, 273 }) 274 } 275 ts = ts.Add(cmd.gap) 276 } 277 cmd.defs[fp] = samples 278 cmd.metrics[fp] = m 279} 280 281// append the defined time series to the storage. 282func (cmd *loadCmd) append(a storage.SampleAppender) { 283 for fp, samples := range cmd.defs { 284 met := cmd.metrics[fp] 285 for _, smpl := range samples { 286 s := &model.Sample{ 287 Metric: met, 288 Value: smpl.Value, 289 Timestamp: smpl.Timestamp, 290 } 291 a.Append(s) 292 } 293 } 294} 295 296// evalCmd is a command that evaluates an expression for the given time (range) 297// and expects a specific result. 298type evalCmd struct { 299 expr Expr 300 start, end model.Time 301 interval time.Duration 302 303 instant bool 304 fail, ordered bool 305 306 metrics map[model.Fingerprint]model.Metric 307 expected map[model.Fingerprint]entry 308} 309 310type entry struct { 311 pos int 312 vals []sequenceValue 313} 314 315func (e entry) String() string { 316 return fmt.Sprintf("%d: %s", e.pos, e.vals) 317} 318 319func newEvalCmd(expr Expr, start, end model.Time, interval time.Duration) *evalCmd { 320 return &evalCmd{ 321 expr: expr, 322 start: start, 323 end: end, 324 interval: interval, 325 instant: start == end && interval == 0, 326 327 metrics: map[model.Fingerprint]model.Metric{}, 328 expected: map[model.Fingerprint]entry{}, 329 } 330} 331 332func (ev *evalCmd) String() string { 333 return "eval" 334} 335 336// expect adds a new metric with a sequence of values to the set of expected 337// results for the query. 338func (ev *evalCmd) expect(pos int, m model.Metric, vals ...sequenceValue) { 339 if m == nil { 340 ev.expected[0] = entry{pos: pos, vals: vals} 341 return 342 } 343 fp := m.Fingerprint() 344 ev.metrics[fp] = m 345 ev.expected[fp] = entry{pos: pos, vals: vals} 346} 347 348// compareResult compares the result value with the defined expectation. 349func (ev *evalCmd) compareResult(result model.Value) error { 350 switch val := result.(type) { 351 case model.Matrix: 352 if ev.instant { 353 return fmt.Errorf("received range result on instant evaluation") 354 } 355 seen := map[model.Fingerprint]bool{} 356 for pos, v := range val { 357 fp := v.Metric.Fingerprint() 358 if _, ok := ev.metrics[fp]; !ok { 359 return fmt.Errorf("unexpected metric %s in result", v.Metric) 360 } 361 exp := ev.expected[fp] 362 if ev.ordered && exp.pos != pos+1 { 363 return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric, exp.vals, exp.pos, pos+1) 364 } 365 for i, expVal := range exp.vals { 366 if !almostEqual(float64(expVal.value), float64(v.Values[i].Value)) { 367 return fmt.Errorf("expected %v for %s but got %v", expVal, v.Metric, v.Values) 368 } 369 } 370 seen[fp] = true 371 } 372 for fp, expVals := range ev.expected { 373 if !seen[fp] { 374 return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals) 375 } 376 } 377 378 case model.Vector: 379 if !ev.instant { 380 return fmt.Errorf("received instant result on range evaluation") 381 } 382 seen := map[model.Fingerprint]bool{} 383 for pos, v := range val { 384 fp := v.Metric.Fingerprint() 385 if _, ok := ev.metrics[fp]; !ok { 386 return fmt.Errorf("unexpected metric %s in result", v.Metric) 387 } 388 exp := ev.expected[fp] 389 if ev.ordered && exp.pos != pos+1 { 390 return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric, exp.vals, exp.pos, pos+1) 391 } 392 if !almostEqual(float64(exp.vals[0].value), float64(v.Value)) { 393 return fmt.Errorf("expected %v for %s but got %v", exp.vals[0].value, v.Metric, v.Value) 394 } 395 396 seen[fp] = true 397 } 398 for fp, expVals := range ev.expected { 399 if !seen[fp] { 400 return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals) 401 } 402 } 403 404 case *model.Scalar: 405 if !almostEqual(float64(ev.expected[0].vals[0].value), float64(val.Value)) { 406 return fmt.Errorf("expected scalar %v but got %v", val.Value, ev.expected[0].vals[0].value) 407 } 408 409 default: 410 panic(fmt.Errorf("promql.Test.compareResult: unexpected result type %T", result)) 411 } 412 return nil 413} 414 415// clearCmd is a command that wipes the test's storage state. 416type clearCmd struct{} 417 418func (cmd clearCmd) String() string { 419 return "clear" 420} 421 422// Run executes the command sequence of the test. Until the maximum error number 423// is reached, evaluation errors do not terminate execution. 424func (t *Test) Run() error { 425 for _, cmd := range t.cmds { 426 err := t.exec(cmd) 427 // TODO(fabxc): aggregate command errors, yield diffs for result 428 // comparison errors. 429 if err != nil { 430 return err 431 } 432 } 433 return nil 434} 435 436// exec processes a single step of the test. 437func (t *Test) exec(tc testCommand) error { 438 switch cmd := tc.(type) { 439 case *clearCmd: 440 t.clear() 441 442 case *loadCmd: 443 cmd.append(t.storage) 444 t.storage.WaitForIndexing() 445 446 case *evalCmd: 447 q := t.queryEngine.newQuery(cmd.expr, cmd.start, cmd.end, cmd.interval) 448 res := q.Exec(t.context) 449 if res.Err != nil { 450 if cmd.fail { 451 return nil 452 } 453 return fmt.Errorf("error evaluating query: %s", res.Err) 454 } 455 if res.Err == nil && cmd.fail { 456 return fmt.Errorf("expected error evaluating query but got none") 457 } 458 459 err := cmd.compareResult(res.Value) 460 if err != nil { 461 return fmt.Errorf("error in %s %s: %s", cmd, cmd.expr, err) 462 } 463 464 default: 465 panic("promql.Test.exec: unknown test command type") 466 } 467 return nil 468} 469 470// clear the current test storage of all inserted samples. 471func (t *Test) clear() { 472 if t.closeStorage != nil { 473 t.closeStorage() 474 } 475 if t.cancelCtx != nil { 476 t.cancelCtx() 477 } 478 479 var closer testutil.Closer 480 t.storage, closer = local.NewTestStorage(t, 2) 481 482 t.closeStorage = closer.Close 483 t.queryEngine = NewEngine(t.storage, nil) 484 t.context, t.cancelCtx = context.WithCancel(context.Background()) 485} 486 487// Close closes resources associated with the Test. 488func (t *Test) Close() { 489 t.cancelCtx() 490 t.closeStorage() 491} 492 493// samplesAlmostEqual returns true if the two sample lines only differ by a 494// small relative error in their sample value. 495func almostEqual(a, b float64) bool { 496 // NaN has no equality but for testing we still want to know whether both values 497 // are NaN. 498 if math.IsNaN(a) && math.IsNaN(b) { 499 return true 500 } 501 502 // Cf. http://floating-point-gui.de/errors/comparison/ 503 if a == b { 504 return true 505 } 506 507 diff := math.Abs(a - b) 508 509 if a == 0 || b == 0 || diff < minNormal { 510 return diff < epsilon*minNormal 511 } 512 return diff/(math.Abs(a)+math.Abs(b)) < epsilon 513} 514 515func parseNumber(s string) (float64, error) { 516 n, err := strconv.ParseInt(s, 0, 64) 517 f := float64(n) 518 if err != nil { 519 f, err = strconv.ParseFloat(s, 64) 520 } 521 if err != nil { 522 return 0, fmt.Errorf("error parsing number: %s", err) 523 } 524 return f, nil 525} 526