1// Copyright 2016 The Prometheus Authors
2// Licensed under the Apache License, Version 2.0 (the "License");
3// you may not use this file except in compliance with the License.
4// You may obtain a copy of the License at
5//
6//     http://www.apache.org/licenses/LICENSE-2.0
7//
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13
14package v1
15
16import (
17	"bytes"
18	"context"
19	"encoding/json"
20	"errors"
21	"fmt"
22	"io/ioutil"
23	"math"
24	"net/http"
25	"net/http/httptest"
26	"net/url"
27	"os"
28	"reflect"
29	"strings"
30	"testing"
31	"time"
32
33	"github.com/go-kit/kit/log"
34	"github.com/gogo/protobuf/proto"
35	"github.com/golang/snappy"
36	config_util "github.com/prometheus/common/config"
37	"github.com/prometheus/common/model"
38	"github.com/prometheus/common/promlog"
39	"github.com/prometheus/common/route"
40
41	"github.com/prometheus/prometheus/config"
42	"github.com/prometheus/prometheus/pkg/gate"
43	"github.com/prometheus/prometheus/pkg/labels"
44	"github.com/prometheus/prometheus/pkg/timestamp"
45	"github.com/prometheus/prometheus/prompb"
46	"github.com/prometheus/prometheus/promql"
47	"github.com/prometheus/prometheus/rules"
48	"github.com/prometheus/prometheus/scrape"
49	"github.com/prometheus/prometheus/storage"
50	"github.com/prometheus/prometheus/storage/remote"
51	"github.com/prometheus/prometheus/util/testutil"
52	tsdbLabels "github.com/prometheus/tsdb/labels"
53)
54
55type testTargetRetriever struct{}
56
57func (t testTargetRetriever) TargetsActive() map[string][]*scrape.Target {
58	return map[string][]*scrape.Target{
59		"test": {
60			scrape.NewTarget(
61				labels.FromMap(map[string]string{
62					model.SchemeLabel:      "http",
63					model.AddressLabel:     "example.com:8080",
64					model.MetricsPathLabel: "/metrics",
65					model.JobLabel:         "test",
66				}),
67				nil,
68				url.Values{},
69			),
70		},
71		"blackbox": {
72			scrape.NewTarget(
73				labels.FromMap(map[string]string{
74					model.SchemeLabel:      "http",
75					model.AddressLabel:     "localhost:9115",
76					model.MetricsPathLabel: "/probe",
77					model.JobLabel:         "blackbox",
78				}),
79				nil,
80				url.Values{"target": []string{"example.com"}},
81			),
82		},
83	}
84}
85func (t testTargetRetriever) TargetsDropped() map[string][]*scrape.Target {
86	return map[string][]*scrape.Target{
87		"blackbox": {
88			scrape.NewTarget(
89				nil,
90				labels.FromMap(map[string]string{
91					model.AddressLabel:     "http://dropped.example.com:9115",
92					model.MetricsPathLabel: "/probe",
93					model.SchemeLabel:      "http",
94					model.JobLabel:         "blackbox",
95				}),
96				url.Values{},
97			),
98		},
99	}
100}
101
102type testAlertmanagerRetriever struct{}
103
104func (t testAlertmanagerRetriever) Alertmanagers() []*url.URL {
105	return []*url.URL{
106		{
107			Scheme: "http",
108			Host:   "alertmanager.example.com:8080",
109			Path:   "/api/v1/alerts",
110		},
111	}
112}
113
114func (t testAlertmanagerRetriever) DroppedAlertmanagers() []*url.URL {
115	return []*url.URL{
116		{
117			Scheme: "http",
118			Host:   "dropped.alertmanager.example.com:8080",
119			Path:   "/api/v1/alerts",
120		},
121	}
122}
123
124type rulesRetrieverMock struct {
125	testing *testing.T
126}
127
128func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
129	expr1, err := promql.ParseExpr(`absent(test_metric3) != 1`)
130	if err != nil {
131		m.testing.Fatalf("unable to parse alert expression: %s", err)
132	}
133	expr2, err := promql.ParseExpr(`up == 1`)
134	if err != nil {
135		m.testing.Fatalf("Unable to parse alert expression: %s", err)
136	}
137
138	rule1 := rules.NewAlertingRule(
139		"test_metric3",
140		expr1,
141		time.Second,
142		labels.Labels{},
143		labels.Labels{},
144		true,
145		log.NewNopLogger(),
146	)
147	rule2 := rules.NewAlertingRule(
148		"test_metric4",
149		expr2,
150		time.Second,
151		labels.Labels{},
152		labels.Labels{},
153		true,
154		log.NewNopLogger(),
155	)
156	var r []*rules.AlertingRule
157	r = append(r, rule1)
158	r = append(r, rule2)
159	return r
160}
161
162func (m rulesRetrieverMock) RuleGroups() []*rules.Group {
163	var ar rulesRetrieverMock
164	arules := ar.AlertingRules()
165	storage := testutil.NewStorage(m.testing)
166	defer storage.Close()
167
168	engineOpts := promql.EngineOpts{
169		Logger:        nil,
170		Reg:           nil,
171		MaxConcurrent: 10,
172		MaxSamples:    10,
173		Timeout:       100 * time.Second,
174	}
175
176	engine := promql.NewEngine(engineOpts)
177	opts := &rules.ManagerOptions{
178		QueryFunc:  rules.EngineQueryFunc(engine, storage),
179		Appendable: storage,
180		Context:    context.Background(),
181		Logger:     log.NewNopLogger(),
182	}
183
184	var r []rules.Rule
185
186	for _, alertrule := range arules {
187		r = append(r, alertrule)
188	}
189
190	recordingExpr, err := promql.ParseExpr(`vector(1)`)
191	if err != nil {
192		m.testing.Fatalf("unable to parse alert expression: %s", err)
193	}
194	recordingRule := rules.NewRecordingRule("recording-rule-1", recordingExpr, labels.Labels{})
195	r = append(r, recordingRule)
196
197	group := rules.NewGroup("grp", "/path/to/file", time.Second, r, false, opts)
198	return []*rules.Group{group}
199}
200
201var samplePrometheusCfg = config.Config{
202	GlobalConfig:       config.GlobalConfig{},
203	AlertingConfig:     config.AlertingConfig{},
204	RuleFiles:          []string{},
205	ScrapeConfigs:      []*config.ScrapeConfig{},
206	RemoteWriteConfigs: []*config.RemoteWriteConfig{},
207	RemoteReadConfigs:  []*config.RemoteReadConfig{},
208}
209
210var sampleFlagMap = map[string]string{
211	"flag1": "value1",
212	"flag2": "value2",
213}
214
215func TestEndpoints(t *testing.T) {
216	suite, err := promql.NewTest(t, `
217		load 1m
218			test_metric1{foo="bar"} 0+100x100
219			test_metric1{foo="boo"} 1+0x100
220			test_metric2{foo="boo"} 1+0x100
221	`)
222	if err != nil {
223		t.Fatal(err)
224	}
225	defer suite.Close()
226
227	if err := suite.Run(); err != nil {
228		t.Fatal(err)
229	}
230
231	now := time.Now()
232
233	var algr rulesRetrieverMock
234	algr.testing = t
235	algr.AlertingRules()
236	algr.RuleGroups()
237
238	t.Run("local", func(t *testing.T) {
239		var algr rulesRetrieverMock
240		algr.testing = t
241
242		algr.AlertingRules()
243
244		algr.RuleGroups()
245
246		api := &API{
247			Queryable:             suite.Storage(),
248			QueryEngine:           suite.QueryEngine(),
249			targetRetriever:       testTargetRetriever{},
250			alertmanagerRetriever: testAlertmanagerRetriever{},
251			flagsMap:              sampleFlagMap,
252			now:                   func() time.Time { return now },
253			config:                func() config.Config { return samplePrometheusCfg },
254			ready:                 func(f http.HandlerFunc) http.HandlerFunc { return f },
255			rulesRetriever:        algr,
256		}
257
258		testEndpoints(t, api, true)
259	})
260
261	// Run all the API tests against a API that is wired to forward queries via
262	// the remote read client to a test server, which in turn sends them to the
263	// data from the test suite.
264	t.Run("remote", func(t *testing.T) {
265		server := setupRemote(suite.Storage())
266		defer server.Close()
267
268		u, err := url.Parse(server.URL)
269		if err != nil {
270			t.Fatal(err)
271		}
272
273		al := promlog.AllowedLevel{}
274		al.Set("debug")
275		af := promlog.AllowedFormat{}
276		al.Set("logfmt")
277		promlogConfig := promlog.Config{
278			Level:  &al,
279			Format: &af,
280		}
281
282		remote := remote.NewStorage(promlog.New(&promlogConfig), func() (int64, error) {
283			return 0, nil
284		}, 1*time.Second)
285
286		err = remote.ApplyConfig(&config.Config{
287			RemoteReadConfigs: []*config.RemoteReadConfig{
288				{
289					URL:           &config_util.URL{URL: u},
290					RemoteTimeout: model.Duration(1 * time.Second),
291					ReadRecent:    true,
292				},
293			},
294		})
295		if err != nil {
296			t.Fatal(err)
297		}
298
299		var algr rulesRetrieverMock
300		algr.testing = t
301
302		algr.AlertingRules()
303
304		algr.RuleGroups()
305
306		api := &API{
307			Queryable:             remote,
308			QueryEngine:           suite.QueryEngine(),
309			targetRetriever:       testTargetRetriever{},
310			alertmanagerRetriever: testAlertmanagerRetriever{},
311			flagsMap:              sampleFlagMap,
312			now:                   func() time.Time { return now },
313			config:                func() config.Config { return samplePrometheusCfg },
314			ready:                 func(f http.HandlerFunc) http.HandlerFunc { return f },
315			rulesRetriever:        algr,
316		}
317
318		testEndpoints(t, api, false)
319	})
320
321}
322
323func TestLabelNames(t *testing.T) {
324	// TestEndpoints doesn't have enough label names to test api.labelNames
325	// endpoint properly. Hence we test it separately.
326	suite, err := promql.NewTest(t, `
327		load 1m
328			test_metric1{foo1="bar", baz="abc"} 0+100x100
329			test_metric1{foo2="boo"} 1+0x100
330			test_metric2{foo="boo"} 1+0x100
331			test_metric2{foo="boo", xyz="qwerty"} 1+0x100
332	`)
333	testutil.Ok(t, err)
334	defer suite.Close()
335	testutil.Ok(t, suite.Run())
336
337	api := &API{
338		Queryable: suite.Storage(),
339	}
340	request := func(m string) (*http.Request, error) {
341		if m == http.MethodPost {
342			r, err := http.NewRequest(m, "http://example.com", nil)
343			r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
344			return r, err
345		}
346		return http.NewRequest(m, "http://example.com", nil)
347	}
348	for _, method := range []string{http.MethodGet, http.MethodPost} {
349		ctx := context.Background()
350		req, err := request(method)
351		testutil.Ok(t, err)
352		res := api.labelNames(req.WithContext(ctx))
353		assertAPIError(t, res.err, "")
354		assertAPIResponse(t, res.data, []string{"__name__", "baz", "foo", "foo1", "foo2", "xyz"})
355	}
356}
357
358func setupRemote(s storage.Storage) *httptest.Server {
359	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
360		req, err := remote.DecodeReadRequest(r)
361		if err != nil {
362			http.Error(w, err.Error(), http.StatusBadRequest)
363			return
364		}
365		resp := prompb.ReadResponse{
366			Results: make([]*prompb.QueryResult, len(req.Queries)),
367		}
368		for i, query := range req.Queries {
369			from, through, matchers, selectParams, err := remote.FromQuery(query)
370			if err != nil {
371				http.Error(w, err.Error(), http.StatusBadRequest)
372				return
373			}
374
375			querier, err := s.Querier(r.Context(), from, through)
376			if err != nil {
377				http.Error(w, err.Error(), http.StatusInternalServerError)
378				return
379			}
380			defer querier.Close()
381
382			set, err, _ := querier.Select(selectParams, matchers...)
383			if err != nil {
384				http.Error(w, err.Error(), http.StatusInternalServerError)
385				return
386			}
387			resp.Results[i], err = remote.ToQueryResult(set, 1e6)
388			if err != nil {
389				http.Error(w, err.Error(), http.StatusInternalServerError)
390				return
391			}
392		}
393
394		if err := remote.EncodeReadResponse(&resp, w); err != nil {
395			http.Error(w, err.Error(), http.StatusInternalServerError)
396			return
397		}
398	})
399
400	return httptest.NewServer(handler)
401}
402
403func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
404	start := time.Unix(0, 0)
405
406	type test struct {
407		endpoint apiFunc
408		params   map[string]string
409		query    url.Values
410		response interface{}
411		errType  errorType
412	}
413
414	var tests = []test{
415		{
416			endpoint: api.query,
417			query: url.Values{
418				"query": []string{"2"},
419				"time":  []string{"123.4"},
420			},
421			response: &queryData{
422				ResultType: promql.ValueTypeScalar,
423				Result: promql.Scalar{
424					V: 2,
425					T: timestamp.FromTime(start.Add(123*time.Second + 400*time.Millisecond)),
426				},
427			},
428		},
429		{
430			endpoint: api.query,
431			query: url.Values{
432				"query": []string{"0.333"},
433				"time":  []string{"1970-01-01T00:02:03Z"},
434			},
435			response: &queryData{
436				ResultType: promql.ValueTypeScalar,
437				Result: promql.Scalar{
438					V: 0.333,
439					T: timestamp.FromTime(start.Add(123 * time.Second)),
440				},
441			},
442		},
443		{
444			endpoint: api.query,
445			query: url.Values{
446				"query": []string{"0.333"},
447				"time":  []string{"1970-01-01T01:02:03+01:00"},
448			},
449			response: &queryData{
450				ResultType: promql.ValueTypeScalar,
451				Result: promql.Scalar{
452					V: 0.333,
453					T: timestamp.FromTime(start.Add(123 * time.Second)),
454				},
455			},
456		},
457		{
458			endpoint: api.query,
459			query: url.Values{
460				"query": []string{"0.333"},
461			},
462			response: &queryData{
463				ResultType: promql.ValueTypeScalar,
464				Result: promql.Scalar{
465					V: 0.333,
466					T: timestamp.FromTime(api.now()),
467				},
468			},
469		},
470		{
471			endpoint: api.queryRange,
472			query: url.Values{
473				"query": []string{"time()"},
474				"start": []string{"0"},
475				"end":   []string{"2"},
476				"step":  []string{"1"},
477			},
478			response: &queryData{
479				ResultType: promql.ValueTypeMatrix,
480				Result: promql.Matrix{
481					promql.Series{
482						Points: []promql.Point{
483							{V: 0, T: timestamp.FromTime(start)},
484							{V: 1, T: timestamp.FromTime(start.Add(1 * time.Second))},
485							{V: 2, T: timestamp.FromTime(start.Add(2 * time.Second))},
486						},
487						Metric: nil,
488					},
489				},
490			},
491		},
492		// Missing query params in range queries.
493		{
494			endpoint: api.queryRange,
495			query: url.Values{
496				"query": []string{"time()"},
497				"end":   []string{"2"},
498				"step":  []string{"1"},
499			},
500			errType: errorBadData,
501		},
502		{
503			endpoint: api.queryRange,
504			query: url.Values{
505				"query": []string{"time()"},
506				"start": []string{"0"},
507				"step":  []string{"1"},
508			},
509			errType: errorBadData,
510		},
511		{
512			endpoint: api.queryRange,
513			query: url.Values{
514				"query": []string{"time()"},
515				"start": []string{"0"},
516				"end":   []string{"2"},
517			},
518			errType: errorBadData,
519		},
520		// Bad query expression.
521		{
522			endpoint: api.query,
523			query: url.Values{
524				"query": []string{"invalid][query"},
525				"time":  []string{"1970-01-01T01:02:03+01:00"},
526			},
527			errType: errorBadData,
528		},
529		{
530			endpoint: api.queryRange,
531			query: url.Values{
532				"query": []string{"invalid][query"},
533				"start": []string{"0"},
534				"end":   []string{"100"},
535				"step":  []string{"1"},
536			},
537			errType: errorBadData,
538		},
539		// Invalid step.
540		{
541			endpoint: api.queryRange,
542			query: url.Values{
543				"query": []string{"time()"},
544				"start": []string{"1"},
545				"end":   []string{"2"},
546				"step":  []string{"0"},
547			},
548			errType: errorBadData,
549		},
550		// Start after end.
551		{
552			endpoint: api.queryRange,
553			query: url.Values{
554				"query": []string{"time()"},
555				"start": []string{"2"},
556				"end":   []string{"1"},
557				"step":  []string{"1"},
558			},
559			errType: errorBadData,
560		},
561		// Start overflows int64 internally.
562		{
563			endpoint: api.queryRange,
564			query: url.Values{
565				"query": []string{"time()"},
566				"start": []string{"148966367200.372"},
567				"end":   []string{"1489667272.372"},
568				"step":  []string{"1"},
569			},
570			errType: errorBadData,
571		},
572		{
573			endpoint: api.series,
574			query: url.Values{
575				"match[]": []string{`test_metric2`},
576			},
577			response: []labels.Labels{
578				labels.FromStrings("__name__", "test_metric2", "foo", "boo"),
579			},
580		},
581		{
582			endpoint: api.series,
583			query: url.Values{
584				"match[]": []string{`test_metric1{foo=~".+o"}`},
585			},
586			response: []labels.Labels{
587				labels.FromStrings("__name__", "test_metric1", "foo", "boo"),
588			},
589		},
590		{
591			endpoint: api.series,
592			query: url.Values{
593				"match[]": []string{`test_metric1{foo=~".+o$"}`, `test_metric1{foo=~".+o"}`},
594			},
595			response: []labels.Labels{
596				labels.FromStrings("__name__", "test_metric1", "foo", "boo"),
597			},
598		},
599		{
600			endpoint: api.series,
601			query: url.Values{
602				"match[]": []string{`test_metric1{foo=~".+o"}`, `none`},
603			},
604			response: []labels.Labels{
605				labels.FromStrings("__name__", "test_metric1", "foo", "boo"),
606			},
607		},
608		// Start and end before series starts.
609		{
610			endpoint: api.series,
611			query: url.Values{
612				"match[]": []string{`test_metric2`},
613				"start":   []string{"-2"},
614				"end":     []string{"-1"},
615			},
616			response: []labels.Labels{},
617		},
618		// Start and end after series ends.
619		{
620			endpoint: api.series,
621			query: url.Values{
622				"match[]": []string{`test_metric2`},
623				"start":   []string{"100000"},
624				"end":     []string{"100001"},
625			},
626			response: []labels.Labels{},
627		},
628		// Start before series starts, end after series ends.
629		{
630			endpoint: api.series,
631			query: url.Values{
632				"match[]": []string{`test_metric2`},
633				"start":   []string{"-1"},
634				"end":     []string{"100000"},
635			},
636			response: []labels.Labels{
637				labels.FromStrings("__name__", "test_metric2", "foo", "boo"),
638			},
639		},
640		// Start and end within series.
641		{
642			endpoint: api.series,
643			query: url.Values{
644				"match[]": []string{`test_metric2`},
645				"start":   []string{"1"},
646				"end":     []string{"100"},
647			},
648			response: []labels.Labels{
649				labels.FromStrings("__name__", "test_metric2", "foo", "boo"),
650			},
651		},
652		// Start within series, end after.
653		{
654			endpoint: api.series,
655			query: url.Values{
656				"match[]": []string{`test_metric2`},
657				"start":   []string{"1"},
658				"end":     []string{"100000"},
659			},
660			response: []labels.Labels{
661				labels.FromStrings("__name__", "test_metric2", "foo", "boo"),
662			},
663		},
664		// Start before series, end within series.
665		{
666			endpoint: api.series,
667			query: url.Values{
668				"match[]": []string{`test_metric2`},
669				"start":   []string{"-1"},
670				"end":     []string{"1"},
671			},
672			response: []labels.Labels{
673				labels.FromStrings("__name__", "test_metric2", "foo", "boo"),
674			},
675		},
676		// Missing match[] query params in series requests.
677		{
678			endpoint: api.series,
679			errType:  errorBadData,
680		},
681		{
682			endpoint: api.dropSeries,
683			errType:  errorInternal,
684		},
685		{
686			endpoint: api.targets,
687			response: &TargetDiscovery{
688				ActiveTargets: []*Target{
689					{
690						DiscoveredLabels: map[string]string{},
691						Labels: map[string]string{
692							"job": "blackbox",
693						},
694						ScrapeURL: "http://localhost:9115/probe?target=example.com",
695						Health:    "unknown",
696					},
697					{
698						DiscoveredLabels: map[string]string{},
699						Labels: map[string]string{
700							"job": "test",
701						},
702						ScrapeURL: "http://example.com:8080/metrics",
703						Health:    "unknown",
704					},
705				},
706				DroppedTargets: []*DroppedTarget{
707					{
708						DiscoveredLabels: map[string]string{
709							"__address__":      "http://dropped.example.com:9115",
710							"__metrics_path__": "/probe",
711							"__scheme__":       "http",
712							"job":              "blackbox",
713						},
714					},
715				},
716			},
717		},
718		{
719			endpoint: api.alertmanagers,
720			response: &AlertmanagerDiscovery{
721				ActiveAlertmanagers: []*AlertmanagerTarget{
722					{
723						URL: "http://alertmanager.example.com:8080/api/v1/alerts",
724					},
725				},
726				DroppedAlertmanagers: []*AlertmanagerTarget{
727					{
728						URL: "http://dropped.alertmanager.example.com:8080/api/v1/alerts",
729					},
730				},
731			},
732		},
733		{
734			endpoint: api.serveConfig,
735			response: &prometheusConfig{
736				YAML: samplePrometheusCfg.String(),
737			},
738		},
739		{
740			endpoint: api.serveFlags,
741			response: sampleFlagMap,
742		},
743		{
744			endpoint: api.alerts,
745			response: &AlertDiscovery{
746				Alerts: []*Alert{},
747			},
748		},
749		{
750			endpoint: api.rules,
751			response: &RuleDiscovery{
752				RuleGroups: []*RuleGroup{
753					{
754						Name:     "grp",
755						File:     "/path/to/file",
756						Interval: 1,
757						Rules: []rule{
758							alertingRule{
759								Name:        "test_metric3",
760								Query:       "absent(test_metric3) != 1",
761								Duration:    1,
762								Labels:      labels.Labels{},
763								Annotations: labels.Labels{},
764								Alerts:      []*Alert{},
765								Health:      "unknown",
766								Type:        "alerting",
767							},
768							alertingRule{
769								Name:        "test_metric4",
770								Query:       "up == 1",
771								Duration:    1,
772								Labels:      labels.Labels{},
773								Annotations: labels.Labels{},
774								Alerts:      []*Alert{},
775								Health:      "unknown",
776								Type:        "alerting",
777							},
778							recordingRule{
779								Name:   "recording-rule-1",
780								Query:  "vector(1)",
781								Labels: labels.Labels{},
782								Health: "unknown",
783								Type:   "recording",
784							},
785						},
786					},
787				},
788			},
789		},
790	}
791
792	if testLabelAPI {
793		tests = append(tests, []test{
794			{
795				endpoint: api.labelValues,
796				params: map[string]string{
797					"name": "__name__",
798				},
799				response: []string{
800					"test_metric1",
801					"test_metric2",
802				},
803			},
804			{
805				endpoint: api.labelValues,
806				params: map[string]string{
807					"name": "foo",
808				},
809				response: []string{
810					"bar",
811					"boo",
812				},
813			},
814			// Bad name parameter.
815			{
816				endpoint: api.labelValues,
817				params: map[string]string{
818					"name": "not!!!allowed",
819				},
820				errType: errorBadData,
821			},
822			// Label names.
823			{
824				endpoint: api.labelNames,
825				response: []string{"__name__", "foo"},
826			},
827		}...)
828	}
829
830	methods := func(f apiFunc) []string {
831		fp := reflect.ValueOf(f).Pointer()
832		if fp == reflect.ValueOf(api.query).Pointer() || fp == reflect.ValueOf(api.queryRange).Pointer() {
833			return []string{http.MethodGet, http.MethodPost}
834		}
835		return []string{http.MethodGet}
836	}
837
838	request := func(m string, q url.Values) (*http.Request, error) {
839		if m == http.MethodPost {
840			r, err := http.NewRequest(m, "http://example.com", strings.NewReader(q.Encode()))
841			r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
842			return r, err
843		}
844		return http.NewRequest(m, fmt.Sprintf("http://example.com?%s", q.Encode()), nil)
845	}
846
847	for i, test := range tests {
848		for _, method := range methods(test.endpoint) {
849			// Build a context with the correct request params.
850			ctx := context.Background()
851			for p, v := range test.params {
852				ctx = route.WithParam(ctx, p, v)
853			}
854			t.Logf("run %d\t%s\t%q", i, method, test.query.Encode())
855
856			req, err := request(method, test.query)
857			if err != nil {
858				t.Fatal(err)
859			}
860			res := test.endpoint(req.WithContext(ctx))
861			assertAPIError(t, res.err, test.errType)
862			assertAPIResponse(t, res.data, test.response)
863		}
864	}
865}
866
867func assertAPIError(t *testing.T, got *apiError, exp errorType) {
868	t.Helper()
869
870	if got != nil {
871		if exp == errorNone {
872			t.Fatalf("Unexpected error: %s", got)
873		}
874		if exp != got.typ {
875			t.Fatalf("Expected error of type %q but got type %q (%q)", exp, got.typ, got)
876		}
877		return
878	}
879	if got == nil && exp != errorNone {
880		t.Fatalf("Expected error of type %q but got none", exp)
881	}
882}
883
884func assertAPIResponse(t *testing.T, got interface{}, exp interface{}) {
885	if !reflect.DeepEqual(exp, got) {
886		respJSON, err := json.Marshal(got)
887		if err != nil {
888			t.Fatalf("failed to marshal response as JSON: %v", err.Error())
889		}
890
891		expectedRespJSON, err := json.Marshal(exp)
892		if err != nil {
893			t.Fatalf("failed to marshal expected response as JSON: %v", err.Error())
894		}
895
896		t.Fatalf(
897			"Response does not match, expected:\n%+v\ngot:\n%+v",
898			string(expectedRespJSON),
899			string(respJSON),
900		)
901	}
902}
903
904func TestReadEndpoint(t *testing.T) {
905	suite, err := promql.NewTest(t, `
906		load 1m
907			test_metric1{foo="bar",baz="qux"} 1
908	`)
909	if err != nil {
910		t.Fatal(err)
911	}
912	defer suite.Close()
913
914	if err := suite.Run(); err != nil {
915		t.Fatal(err)
916	}
917
918	api := &API{
919		Queryable:   suite.Storage(),
920		QueryEngine: suite.QueryEngine(),
921		config: func() config.Config {
922			return config.Config{
923				GlobalConfig: config.GlobalConfig{
924					ExternalLabels: model.LabelSet{
925						"baz": "a",
926						"b":   "c",
927						"d":   "e",
928					},
929				},
930			}
931		},
932		remoteReadSampleLimit: 1e6,
933		remoteReadGate:        gate.New(1),
934	}
935
936	// Encode the request.
937	matcher1, err := labels.NewMatcher(labels.MatchEqual, "__name__", "test_metric1")
938	if err != nil {
939		t.Fatal(err)
940	}
941	matcher2, err := labels.NewMatcher(labels.MatchEqual, "d", "e")
942	if err != nil {
943		t.Fatal(err)
944	}
945	query, err := remote.ToQuery(0, 1, []*labels.Matcher{matcher1, matcher2}, &storage.SelectParams{Step: 0, Func: "avg"})
946	if err != nil {
947		t.Fatal(err)
948	}
949	req := &prompb.ReadRequest{Queries: []*prompb.Query{query}}
950	data, err := proto.Marshal(req)
951	if err != nil {
952		t.Fatal(err)
953	}
954	compressed := snappy.Encode(nil, data)
955	request, err := http.NewRequest("POST", "", bytes.NewBuffer(compressed))
956	if err != nil {
957		t.Fatal(err)
958	}
959	recorder := httptest.NewRecorder()
960	api.remoteRead(recorder, request)
961
962	if recorder.Code/100 != 2 {
963		t.Fatal(recorder.Code)
964	}
965
966	// Decode the response.
967	compressed, err = ioutil.ReadAll(recorder.Result().Body)
968	if err != nil {
969		t.Fatal(err)
970	}
971	uncompressed, err := snappy.Decode(nil, compressed)
972	if err != nil {
973		t.Fatal(err)
974	}
975
976	var resp prompb.ReadResponse
977	err = proto.Unmarshal(uncompressed, &resp)
978	if err != nil {
979		t.Fatal(err)
980	}
981
982	if len(resp.Results) != 1 {
983		t.Fatalf("Expected 1 result, got %d", len(resp.Results))
984	}
985
986	result := resp.Results[0]
987	expected := &prompb.QueryResult{
988		Timeseries: []*prompb.TimeSeries{
989			{
990				Labels: []*prompb.Label{
991					{Name: "__name__", Value: "test_metric1"},
992					{Name: "b", Value: "c"},
993					{Name: "baz", Value: "qux"},
994					{Name: "d", Value: "e"},
995					{Name: "foo", Value: "bar"},
996				},
997				Samples: []prompb.Sample{{Value: 1, Timestamp: 0}},
998			},
999		},
1000	}
1001	if !reflect.DeepEqual(result, expected) {
1002		t.Fatalf("Expected response \n%v\n but got \n%v\n", result, expected)
1003	}
1004}
1005
1006type fakeDB struct {
1007	err    error
1008	closer func()
1009}
1010
1011func (f *fakeDB) CleanTombstones() error                                  { return f.err }
1012func (f *fakeDB) Delete(mint, maxt int64, ms ...tsdbLabels.Matcher) error { return f.err }
1013func (f *fakeDB) Dir() string {
1014	dir, _ := ioutil.TempDir("", "fakeDB")
1015	f.closer = func() {
1016		os.RemoveAll(dir)
1017	}
1018	return dir
1019}
1020func (f *fakeDB) Snapshot(dir string, withHead bool) error { return f.err }
1021
1022func TestAdminEndpoints(t *testing.T) {
1023	tsdb, tsdbWithError := &fakeDB{}, &fakeDB{err: fmt.Errorf("some error")}
1024	snapshotAPI := func(api *API) apiFunc { return api.snapshot }
1025	cleanAPI := func(api *API) apiFunc { return api.cleanTombstones }
1026	deleteAPI := func(api *API) apiFunc { return api.deleteSeries }
1027
1028	for i, tc := range []struct {
1029		db          *fakeDB
1030		enableAdmin bool
1031		endpoint    func(api *API) apiFunc
1032		method      string
1033		values      url.Values
1034
1035		errType errorType
1036	}{
1037		// Tests for the snapshot endpoint.
1038		{
1039			db:          tsdb,
1040			enableAdmin: false,
1041			endpoint:    snapshotAPI,
1042
1043			errType: errorUnavailable,
1044		},
1045		{
1046			db:          tsdb,
1047			enableAdmin: true,
1048			endpoint:    snapshotAPI,
1049
1050			errType: errorNone,
1051		},
1052		{
1053			db:          tsdb,
1054			enableAdmin: true,
1055			endpoint:    snapshotAPI,
1056			values:      map[string][]string{"skip_head": []string{"true"}},
1057
1058			errType: errorNone,
1059		},
1060		{
1061			db:          tsdb,
1062			enableAdmin: true,
1063			endpoint:    snapshotAPI,
1064			values:      map[string][]string{"skip_head": []string{"xxx"}},
1065
1066			errType: errorBadData,
1067		},
1068		{
1069			db:          tsdbWithError,
1070			enableAdmin: true,
1071			endpoint:    snapshotAPI,
1072
1073			errType: errorInternal,
1074		},
1075		{
1076			db:          nil,
1077			enableAdmin: true,
1078			endpoint:    snapshotAPI,
1079
1080			errType: errorUnavailable,
1081		},
1082		// Tests for the cleanTombstones endpoint.
1083		{
1084			db:          tsdb,
1085			enableAdmin: false,
1086			endpoint:    cleanAPI,
1087
1088			errType: errorUnavailable,
1089		},
1090		{
1091			db:          tsdb,
1092			enableAdmin: true,
1093			endpoint:    cleanAPI,
1094
1095			errType: errorNone,
1096		},
1097		{
1098			db:          tsdbWithError,
1099			enableAdmin: true,
1100			endpoint:    cleanAPI,
1101
1102			errType: errorInternal,
1103		},
1104		{
1105			db:          nil,
1106			enableAdmin: true,
1107			endpoint:    cleanAPI,
1108
1109			errType: errorUnavailable,
1110		},
1111		// Tests for the deleteSeries endpoint.
1112		{
1113			db:          tsdb,
1114			enableAdmin: false,
1115			endpoint:    deleteAPI,
1116
1117			errType: errorUnavailable,
1118		},
1119		{
1120			db:          tsdb,
1121			enableAdmin: true,
1122			endpoint:    deleteAPI,
1123
1124			errType: errorBadData,
1125		},
1126		{
1127			db:          tsdb,
1128			enableAdmin: true,
1129			endpoint:    deleteAPI,
1130			values:      map[string][]string{"match[]": []string{"123"}},
1131
1132			errType: errorBadData,
1133		},
1134		{
1135			db:          tsdb,
1136			enableAdmin: true,
1137			endpoint:    deleteAPI,
1138			values:      map[string][]string{"match[]": []string{"up"}, "start": []string{"xxx"}},
1139
1140			errType: errorBadData,
1141		},
1142		{
1143			db:          tsdb,
1144			enableAdmin: true,
1145			endpoint:    deleteAPI,
1146			values:      map[string][]string{"match[]": []string{"up"}, "end": []string{"xxx"}},
1147
1148			errType: errorBadData,
1149		},
1150		{
1151			db:          tsdb,
1152			enableAdmin: true,
1153			endpoint:    deleteAPI,
1154			values:      map[string][]string{"match[]": []string{"up"}},
1155
1156			errType: errorNone,
1157		},
1158		{
1159			db:          tsdb,
1160			enableAdmin: true,
1161			endpoint:    deleteAPI,
1162			values:      map[string][]string{"match[]": []string{"up{job!=\"foo\"}", "{job=~\"bar.+\"}", "up{instance!~\"fred.+\"}"}},
1163
1164			errType: errorNone,
1165		},
1166		{
1167			db:          tsdbWithError,
1168			enableAdmin: true,
1169			endpoint:    deleteAPI,
1170			values:      map[string][]string{"match[]": []string{"up"}},
1171
1172			errType: errorInternal,
1173		},
1174		{
1175			db:          nil,
1176			enableAdmin: true,
1177			endpoint:    deleteAPI,
1178
1179			errType: errorUnavailable,
1180		},
1181	} {
1182		tc := tc
1183		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
1184			api := &API{
1185				db: func() TSDBAdmin {
1186					if tc.db != nil {
1187						return tc.db
1188					}
1189					return nil
1190				},
1191				ready:       func(f http.HandlerFunc) http.HandlerFunc { return f },
1192				enableAdmin: tc.enableAdmin,
1193			}
1194			defer func() {
1195				if tc.db != nil && tc.db.closer != nil {
1196					tc.db.closer()
1197				}
1198			}()
1199
1200			endpoint := tc.endpoint(api)
1201			req, err := http.NewRequest(tc.method, fmt.Sprintf("?%s", tc.values.Encode()), nil)
1202			if err != nil {
1203				t.Fatalf("Error when creating test request: %s", err)
1204			}
1205			res := endpoint(req)
1206			assertAPIError(t, res.err, tc.errType)
1207		})
1208	}
1209}
1210
1211func TestRespondSuccess(t *testing.T) {
1212	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1213		api := API{}
1214		api.respond(w, "test", nil)
1215	}))
1216	defer s.Close()
1217
1218	resp, err := http.Get(s.URL)
1219	if err != nil {
1220		t.Fatalf("Error on test request: %s", err)
1221	}
1222	body, err := ioutil.ReadAll(resp.Body)
1223	defer resp.Body.Close()
1224	if err != nil {
1225		t.Fatalf("Error reading response body: %s", err)
1226	}
1227
1228	if resp.StatusCode != 200 {
1229		t.Fatalf("Return code %d expected in success response but got %d", 200, resp.StatusCode)
1230	}
1231	if h := resp.Header.Get("Content-Type"); h != "application/json" {
1232		t.Fatalf("Expected Content-Type %q but got %q", "application/json", h)
1233	}
1234
1235	var res response
1236	if err = json.Unmarshal([]byte(body), &res); err != nil {
1237		t.Fatalf("Error unmarshaling JSON body: %s", err)
1238	}
1239
1240	exp := &response{
1241		Status: statusSuccess,
1242		Data:   "test",
1243	}
1244	if !reflect.DeepEqual(&res, exp) {
1245		t.Fatalf("Expected response \n%v\n but got \n%v\n", res, exp)
1246	}
1247}
1248
1249func TestRespondError(t *testing.T) {
1250	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1251		api := API{}
1252		api.respondError(w, &apiError{errorTimeout, errors.New("message")}, "test")
1253	}))
1254	defer s.Close()
1255
1256	resp, err := http.Get(s.URL)
1257	if err != nil {
1258		t.Fatalf("Error on test request: %s", err)
1259	}
1260	body, err := ioutil.ReadAll(resp.Body)
1261	defer resp.Body.Close()
1262	if err != nil {
1263		t.Fatalf("Error reading response body: %s", err)
1264	}
1265
1266	if want, have := http.StatusServiceUnavailable, resp.StatusCode; want != have {
1267		t.Fatalf("Return code %d expected in error response but got %d", want, have)
1268	}
1269	if h := resp.Header.Get("Content-Type"); h != "application/json" {
1270		t.Fatalf("Expected Content-Type %q but got %q", "application/json", h)
1271	}
1272
1273	var res response
1274	if err = json.Unmarshal([]byte(body), &res); err != nil {
1275		t.Fatalf("Error unmarshaling JSON body: %s", err)
1276	}
1277
1278	exp := &response{
1279		Status:    statusError,
1280		Data:      "test",
1281		ErrorType: errorTimeout,
1282		Error:     "message",
1283	}
1284	if !reflect.DeepEqual(&res, exp) {
1285		t.Fatalf("Expected response \n%v\n but got \n%v\n", res, exp)
1286	}
1287}
1288
1289func TestParseTime(t *testing.T) {
1290	ts, err := time.Parse(time.RFC3339Nano, "2015-06-03T13:21:58.555Z")
1291	if err != nil {
1292		panic(err)
1293	}
1294
1295	var tests = []struct {
1296		input  string
1297		fail   bool
1298		result time.Time
1299	}{
1300		{
1301			input: "",
1302			fail:  true,
1303		}, {
1304			input: "abc",
1305			fail:  true,
1306		}, {
1307			input: "30s",
1308			fail:  true,
1309		}, {
1310			input:  "123",
1311			result: time.Unix(123, 0),
1312		}, {
1313			input:  "123.123",
1314			result: time.Unix(123, 123000000),
1315		}, {
1316			input:  "2015-06-03T13:21:58.555Z",
1317			result: ts,
1318		}, {
1319			input:  "2015-06-03T14:21:58.555+01:00",
1320			result: ts,
1321		}, {
1322			// Test float rounding.
1323			input:  "1543578564.705",
1324			result: time.Unix(1543578564, 705*1e6),
1325		},
1326	}
1327
1328	for _, test := range tests {
1329		ts, err := parseTime(test.input)
1330		if err != nil && !test.fail {
1331			t.Errorf("Unexpected error for %q: %s", test.input, err)
1332			continue
1333		}
1334		if err == nil && test.fail {
1335			t.Errorf("Expected error for %q but got none", test.input)
1336			continue
1337		}
1338		if !test.fail && !ts.Equal(test.result) {
1339			t.Errorf("Expected time %v for input %q but got %v", test.result, test.input, ts)
1340		}
1341	}
1342}
1343
1344func TestParseDuration(t *testing.T) {
1345	var tests = []struct {
1346		input  string
1347		fail   bool
1348		result time.Duration
1349	}{
1350		{
1351			input: "",
1352			fail:  true,
1353		}, {
1354			input: "abc",
1355			fail:  true,
1356		}, {
1357			input: "2015-06-03T13:21:58.555Z",
1358			fail:  true,
1359		}, {
1360			// Internal int64 overflow.
1361			input: "-148966367200.372",
1362			fail:  true,
1363		}, {
1364			// Internal int64 overflow.
1365			input: "148966367200.372",
1366			fail:  true,
1367		}, {
1368			input:  "123",
1369			result: 123 * time.Second,
1370		}, {
1371			input:  "123.333",
1372			result: 123*time.Second + 333*time.Millisecond,
1373		}, {
1374			input:  "15s",
1375			result: 15 * time.Second,
1376		}, {
1377			input:  "5m",
1378			result: 5 * time.Minute,
1379		},
1380	}
1381
1382	for _, test := range tests {
1383		d, err := parseDuration(test.input)
1384		if err != nil && !test.fail {
1385			t.Errorf("Unexpected error for %q: %s", test.input, err)
1386			continue
1387		}
1388		if err == nil && test.fail {
1389			t.Errorf("Expected error for %q but got none", test.input)
1390			continue
1391		}
1392		if !test.fail && d != test.result {
1393			t.Errorf("Expected duration %v for input %q but got %v", test.result, test.input, d)
1394		}
1395	}
1396}
1397
1398func TestOptionsMethod(t *testing.T) {
1399	r := route.New()
1400	api := &API{ready: func(f http.HandlerFunc) http.HandlerFunc { return f }}
1401	api.Register(r)
1402
1403	s := httptest.NewServer(r)
1404	defer s.Close()
1405
1406	req, err := http.NewRequest("OPTIONS", s.URL+"/any_path", nil)
1407	if err != nil {
1408		t.Fatalf("Error creating OPTIONS request: %s", err)
1409	}
1410	client := &http.Client{}
1411	resp, err := client.Do(req)
1412	if err != nil {
1413		t.Fatalf("Error executing OPTIONS request: %s", err)
1414	}
1415
1416	if resp.StatusCode != http.StatusNoContent {
1417		t.Fatalf("Expected status %d, got %d", http.StatusNoContent, resp.StatusCode)
1418	}
1419
1420	for h, v := range corsHeaders {
1421		if resp.Header.Get(h) != v {
1422			t.Fatalf("Expected %q for header %q, got %q", v, h, resp.Header.Get(h))
1423		}
1424	}
1425}
1426
1427func TestRespond(t *testing.T) {
1428	cases := []struct {
1429		response interface{}
1430		expected string
1431	}{
1432		{
1433			response: &queryData{
1434				ResultType: promql.ValueTypeMatrix,
1435				Result: promql.Matrix{
1436					promql.Series{
1437						Points: []promql.Point{{V: 1, T: 1000}},
1438						Metric: labels.FromStrings("__name__", "foo"),
1439					},
1440				},
1441			},
1442			expected: `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"foo"},"values":[[1,"1"]]}]}}`,
1443		},
1444		{
1445			response: promql.Point{V: 0, T: 0},
1446			expected: `{"status":"success","data":[0,"0"]}`,
1447		},
1448		{
1449			response: promql.Point{V: 20, T: 1},
1450			expected: `{"status":"success","data":[0.001,"20"]}`,
1451		},
1452		{
1453			response: promql.Point{V: 20, T: 10},
1454			expected: `{"status":"success","data":[0.010,"20"]}`,
1455		},
1456		{
1457			response: promql.Point{V: 20, T: 100},
1458			expected: `{"status":"success","data":[0.100,"20"]}`,
1459		},
1460		{
1461			response: promql.Point{V: 20, T: 1001},
1462			expected: `{"status":"success","data":[1.001,"20"]}`,
1463		},
1464		{
1465			response: promql.Point{V: 20, T: 1010},
1466			expected: `{"status":"success","data":[1.010,"20"]}`,
1467		},
1468		{
1469			response: promql.Point{V: 20, T: 1100},
1470			expected: `{"status":"success","data":[1.100,"20"]}`,
1471		},
1472		{
1473			response: promql.Point{V: 20, T: 12345678123456555},
1474			expected: `{"status":"success","data":[12345678123456.555,"20"]}`,
1475		},
1476		{
1477			response: promql.Point{V: 20, T: -1},
1478			expected: `{"status":"success","data":[-0.001,"20"]}`,
1479		},
1480		{
1481			response: promql.Point{V: math.NaN(), T: 0},
1482			expected: `{"status":"success","data":[0,"NaN"]}`,
1483		},
1484		{
1485			response: promql.Point{V: math.Inf(1), T: 0},
1486			expected: `{"status":"success","data":[0,"+Inf"]}`,
1487		},
1488		{
1489			response: promql.Point{V: math.Inf(-1), T: 0},
1490			expected: `{"status":"success","data":[0,"-Inf"]}`,
1491		},
1492		{
1493			response: promql.Point{V: 1.2345678e6, T: 0},
1494			expected: `{"status":"success","data":[0,"1234567.8"]}`,
1495		},
1496		{
1497			response: promql.Point{V: 1.2345678e-6, T: 0},
1498			expected: `{"status":"success","data":[0,"0.0000012345678"]}`,
1499		},
1500		{
1501			response: promql.Point{V: 1.2345678e-67, T: 0},
1502			expected: `{"status":"success","data":[0,"1.2345678e-67"]}`,
1503		},
1504	}
1505
1506	for _, c := range cases {
1507		s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1508			api := API{}
1509			api.respond(w, c.response, nil)
1510		}))
1511		defer s.Close()
1512
1513		resp, err := http.Get(s.URL)
1514		if err != nil {
1515			t.Fatalf("Error on test request: %s", err)
1516		}
1517		body, err := ioutil.ReadAll(resp.Body)
1518		defer resp.Body.Close()
1519		if err != nil {
1520			t.Fatalf("Error reading response body: %s", err)
1521		}
1522
1523		if string(body) != c.expected {
1524			t.Fatalf("Expected response \n%v\n but got \n%v\n", c.expected, string(body))
1525		}
1526	}
1527}
1528
1529// This is a global to avoid the benchmark being optimized away.
1530var testResponseWriter = httptest.ResponseRecorder{}
1531
1532func BenchmarkRespond(b *testing.B) {
1533	b.ReportAllocs()
1534	points := []promql.Point{}
1535	for i := 0; i < 10000; i++ {
1536		points = append(points, promql.Point{V: float64(i * 1000000), T: int64(i)})
1537	}
1538	response := &queryData{
1539		ResultType: promql.ValueTypeMatrix,
1540		Result: promql.Matrix{
1541			promql.Series{
1542				Points: points,
1543				Metric: nil,
1544			},
1545		},
1546	}
1547	b.ResetTimer()
1548	api := API{}
1549	for n := 0; n < b.N; n++ {
1550		api.respond(&testResponseWriter, response, nil)
1551	}
1552}
1553