1// Copyright 2016 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package web 15 16import ( 17 "bytes" 18 "context" 19 "net/http" 20 "net/http/httptest" 21 "sort" 22 "strings" 23 "testing" 24 "time" 25 26 "github.com/pkg/errors" 27 "github.com/prometheus/common/model" 28 "github.com/stretchr/testify/require" 29 30 "github.com/prometheus/prometheus/config" 31 "github.com/prometheus/prometheus/pkg/labels" 32 "github.com/prometheus/prometheus/promql" 33 "github.com/prometheus/prometheus/storage" 34 "github.com/prometheus/prometheus/tsdb" 35) 36 37var scenarios = map[string]struct { 38 params string 39 externalLabels labels.Labels 40 code int 41 body string 42}{ 43 "empty": { 44 params: "", 45 code: 200, 46 body: ``, 47 }, 48 "match nothing": { 49 params: "match[]=does_not_match_anything", 50 code: 200, 51 body: ``, 52 }, 53 "invalid params from the beginning": { 54 params: "match[]=-not-a-valid-metric-name", 55 code: 400, 56 body: `1:1: parse error: unexpected <op:-> 57`, 58 }, 59 "invalid params somewhere in the middle": { 60 params: "match[]=not-a-valid-metric-name", 61 code: 400, 62 body: `1:4: parse error: unexpected <op:-> 63`, 64 }, 65 "test_metric1": { 66 params: "match[]=test_metric1", 67 code: 200, 68 body: `# TYPE test_metric1 untyped 69test_metric1{foo="bar",instance="i"} 10000 6000000 70test_metric1{foo="boo",instance="i"} 1 6000000 71`, 72 }, 73 "test_metric2": { 74 params: "match[]=test_metric2", 75 code: 200, 76 body: `# TYPE test_metric2 untyped 77test_metric2{foo="boo",instance="i"} 1 6000000 78`, 79 }, 80 "test_metric_without_labels": { 81 params: "match[]=test_metric_without_labels", 82 code: 200, 83 body: `# TYPE test_metric_without_labels untyped 84test_metric_without_labels{instance=""} 1001 6000000 85`, 86 }, 87 "test_stale_metric": { 88 params: "match[]=test_metric_stale", 89 code: 200, 90 body: ``, 91 }, 92 "test_old_metric": { 93 params: "match[]=test_metric_old", 94 code: 200, 95 body: `# TYPE test_metric_old untyped 96test_metric_old{instance=""} 981 5880000 97`, 98 }, 99 "{foo='boo'}": { 100 params: "match[]={foo='boo'}", 101 code: 200, 102 body: `# TYPE test_metric1 untyped 103test_metric1{foo="boo",instance="i"} 1 6000000 104# TYPE test_metric2 untyped 105test_metric2{foo="boo",instance="i"} 1 6000000 106`, 107 }, 108 "two matchers": { 109 params: "match[]=test_metric1&match[]=test_metric2", 110 code: 200, 111 body: `# TYPE test_metric1 untyped 112test_metric1{foo="bar",instance="i"} 10000 6000000 113test_metric1{foo="boo",instance="i"} 1 6000000 114# TYPE test_metric2 untyped 115test_metric2{foo="boo",instance="i"} 1 6000000 116`, 117 }, 118 "everything": { 119 params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'. 120 code: 200, 121 body: `# TYPE test_metric1 untyped 122test_metric1{foo="bar",instance="i"} 10000 6000000 123test_metric1{foo="boo",instance="i"} 1 6000000 124# TYPE test_metric2 untyped 125test_metric2{foo="boo",instance="i"} 1 6000000 126# TYPE test_metric_old untyped 127test_metric_old{instance=""} 981 5880000 128# TYPE test_metric_without_labels untyped 129test_metric_without_labels{instance=""} 1001 6000000 130`, 131 }, 132 "empty label value matches everything that doesn't have that label": { 133 params: "match[]={foo='',__name__=~'.%2b'}", 134 code: 200, 135 body: `# TYPE test_metric_old untyped 136test_metric_old{instance=""} 981 5880000 137# TYPE test_metric_without_labels untyped 138test_metric_without_labels{instance=""} 1001 6000000 139`, 140 }, 141 "empty label value for a label that doesn't exist at all, matches everything": { 142 params: "match[]={bar='',__name__=~'.%2b'}", 143 code: 200, 144 body: `# TYPE test_metric1 untyped 145test_metric1{foo="bar",instance="i"} 10000 6000000 146test_metric1{foo="boo",instance="i"} 1 6000000 147# TYPE test_metric2 untyped 148test_metric2{foo="boo",instance="i"} 1 6000000 149# TYPE test_metric_old untyped 150test_metric_old{instance=""} 981 5880000 151# TYPE test_metric_without_labels untyped 152test_metric_without_labels{instance=""} 1001 6000000 153`, 154 }, 155 "external labels are added if not already present": { 156 params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'. 157 externalLabels: labels.Labels{{Name: "zone", Value: "ie"}, {Name: "foo", Value: "baz"}}, 158 code: 200, 159 body: `# TYPE test_metric1 untyped 160test_metric1{foo="bar",instance="i",zone="ie"} 10000 6000000 161test_metric1{foo="boo",instance="i",zone="ie"} 1 6000000 162# TYPE test_metric2 untyped 163test_metric2{foo="boo",instance="i",zone="ie"} 1 6000000 164# TYPE test_metric_old untyped 165test_metric_old{foo="baz",instance="",zone="ie"} 981 5880000 166# TYPE test_metric_without_labels untyped 167test_metric_without_labels{foo="baz",instance="",zone="ie"} 1001 6000000 168`, 169 }, 170 "instance is an external label": { 171 // This makes no sense as a configuration, but we should 172 // know what it does anyway. 173 params: "match[]={__name__=~'.%2b'}", // '%2b' is an URL-encoded '+'. 174 externalLabels: labels.Labels{{Name: "instance", Value: "baz"}}, 175 code: 200, 176 body: `# TYPE test_metric1 untyped 177test_metric1{foo="bar",instance="i"} 10000 6000000 178test_metric1{foo="boo",instance="i"} 1 6000000 179# TYPE test_metric2 untyped 180test_metric2{foo="boo",instance="i"} 1 6000000 181# TYPE test_metric_old untyped 182test_metric_old{instance="baz"} 981 5880000 183# TYPE test_metric_without_labels untyped 184test_metric_without_labels{instance="baz"} 1001 6000000 185`, 186 }, 187} 188 189func TestFederation(t *testing.T) { 190 suite, err := promql.NewTest(t, ` 191 load 1m 192 test_metric1{foo="bar",instance="i"} 0+100x100 193 test_metric1{foo="boo",instance="i"} 1+0x100 194 test_metric2{foo="boo",instance="i"} 1+0x100 195 test_metric_without_labels 1+10x100 196 test_metric_stale 1+10x99 stale 197 test_metric_old 1+10x98 198 `) 199 if err != nil { 200 t.Fatal(err) 201 } 202 defer suite.Close() 203 204 if err := suite.Run(); err != nil { 205 t.Fatal(err) 206 } 207 208 h := &Handler{ 209 localStorage: &dbAdapter{suite.TSDB()}, 210 lookbackDelta: 5 * time.Minute, 211 now: func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch. 212 config: &config.Config{ 213 GlobalConfig: config.GlobalConfig{}, 214 }, 215 } 216 217 for name, scenario := range scenarios { 218 t.Run(name, func(t *testing.T) { 219 h.config.GlobalConfig.ExternalLabels = scenario.externalLabels 220 req := httptest.NewRequest("GET", "http://example.org/federate?"+scenario.params, nil) 221 res := httptest.NewRecorder() 222 223 h.federation(res, req) 224 require.Equal(t, scenario.code, res.Code) 225 require.Equal(t, scenario.body, normalizeBody(res.Body)) 226 }) 227 } 228} 229 230type notReadyReadStorage struct { 231 LocalStorage 232} 233 234func (notReadyReadStorage) Querier(context.Context, int64, int64) (storage.Querier, error) { 235 return nil, errors.Wrap(tsdb.ErrNotReady, "wrap") 236} 237 238func (notReadyReadStorage) StartTime() (int64, error) { 239 return 0, errors.Wrap(tsdb.ErrNotReady, "wrap") 240} 241 242func (notReadyReadStorage) Stats(string) (*tsdb.Stats, error) { 243 return nil, errors.Wrap(tsdb.ErrNotReady, "wrap") 244} 245 246// Regression test for https://github.com/prometheus/prometheus/issues/7181. 247func TestFederation_NotReady(t *testing.T) { 248 h := &Handler{ 249 localStorage: notReadyReadStorage{}, 250 lookbackDelta: 5 * time.Minute, 251 now: func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch. 252 config: &config.Config{ 253 GlobalConfig: config.GlobalConfig{}, 254 }, 255 } 256 257 for name, scenario := range scenarios { 258 t.Run(name, func(t *testing.T) { 259 h.config.GlobalConfig.ExternalLabels = scenario.externalLabels 260 req := httptest.NewRequest("GET", "http://example.org/federate?"+scenario.params, nil) 261 res := httptest.NewRecorder() 262 263 h.federation(res, req) 264 if scenario.code == http.StatusBadRequest { 265 // Request are expected to be checked before DB readiness. 266 require.Equal(t, http.StatusBadRequest, res.Code) 267 return 268 } 269 require.Equal(t, http.StatusServiceUnavailable, res.Code) 270 }) 271 } 272} 273 274// normalizeBody sorts the lines within a metric to make it easy to verify the body. 275// (Federation is not taking care of sorting within a metric family.) 276func normalizeBody(body *bytes.Buffer) string { 277 var ( 278 lines []string 279 lastHash int 280 ) 281 for line, err := body.ReadString('\n'); err == nil; line, err = body.ReadString('\n') { 282 if line[0] == '#' && len(lines) > 0 { 283 sort.Strings(lines[lastHash+1:]) 284 lastHash = len(lines) 285 } 286 lines = append(lines, line) 287 } 288 if len(lines) > 0 { 289 sort.Strings(lines[lastHash+1:]) 290 } 291 return strings.Join(lines, "") 292} 293