1// Copyright 2016 Prometheus Team 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package inhibit 15 16import ( 17 "testing" 18 "time" 19 20 "github.com/go-kit/log" 21 "github.com/prometheus/client_golang/prometheus" 22 "github.com/prometheus/common/model" 23 24 "github.com/prometheus/alertmanager/config" 25 "github.com/prometheus/alertmanager/pkg/labels" 26 "github.com/prometheus/alertmanager/provider" 27 "github.com/prometheus/alertmanager/store" 28 "github.com/prometheus/alertmanager/types" 29) 30 31var nopLogger = log.NewNopLogger() 32 33func TestInhibitRuleHasEqual(t *testing.T) { 34 t.Parallel() 35 36 now := time.Now() 37 cases := []struct { 38 initial map[model.Fingerprint]*types.Alert 39 equal model.LabelNames 40 input model.LabelSet 41 result bool 42 }{ 43 { 44 // No source alerts at all. 45 initial: map[model.Fingerprint]*types.Alert{}, 46 input: model.LabelSet{"a": "b"}, 47 result: false, 48 }, 49 { 50 // No equal labels, any source alerts satisfies the requirement. 51 initial: map[model.Fingerprint]*types.Alert{1: &types.Alert{}}, 52 input: model.LabelSet{"a": "b"}, 53 result: true, 54 }, 55 { 56 // Matching but already resolved. 57 initial: map[model.Fingerprint]*types.Alert{ 58 1: &types.Alert{ 59 Alert: model.Alert{ 60 Labels: model.LabelSet{"a": "b", "b": "f"}, 61 StartsAt: now.Add(-time.Minute), 62 EndsAt: now.Add(-time.Second), 63 }, 64 }, 65 2: &types.Alert{ 66 Alert: model.Alert{ 67 Labels: model.LabelSet{"a": "b", "b": "c"}, 68 StartsAt: now.Add(-time.Minute), 69 EndsAt: now.Add(-time.Second), 70 }, 71 }, 72 }, 73 equal: model.LabelNames{"a", "b"}, 74 input: model.LabelSet{"a": "b", "b": "c"}, 75 result: false, 76 }, 77 { 78 // Matching and unresolved. 79 initial: map[model.Fingerprint]*types.Alert{ 80 1: &types.Alert{ 81 Alert: model.Alert{ 82 Labels: model.LabelSet{"a": "b", "c": "d"}, 83 StartsAt: now.Add(-time.Minute), 84 EndsAt: now.Add(-time.Second), 85 }, 86 }, 87 2: &types.Alert{ 88 Alert: model.Alert{ 89 Labels: model.LabelSet{"a": "b", "c": "f"}, 90 StartsAt: now.Add(-time.Minute), 91 EndsAt: now.Add(time.Hour), 92 }, 93 }, 94 }, 95 equal: model.LabelNames{"a"}, 96 input: model.LabelSet{"a": "b"}, 97 result: true, 98 }, 99 { 100 // Equal label does not match. 101 initial: map[model.Fingerprint]*types.Alert{ 102 1: &types.Alert{ 103 Alert: model.Alert{ 104 Labels: model.LabelSet{"a": "c", "c": "d"}, 105 StartsAt: now.Add(-time.Minute), 106 EndsAt: now.Add(-time.Second), 107 }, 108 }, 109 2: &types.Alert{ 110 Alert: model.Alert{ 111 Labels: model.LabelSet{"a": "c", "c": "f"}, 112 StartsAt: now.Add(-time.Minute), 113 EndsAt: now.Add(-time.Second), 114 }, 115 }, 116 }, 117 equal: model.LabelNames{"a"}, 118 input: model.LabelSet{"a": "b"}, 119 result: false, 120 }, 121 } 122 123 for _, c := range cases { 124 r := &InhibitRule{ 125 Equal: map[model.LabelName]struct{}{}, 126 scache: store.NewAlerts(), 127 } 128 for _, ln := range c.equal { 129 r.Equal[ln] = struct{}{} 130 } 131 for _, v := range c.initial { 132 r.scache.Set(v) 133 } 134 135 if _, have := r.hasEqual(c.input, false); have != c.result { 136 t.Errorf("Unexpected result %t, expected %t", have, c.result) 137 } 138 } 139} 140 141func TestInhibitRuleMatches(t *testing.T) { 142 t.Parallel() 143 144 rule1 := config.InhibitRule{ 145 SourceMatch: map[string]string{"s1": "1"}, 146 TargetMatch: map[string]string{"t1": "1"}, 147 Equal: model.LabelNames{"e"}, 148 } 149 rule2 := config.InhibitRule{ 150 SourceMatch: map[string]string{"s2": "1"}, 151 TargetMatch: map[string]string{"t2": "1"}, 152 Equal: model.LabelNames{"e"}, 153 } 154 155 m := types.NewMarker(prometheus.NewRegistry()) 156 ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger) 157 now := time.Now() 158 // Active alert that matches the source filter of rule1. 159 sourceAlert1 := &types.Alert{ 160 Alert: model.Alert{ 161 Labels: model.LabelSet{"s1": "1", "t1": "2", "e": "1"}, 162 StartsAt: now.Add(-time.Minute), 163 EndsAt: now.Add(time.Hour), 164 }, 165 } 166 // Active alert that matches the source filter _and_ the target filter of rule2. 167 sourceAlert2 := &types.Alert{ 168 Alert: model.Alert{ 169 Labels: model.LabelSet{"s2": "1", "t2": "1", "e": "1"}, 170 StartsAt: now.Add(-time.Minute), 171 EndsAt: now.Add(time.Hour), 172 }, 173 } 174 175 ih.rules[0].scache = store.NewAlerts() 176 ih.rules[0].scache.Set(sourceAlert1) 177 ih.rules[1].scache = store.NewAlerts() 178 ih.rules[1].scache.Set(sourceAlert2) 179 180 cases := []struct { 181 target model.LabelSet 182 expected bool 183 }{ 184 { 185 // Matches target filter of rule1, inhibited. 186 target: model.LabelSet{"t1": "1", "e": "1"}, 187 expected: true, 188 }, 189 { 190 // Matches target filter of rule2, inhibited. 191 target: model.LabelSet{"t2": "1", "e": "1"}, 192 expected: true, 193 }, 194 { 195 // Matches target filter of rule1 (plus noise), inhibited. 196 target: model.LabelSet{"t1": "1", "t3": "1", "e": "1"}, 197 expected: true, 198 }, 199 { 200 // Matches target filter of rule1 plus rule2, inhibited. 201 target: model.LabelSet{"t1": "1", "t2": "1", "e": "1"}, 202 expected: true, 203 }, 204 { 205 // Doesn't match target filter, not inhibited. 206 target: model.LabelSet{"t1": "0", "e": "1"}, 207 expected: false, 208 }, 209 { 210 // Matches both source and target filters of rule1, 211 // inhibited because sourceAlert1 matches only the 212 // source filter of rule1. 213 target: model.LabelSet{"s1": "1", "t1": "1", "e": "1"}, 214 expected: true, 215 }, 216 { 217 // Matches both source and target filters of rule2, 218 // not inhibited because sourceAlert2 matches also both the 219 // source and target filter of rule2. 220 target: model.LabelSet{"s2": "1", "t2": "1", "e": "1"}, 221 expected: false, 222 }, 223 { 224 // Matches target filter, equal label doesn't match, not inhibited 225 target: model.LabelSet{"t1": "1", "e": "0"}, 226 expected: false, 227 }, 228 } 229 230 for _, c := range cases { 231 if actual := ih.Mutes(c.target); actual != c.expected { 232 t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual) 233 } 234 } 235} 236func TestInhibitRuleMatchers(t *testing.T) { 237 t.Parallel() 238 239 rule1 := config.InhibitRule{ 240 SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s1", Value: "1"}}, 241 TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchNotEqual, Name: "t1", Value: "1"}}, 242 Equal: model.LabelNames{"e"}, 243 } 244 rule2 := config.InhibitRule{ 245 SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s2", Value: "1"}}, 246 TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "t2", Value: "1"}}, 247 Equal: model.LabelNames{"e"}, 248 } 249 250 m := types.NewMarker(prometheus.NewRegistry()) 251 ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger) 252 now := time.Now() 253 // Active alert that matches the source filter of rule1. 254 sourceAlert1 := &types.Alert{ 255 Alert: model.Alert{ 256 Labels: model.LabelSet{"s1": "1", "t1": "2", "e": "1"}, 257 StartsAt: now.Add(-time.Minute), 258 EndsAt: now.Add(time.Hour), 259 }, 260 } 261 // Active alert that matches the source filter _and_ the target filter of rule2. 262 sourceAlert2 := &types.Alert{ 263 Alert: model.Alert{ 264 Labels: model.LabelSet{"s2": "1", "t2": "1", "e": "1"}, 265 StartsAt: now.Add(-time.Minute), 266 EndsAt: now.Add(time.Hour), 267 }, 268 } 269 270 ih.rules[0].scache = store.NewAlerts() 271 ih.rules[0].scache.Set(sourceAlert1) 272 ih.rules[1].scache = store.NewAlerts() 273 ih.rules[1].scache.Set(sourceAlert2) 274 275 cases := []struct { 276 target model.LabelSet 277 expected bool 278 }{ 279 { 280 // Matches target filter of rule1, inhibited. 281 target: model.LabelSet{"t1": "1", "e": "1"}, 282 expected: false, 283 }, 284 { 285 // Matches target filter of rule2, inhibited. 286 target: model.LabelSet{"t2": "1", "e": "1"}, 287 expected: true, 288 }, 289 { 290 // Matches target filter of rule1 (plus noise), inhibited. 291 target: model.LabelSet{"t1": "1", "t3": "1", "e": "1"}, 292 expected: false, 293 }, 294 { 295 // Matches target filter of rule1 plus rule2, inhibited. 296 target: model.LabelSet{"t1": "1", "t2": "1", "e": "1"}, 297 expected: true, 298 }, 299 { 300 // Doesn't match target filter, not inhibited. 301 target: model.LabelSet{"t1": "0", "e": "1"}, 302 expected: true, 303 }, 304 { 305 // Matches both source and target filters of rule1, 306 // inhibited because sourceAlert1 matches only the 307 // source filter of rule1. 308 target: model.LabelSet{"s1": "1", "t1": "1", "e": "1"}, 309 expected: false, 310 }, 311 { 312 // Matches both source and target filters of rule2, 313 // not inhibited because sourceAlert2 matches also both the 314 // source and target filter of rule2. 315 target: model.LabelSet{"s2": "1", "t2": "1", "e": "1"}, 316 expected: true, 317 }, 318 { 319 // Matches target filter, equal label doesn't match, not inhibited 320 target: model.LabelSet{"t1": "1", "e": "0"}, 321 expected: false, 322 }, 323 } 324 325 for _, c := range cases { 326 if actual := ih.Mutes(c.target); actual != c.expected { 327 t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual) 328 } 329 } 330} 331 332type fakeAlerts struct { 333 alerts []*types.Alert 334 finished chan struct{} 335} 336 337func newFakeAlerts(alerts []*types.Alert) *fakeAlerts { 338 return &fakeAlerts{ 339 alerts: alerts, 340 finished: make(chan struct{}), 341 } 342} 343 344func (f *fakeAlerts) GetPending() provider.AlertIterator { return nil } 345func (f *fakeAlerts) Get(model.Fingerprint) (*types.Alert, error) { return nil, nil } 346func (f *fakeAlerts) Put(...*types.Alert) error { return nil } 347func (f *fakeAlerts) Subscribe() provider.AlertIterator { 348 ch := make(chan *types.Alert) 349 done := make(chan struct{}) 350 go func() { 351 for _, a := range f.alerts { 352 ch <- a 353 } 354 // Send another (meaningless) alert to make sure that the inhibitor has 355 // processed everything. 356 ch <- &types.Alert{ 357 Alert: model.Alert{ 358 Labels: model.LabelSet{}, 359 StartsAt: time.Now(), 360 }, 361 } 362 close(f.finished) 363 <-done 364 }() 365 return provider.NewAlertIterator(ch, done, nil) 366} 367 368func TestInhibit(t *testing.T) { 369 t.Parallel() 370 371 now := time.Now() 372 inhibitRule := func() *config.InhibitRule { 373 return &config.InhibitRule{ 374 SourceMatch: map[string]string{"s": "1"}, 375 TargetMatch: map[string]string{"t": "1"}, 376 Equal: model.LabelNames{"e"}, 377 } 378 } 379 // alertOne is muted by alertTwo when it is active. 380 alertOne := func() *types.Alert { 381 return &types.Alert{ 382 Alert: model.Alert{ 383 Labels: model.LabelSet{"t": "1", "e": "f"}, 384 StartsAt: now.Add(-time.Minute), 385 EndsAt: now.Add(time.Hour), 386 }, 387 } 388 } 389 alertTwo := func(resolved bool) *types.Alert { 390 var end time.Time 391 if resolved { 392 end = now.Add(-time.Second) 393 } else { 394 end = now.Add(time.Hour) 395 } 396 return &types.Alert{ 397 Alert: model.Alert{ 398 Labels: model.LabelSet{"s": "1", "e": "f"}, 399 StartsAt: now.Add(-time.Minute), 400 EndsAt: end, 401 }, 402 } 403 } 404 405 type exp struct { 406 lbls model.LabelSet 407 muted bool 408 } 409 for i, tc := range []struct { 410 alerts []*types.Alert 411 expected []exp 412 }{ 413 { 414 // alertOne shouldn't be muted since alertTwo hasn't fired. 415 alerts: []*types.Alert{alertOne()}, 416 expected: []exp{ 417 { 418 lbls: model.LabelSet{"t": "1", "e": "f"}, 419 muted: false, 420 }, 421 }, 422 }, 423 { 424 // alertOne should be muted by alertTwo which is active. 425 alerts: []*types.Alert{alertOne(), alertTwo(false)}, 426 expected: []exp{ 427 { 428 lbls: model.LabelSet{"t": "1", "e": "f"}, 429 muted: true, 430 }, 431 { 432 lbls: model.LabelSet{"s": "1", "e": "f"}, 433 muted: false, 434 }, 435 }, 436 }, 437 { 438 // alertOne shouldn't be muted since alertTwo is resolved. 439 alerts: []*types.Alert{alertOne(), alertTwo(false), alertTwo(true)}, 440 expected: []exp{ 441 { 442 lbls: model.LabelSet{"t": "1", "e": "f"}, 443 muted: false, 444 }, 445 { 446 lbls: model.LabelSet{"s": "1", "e": "f"}, 447 muted: false, 448 }, 449 }, 450 }, 451 } { 452 ap := newFakeAlerts(tc.alerts) 453 mk := types.NewMarker(prometheus.NewRegistry()) 454 inhibitor := NewInhibitor(ap, []*config.InhibitRule{inhibitRule()}, mk, nopLogger) 455 456 go func() { 457 for ap.finished != nil { 458 select { 459 case <-ap.finished: 460 ap.finished = nil 461 default: 462 } 463 } 464 inhibitor.Stop() 465 }() 466 inhibitor.Run() 467 468 for _, expected := range tc.expected { 469 if inhibitor.Mutes(expected.lbls) != expected.muted { 470 mute := "unmuted" 471 if expected.muted { 472 mute = "muted" 473 } 474 t.Errorf("tc: %d, expected alert with labels %q to be %s", i, expected.lbls, mute) 475 } 476 } 477 } 478} 479