1// Copyright 2016 Prometheus Team
2// Licensed under the Apache License, Version 2.0 (the "License");
3// you may not use this file except in compliance with the License.
4// You may obtain a copy of the License at
5//
6// http://www.apache.org/licenses/LICENSE-2.0
7//
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13
14package inhibit
15
16import (
17	"testing"
18	"time"
19
20	"github.com/go-kit/log"
21	"github.com/prometheus/client_golang/prometheus"
22	"github.com/prometheus/common/model"
23
24	"github.com/prometheus/alertmanager/config"
25	"github.com/prometheus/alertmanager/pkg/labels"
26	"github.com/prometheus/alertmanager/provider"
27	"github.com/prometheus/alertmanager/store"
28	"github.com/prometheus/alertmanager/types"
29)
30
31var nopLogger = log.NewNopLogger()
32
33func TestInhibitRuleHasEqual(t *testing.T) {
34	t.Parallel()
35
36	now := time.Now()
37	cases := []struct {
38		initial map[model.Fingerprint]*types.Alert
39		equal   model.LabelNames
40		input   model.LabelSet
41		result  bool
42	}{
43		{
44			// No source alerts at all.
45			initial: map[model.Fingerprint]*types.Alert{},
46			input:   model.LabelSet{"a": "b"},
47			result:  false,
48		},
49		{
50			// No equal labels, any source alerts satisfies the requirement.
51			initial: map[model.Fingerprint]*types.Alert{1: &types.Alert{}},
52			input:   model.LabelSet{"a": "b"},
53			result:  true,
54		},
55		{
56			// Matching but already resolved.
57			initial: map[model.Fingerprint]*types.Alert{
58				1: &types.Alert{
59					Alert: model.Alert{
60						Labels:   model.LabelSet{"a": "b", "b": "f"},
61						StartsAt: now.Add(-time.Minute),
62						EndsAt:   now.Add(-time.Second),
63					},
64				},
65				2: &types.Alert{
66					Alert: model.Alert{
67						Labels:   model.LabelSet{"a": "b", "b": "c"},
68						StartsAt: now.Add(-time.Minute),
69						EndsAt:   now.Add(-time.Second),
70					},
71				},
72			},
73			equal:  model.LabelNames{"a", "b"},
74			input:  model.LabelSet{"a": "b", "b": "c"},
75			result: false,
76		},
77		{
78			// Matching and unresolved.
79			initial: map[model.Fingerprint]*types.Alert{
80				1: &types.Alert{
81					Alert: model.Alert{
82						Labels:   model.LabelSet{"a": "b", "c": "d"},
83						StartsAt: now.Add(-time.Minute),
84						EndsAt:   now.Add(-time.Second),
85					},
86				},
87				2: &types.Alert{
88					Alert: model.Alert{
89						Labels:   model.LabelSet{"a": "b", "c": "f"},
90						StartsAt: now.Add(-time.Minute),
91						EndsAt:   now.Add(time.Hour),
92					},
93				},
94			},
95			equal:  model.LabelNames{"a"},
96			input:  model.LabelSet{"a": "b"},
97			result: true,
98		},
99		{
100			// Equal label does not match.
101			initial: map[model.Fingerprint]*types.Alert{
102				1: &types.Alert{
103					Alert: model.Alert{
104						Labels:   model.LabelSet{"a": "c", "c": "d"},
105						StartsAt: now.Add(-time.Minute),
106						EndsAt:   now.Add(-time.Second),
107					},
108				},
109				2: &types.Alert{
110					Alert: model.Alert{
111						Labels:   model.LabelSet{"a": "c", "c": "f"},
112						StartsAt: now.Add(-time.Minute),
113						EndsAt:   now.Add(-time.Second),
114					},
115				},
116			},
117			equal:  model.LabelNames{"a"},
118			input:  model.LabelSet{"a": "b"},
119			result: false,
120		},
121	}
122
123	for _, c := range cases {
124		r := &InhibitRule{
125			Equal:  map[model.LabelName]struct{}{},
126			scache: store.NewAlerts(),
127		}
128		for _, ln := range c.equal {
129			r.Equal[ln] = struct{}{}
130		}
131		for _, v := range c.initial {
132			r.scache.Set(v)
133		}
134
135		if _, have := r.hasEqual(c.input, false); have != c.result {
136			t.Errorf("Unexpected result %t, expected %t", have, c.result)
137		}
138	}
139}
140
141func TestInhibitRuleMatches(t *testing.T) {
142	t.Parallel()
143
144	rule1 := config.InhibitRule{
145		SourceMatch: map[string]string{"s1": "1"},
146		TargetMatch: map[string]string{"t1": "1"},
147		Equal:       model.LabelNames{"e"},
148	}
149	rule2 := config.InhibitRule{
150		SourceMatch: map[string]string{"s2": "1"},
151		TargetMatch: map[string]string{"t2": "1"},
152		Equal:       model.LabelNames{"e"},
153	}
154
155	m := types.NewMarker(prometheus.NewRegistry())
156	ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
157	now := time.Now()
158	// Active alert that matches the source filter of rule1.
159	sourceAlert1 := &types.Alert{
160		Alert: model.Alert{
161			Labels:   model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
162			StartsAt: now.Add(-time.Minute),
163			EndsAt:   now.Add(time.Hour),
164		},
165	}
166	// Active alert that matches the source filter _and_ the target filter of rule2.
167	sourceAlert2 := &types.Alert{
168		Alert: model.Alert{
169			Labels:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
170			StartsAt: now.Add(-time.Minute),
171			EndsAt:   now.Add(time.Hour),
172		},
173	}
174
175	ih.rules[0].scache = store.NewAlerts()
176	ih.rules[0].scache.Set(sourceAlert1)
177	ih.rules[1].scache = store.NewAlerts()
178	ih.rules[1].scache.Set(sourceAlert2)
179
180	cases := []struct {
181		target   model.LabelSet
182		expected bool
183	}{
184		{
185			// Matches target filter of rule1, inhibited.
186			target:   model.LabelSet{"t1": "1", "e": "1"},
187			expected: true,
188		},
189		{
190			// Matches target filter of rule2, inhibited.
191			target:   model.LabelSet{"t2": "1", "e": "1"},
192			expected: true,
193		},
194		{
195			// Matches target filter of rule1 (plus noise), inhibited.
196			target:   model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
197			expected: true,
198		},
199		{
200			// Matches target filter of rule1 plus rule2, inhibited.
201			target:   model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
202			expected: true,
203		},
204		{
205			// Doesn't match target filter, not inhibited.
206			target:   model.LabelSet{"t1": "0", "e": "1"},
207			expected: false,
208		},
209		{
210			// Matches both source and target filters of rule1,
211			// inhibited because sourceAlert1 matches only the
212			// source filter of rule1.
213			target:   model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
214			expected: true,
215		},
216		{
217			// Matches both source and target filters of rule2,
218			// not inhibited because sourceAlert2 matches also both the
219			// source and target filter of rule2.
220			target:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
221			expected: false,
222		},
223		{
224			// Matches target filter, equal label doesn't match, not inhibited
225			target:   model.LabelSet{"t1": "1", "e": "0"},
226			expected: false,
227		},
228	}
229
230	for _, c := range cases {
231		if actual := ih.Mutes(c.target); actual != c.expected {
232			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
233		}
234	}
235}
236func TestInhibitRuleMatchers(t *testing.T) {
237	t.Parallel()
238
239	rule1 := config.InhibitRule{
240		SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s1", Value: "1"}},
241		TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchNotEqual, Name: "t1", Value: "1"}},
242		Equal:          model.LabelNames{"e"},
243	}
244	rule2 := config.InhibitRule{
245		SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s2", Value: "1"}},
246		TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "t2", Value: "1"}},
247		Equal:          model.LabelNames{"e"},
248	}
249
250	m := types.NewMarker(prometheus.NewRegistry())
251	ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
252	now := time.Now()
253	// Active alert that matches the source filter of rule1.
254	sourceAlert1 := &types.Alert{
255		Alert: model.Alert{
256			Labels:   model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
257			StartsAt: now.Add(-time.Minute),
258			EndsAt:   now.Add(time.Hour),
259		},
260	}
261	// Active alert that matches the source filter _and_ the target filter of rule2.
262	sourceAlert2 := &types.Alert{
263		Alert: model.Alert{
264			Labels:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
265			StartsAt: now.Add(-time.Minute),
266			EndsAt:   now.Add(time.Hour),
267		},
268	}
269
270	ih.rules[0].scache = store.NewAlerts()
271	ih.rules[0].scache.Set(sourceAlert1)
272	ih.rules[1].scache = store.NewAlerts()
273	ih.rules[1].scache.Set(sourceAlert2)
274
275	cases := []struct {
276		target   model.LabelSet
277		expected bool
278	}{
279		{
280			// Matches target filter of rule1, inhibited.
281			target:   model.LabelSet{"t1": "1", "e": "1"},
282			expected: false,
283		},
284		{
285			// Matches target filter of rule2, inhibited.
286			target:   model.LabelSet{"t2": "1", "e": "1"},
287			expected: true,
288		},
289		{
290			// Matches target filter of rule1 (plus noise), inhibited.
291			target:   model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
292			expected: false,
293		},
294		{
295			// Matches target filter of rule1 plus rule2, inhibited.
296			target:   model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
297			expected: true,
298		},
299		{
300			// Doesn't match target filter, not inhibited.
301			target:   model.LabelSet{"t1": "0", "e": "1"},
302			expected: true,
303		},
304		{
305			// Matches both source and target filters of rule1,
306			// inhibited because sourceAlert1 matches only the
307			// source filter of rule1.
308			target:   model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
309			expected: false,
310		},
311		{
312			// Matches both source and target filters of rule2,
313			// not inhibited because sourceAlert2 matches also both the
314			// source and target filter of rule2.
315			target:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
316			expected: true,
317		},
318		{
319			// Matches target filter, equal label doesn't match, not inhibited
320			target:   model.LabelSet{"t1": "1", "e": "0"},
321			expected: false,
322		},
323	}
324
325	for _, c := range cases {
326		if actual := ih.Mutes(c.target); actual != c.expected {
327			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
328		}
329	}
330}
331
332type fakeAlerts struct {
333	alerts   []*types.Alert
334	finished chan struct{}
335}
336
337func newFakeAlerts(alerts []*types.Alert) *fakeAlerts {
338	return &fakeAlerts{
339		alerts:   alerts,
340		finished: make(chan struct{}),
341	}
342}
343
344func (f *fakeAlerts) GetPending() provider.AlertIterator          { return nil }
345func (f *fakeAlerts) Get(model.Fingerprint) (*types.Alert, error) { return nil, nil }
346func (f *fakeAlerts) Put(...*types.Alert) error                   { return nil }
347func (f *fakeAlerts) Subscribe() provider.AlertIterator {
348	ch := make(chan *types.Alert)
349	done := make(chan struct{})
350	go func() {
351		for _, a := range f.alerts {
352			ch <- a
353		}
354		// Send another (meaningless) alert to make sure that the inhibitor has
355		// processed everything.
356		ch <- &types.Alert{
357			Alert: model.Alert{
358				Labels:   model.LabelSet{},
359				StartsAt: time.Now(),
360			},
361		}
362		close(f.finished)
363		<-done
364	}()
365	return provider.NewAlertIterator(ch, done, nil)
366}
367
368func TestInhibit(t *testing.T) {
369	t.Parallel()
370
371	now := time.Now()
372	inhibitRule := func() *config.InhibitRule {
373		return &config.InhibitRule{
374			SourceMatch: map[string]string{"s": "1"},
375			TargetMatch: map[string]string{"t": "1"},
376			Equal:       model.LabelNames{"e"},
377		}
378	}
379	// alertOne is muted by alertTwo when it is active.
380	alertOne := func() *types.Alert {
381		return &types.Alert{
382			Alert: model.Alert{
383				Labels:   model.LabelSet{"t": "1", "e": "f"},
384				StartsAt: now.Add(-time.Minute),
385				EndsAt:   now.Add(time.Hour),
386			},
387		}
388	}
389	alertTwo := func(resolved bool) *types.Alert {
390		var end time.Time
391		if resolved {
392			end = now.Add(-time.Second)
393		} else {
394			end = now.Add(time.Hour)
395		}
396		return &types.Alert{
397			Alert: model.Alert{
398				Labels:   model.LabelSet{"s": "1", "e": "f"},
399				StartsAt: now.Add(-time.Minute),
400				EndsAt:   end,
401			},
402		}
403	}
404
405	type exp struct {
406		lbls  model.LabelSet
407		muted bool
408	}
409	for i, tc := range []struct {
410		alerts   []*types.Alert
411		expected []exp
412	}{
413		{
414			// alertOne shouldn't be muted since alertTwo hasn't fired.
415			alerts: []*types.Alert{alertOne()},
416			expected: []exp{
417				{
418					lbls:  model.LabelSet{"t": "1", "e": "f"},
419					muted: false,
420				},
421			},
422		},
423		{
424			// alertOne should be muted by alertTwo which is active.
425			alerts: []*types.Alert{alertOne(), alertTwo(false)},
426			expected: []exp{
427				{
428					lbls:  model.LabelSet{"t": "1", "e": "f"},
429					muted: true,
430				},
431				{
432					lbls:  model.LabelSet{"s": "1", "e": "f"},
433					muted: false,
434				},
435			},
436		},
437		{
438			// alertOne shouldn't be muted since alertTwo is resolved.
439			alerts: []*types.Alert{alertOne(), alertTwo(false), alertTwo(true)},
440			expected: []exp{
441				{
442					lbls:  model.LabelSet{"t": "1", "e": "f"},
443					muted: false,
444				},
445				{
446					lbls:  model.LabelSet{"s": "1", "e": "f"},
447					muted: false,
448				},
449			},
450		},
451	} {
452		ap := newFakeAlerts(tc.alerts)
453		mk := types.NewMarker(prometheus.NewRegistry())
454		inhibitor := NewInhibitor(ap, []*config.InhibitRule{inhibitRule()}, mk, nopLogger)
455
456		go func() {
457			for ap.finished != nil {
458				select {
459				case <-ap.finished:
460					ap.finished = nil
461				default:
462				}
463			}
464			inhibitor.Stop()
465		}()
466		inhibitor.Run()
467
468		for _, expected := range tc.expected {
469			if inhibitor.Mutes(expected.lbls) != expected.muted {
470				mute := "unmuted"
471				if expected.muted {
472					mute = "muted"
473				}
474				t.Errorf("tc: %d, expected alert with labels %q to be %s", i, expected.lbls, mute)
475			}
476		}
477	}
478}
479