1// Copyright 2014 The Prometheus Authors
2// Licensed under the Apache License, Version 2.0 (the "License");
3// you may not use this file except in compliance with the License.
4// You may obtain a copy of the License at
5//
6// http://www.apache.org/licenses/LICENSE-2.0
7//
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13
14package prometheus
15
16import (
17	"math"
18	"math/rand"
19	"sort"
20	"sync"
21	"testing"
22	"testing/quick"
23	"time"
24
25	dto "github.com/coreos/etcd/Godeps/_workspace/src/github.com/prometheus/client_model/go"
26)
27
28func benchmarkSummaryObserve(w int, b *testing.B) {
29	b.StopTimer()
30
31	wg := new(sync.WaitGroup)
32	wg.Add(w)
33
34	g := new(sync.WaitGroup)
35	g.Add(1)
36
37	s := NewSummary(SummaryOpts{})
38
39	for i := 0; i < w; i++ {
40		go func() {
41			g.Wait()
42
43			for i := 0; i < b.N; i++ {
44				s.Observe(float64(i))
45			}
46
47			wg.Done()
48		}()
49	}
50
51	b.StartTimer()
52	g.Done()
53	wg.Wait()
54}
55
56func BenchmarkSummaryObserve1(b *testing.B) {
57	benchmarkSummaryObserve(1, b)
58}
59
60func BenchmarkSummaryObserve2(b *testing.B) {
61	benchmarkSummaryObserve(2, b)
62}
63
64func BenchmarkSummaryObserve4(b *testing.B) {
65	benchmarkSummaryObserve(4, b)
66}
67
68func BenchmarkSummaryObserve8(b *testing.B) {
69	benchmarkSummaryObserve(8, b)
70}
71
72func benchmarkSummaryWrite(w int, b *testing.B) {
73	b.StopTimer()
74
75	wg := new(sync.WaitGroup)
76	wg.Add(w)
77
78	g := new(sync.WaitGroup)
79	g.Add(1)
80
81	s := NewSummary(SummaryOpts{})
82
83	for i := 0; i < 1000000; i++ {
84		s.Observe(float64(i))
85	}
86
87	for j := 0; j < w; j++ {
88		outs := make([]dto.Metric, b.N)
89
90		go func(o []dto.Metric) {
91			g.Wait()
92
93			for i := 0; i < b.N; i++ {
94				s.Write(&o[i])
95			}
96
97			wg.Done()
98		}(outs)
99	}
100
101	b.StartTimer()
102	g.Done()
103	wg.Wait()
104}
105
106func BenchmarkSummaryWrite1(b *testing.B) {
107	benchmarkSummaryWrite(1, b)
108}
109
110func BenchmarkSummaryWrite2(b *testing.B) {
111	benchmarkSummaryWrite(2, b)
112}
113
114func BenchmarkSummaryWrite4(b *testing.B) {
115	benchmarkSummaryWrite(4, b)
116}
117
118func BenchmarkSummaryWrite8(b *testing.B) {
119	benchmarkSummaryWrite(8, b)
120}
121
122func TestSummaryConcurrency(t *testing.T) {
123	if testing.Short() {
124		t.Skip("Skipping test in short mode.")
125	}
126
127	rand.Seed(42)
128
129	it := func(n uint32) bool {
130		mutations := int(n%1e4 + 1e4)
131		concLevel := int(n%5 + 1)
132		total := mutations * concLevel
133
134		var start, end sync.WaitGroup
135		start.Add(1)
136		end.Add(concLevel)
137
138		sum := NewSummary(SummaryOpts{
139			Name: "test_summary",
140			Help: "helpless",
141		})
142
143		allVars := make([]float64, total)
144		var sampleSum float64
145		for i := 0; i < concLevel; i++ {
146			vals := make([]float64, mutations)
147			for j := 0; j < mutations; j++ {
148				v := rand.NormFloat64()
149				vals[j] = v
150				allVars[i*mutations+j] = v
151				sampleSum += v
152			}
153
154			go func(vals []float64) {
155				start.Wait()
156				for _, v := range vals {
157					sum.Observe(v)
158				}
159				end.Done()
160			}(vals)
161		}
162		sort.Float64s(allVars)
163		start.Done()
164		end.Wait()
165
166		m := &dto.Metric{}
167		sum.Write(m)
168		if got, want := int(*m.Summary.SampleCount), total; got != want {
169			t.Errorf("got sample count %d, want %d", got, want)
170		}
171		if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 {
172			t.Errorf("got sample sum %f, want %f", got, want)
173		}
174
175		objectives := make([]float64, 0, len(DefObjectives))
176		for qu := range DefObjectives {
177			objectives = append(objectives, qu)
178		}
179		sort.Float64s(objectives)
180
181		for i, wantQ := range objectives {
182			ε := DefObjectives[wantQ]
183			gotQ := *m.Summary.Quantile[i].Quantile
184			gotV := *m.Summary.Quantile[i].Value
185			min, max := getBounds(allVars, wantQ, ε)
186			if gotQ != wantQ {
187				t.Errorf("got quantile %f, want %f", gotQ, wantQ)
188			}
189			if gotV < min || gotV > max {
190				t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max)
191			}
192		}
193		return true
194	}
195
196	if err := quick.Check(it, nil); err != nil {
197		t.Error(err)
198	}
199}
200
201func TestSummaryVecConcurrency(t *testing.T) {
202	if testing.Short() {
203		t.Skip("Skipping test in short mode.")
204	}
205
206	rand.Seed(42)
207
208	objectives := make([]float64, 0, len(DefObjectives))
209	for qu := range DefObjectives {
210
211		objectives = append(objectives, qu)
212	}
213	sort.Float64s(objectives)
214
215	it := func(n uint32) bool {
216		mutations := int(n%1e4 + 1e4)
217		concLevel := int(n%7 + 1)
218		vecLength := int(n%3 + 1)
219
220		var start, end sync.WaitGroup
221		start.Add(1)
222		end.Add(concLevel)
223
224		sum := NewSummaryVec(
225			SummaryOpts{
226				Name: "test_summary",
227				Help: "helpless",
228			},
229			[]string{"label"},
230		)
231
232		allVars := make([][]float64, vecLength)
233		sampleSums := make([]float64, vecLength)
234		for i := 0; i < concLevel; i++ {
235			vals := make([]float64, mutations)
236			picks := make([]int, mutations)
237			for j := 0; j < mutations; j++ {
238				v := rand.NormFloat64()
239				vals[j] = v
240				pick := rand.Intn(vecLength)
241				picks[j] = pick
242				allVars[pick] = append(allVars[pick], v)
243				sampleSums[pick] += v
244			}
245
246			go func(vals []float64) {
247				start.Wait()
248				for i, v := range vals {
249					sum.WithLabelValues(string('A' + picks[i])).Observe(v)
250				}
251				end.Done()
252			}(vals)
253		}
254		for _, vars := range allVars {
255			sort.Float64s(vars)
256		}
257		start.Done()
258		end.Wait()
259
260		for i := 0; i < vecLength; i++ {
261			m := &dto.Metric{}
262			s := sum.WithLabelValues(string('A' + i))
263			s.Write(m)
264			if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want {
265				t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want)
266			}
267			if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 {
268				t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want)
269			}
270			for j, wantQ := range objectives {
271				ε := DefObjectives[wantQ]
272				gotQ := *m.Summary.Quantile[j].Quantile
273				gotV := *m.Summary.Quantile[j].Value
274				min, max := getBounds(allVars[i], wantQ, ε)
275				if gotQ != wantQ {
276					t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ)
277				}
278				if gotV < min || gotV > max {
279					t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max)
280				}
281			}
282		}
283		return true
284	}
285
286	if err := quick.Check(it, nil); err != nil {
287		t.Error(err)
288	}
289}
290
291func TestSummaryDecay(t *testing.T) {
292	if testing.Short() {
293		t.Skip("Skipping test in short mode.")
294		// More because it depends on timing than because it is particularly long...
295	}
296
297	sum := NewSummary(SummaryOpts{
298		Name:       "test_summary",
299		Help:       "helpless",
300		MaxAge:     100 * time.Millisecond,
301		Objectives: map[float64]float64{0.1: 0.001},
302		AgeBuckets: 10,
303	})
304
305	m := &dto.Metric{}
306	i := 0
307	tick := time.NewTicker(time.Millisecond)
308	for _ = range tick.C {
309		i++
310		sum.Observe(float64(i))
311		if i%10 == 0 {
312			sum.Write(m)
313			if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 {
314				t.Errorf("%d. got %f, want %f", i, got, want)
315			}
316			m.Reset()
317		}
318		if i >= 1000 {
319			break
320		}
321	}
322	tick.Stop()
323	// Wait for MaxAge without observations and make sure quantiles are NaN.
324	time.Sleep(100 * time.Millisecond)
325	sum.Write(m)
326	if got := *m.Summary.Quantile[0].Value; !math.IsNaN(got) {
327		t.Errorf("got %f, want NaN after expiration", got)
328	}
329}
330
331func getBounds(vars []float64, q, ε float64) (min, max float64) {
332	// TODO: This currently tolerates an error of up to 2*ε. The error must
333	// be at most ε, but for some reason, it's sometimes slightly
334	// higher. That's a bug.
335	n := float64(len(vars))
336	lower := int((q - 2*ε) * n)
337	upper := int(math.Ceil((q + 2*ε) * n))
338	min = vars[0]
339	if lower > 1 {
340		min = vars[lower-1]
341	}
342	max = vars[len(vars)-1]
343	if upper < len(vars) {
344		max = vars[upper-1]
345	}
346	return
347}
348