1// Copyright 2014 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package prometheus 15 16import ( 17 "math" 18 "math/rand" 19 "sort" 20 "sync" 21 "testing" 22 "testing/quick" 23 "time" 24 25 dto "github.com/coreos/etcd/Godeps/_workspace/src/github.com/prometheus/client_model/go" 26) 27 28func benchmarkSummaryObserve(w int, b *testing.B) { 29 b.StopTimer() 30 31 wg := new(sync.WaitGroup) 32 wg.Add(w) 33 34 g := new(sync.WaitGroup) 35 g.Add(1) 36 37 s := NewSummary(SummaryOpts{}) 38 39 for i := 0; i < w; i++ { 40 go func() { 41 g.Wait() 42 43 for i := 0; i < b.N; i++ { 44 s.Observe(float64(i)) 45 } 46 47 wg.Done() 48 }() 49 } 50 51 b.StartTimer() 52 g.Done() 53 wg.Wait() 54} 55 56func BenchmarkSummaryObserve1(b *testing.B) { 57 benchmarkSummaryObserve(1, b) 58} 59 60func BenchmarkSummaryObserve2(b *testing.B) { 61 benchmarkSummaryObserve(2, b) 62} 63 64func BenchmarkSummaryObserve4(b *testing.B) { 65 benchmarkSummaryObserve(4, b) 66} 67 68func BenchmarkSummaryObserve8(b *testing.B) { 69 benchmarkSummaryObserve(8, b) 70} 71 72func benchmarkSummaryWrite(w int, b *testing.B) { 73 b.StopTimer() 74 75 wg := new(sync.WaitGroup) 76 wg.Add(w) 77 78 g := new(sync.WaitGroup) 79 g.Add(1) 80 81 s := NewSummary(SummaryOpts{}) 82 83 for i := 0; i < 1000000; i++ { 84 s.Observe(float64(i)) 85 } 86 87 for j := 0; j < w; j++ { 88 outs := make([]dto.Metric, b.N) 89 90 go func(o []dto.Metric) { 91 g.Wait() 92 93 for i := 0; i < b.N; i++ { 94 s.Write(&o[i]) 95 } 96 97 wg.Done() 98 }(outs) 99 } 100 101 b.StartTimer() 102 g.Done() 103 wg.Wait() 104} 105 106func BenchmarkSummaryWrite1(b *testing.B) { 107 benchmarkSummaryWrite(1, b) 108} 109 110func BenchmarkSummaryWrite2(b *testing.B) { 111 benchmarkSummaryWrite(2, b) 112} 113 114func BenchmarkSummaryWrite4(b *testing.B) { 115 benchmarkSummaryWrite(4, b) 116} 117 118func BenchmarkSummaryWrite8(b *testing.B) { 119 benchmarkSummaryWrite(8, b) 120} 121 122func TestSummaryConcurrency(t *testing.T) { 123 if testing.Short() { 124 t.Skip("Skipping test in short mode.") 125 } 126 127 rand.Seed(42) 128 129 it := func(n uint32) bool { 130 mutations := int(n%1e4 + 1e4) 131 concLevel := int(n%5 + 1) 132 total := mutations * concLevel 133 134 var start, end sync.WaitGroup 135 start.Add(1) 136 end.Add(concLevel) 137 138 sum := NewSummary(SummaryOpts{ 139 Name: "test_summary", 140 Help: "helpless", 141 }) 142 143 allVars := make([]float64, total) 144 var sampleSum float64 145 for i := 0; i < concLevel; i++ { 146 vals := make([]float64, mutations) 147 for j := 0; j < mutations; j++ { 148 v := rand.NormFloat64() 149 vals[j] = v 150 allVars[i*mutations+j] = v 151 sampleSum += v 152 } 153 154 go func(vals []float64) { 155 start.Wait() 156 for _, v := range vals { 157 sum.Observe(v) 158 } 159 end.Done() 160 }(vals) 161 } 162 sort.Float64s(allVars) 163 start.Done() 164 end.Wait() 165 166 m := &dto.Metric{} 167 sum.Write(m) 168 if got, want := int(*m.Summary.SampleCount), total; got != want { 169 t.Errorf("got sample count %d, want %d", got, want) 170 } 171 if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 { 172 t.Errorf("got sample sum %f, want %f", got, want) 173 } 174 175 objectives := make([]float64, 0, len(DefObjectives)) 176 for qu := range DefObjectives { 177 objectives = append(objectives, qu) 178 } 179 sort.Float64s(objectives) 180 181 for i, wantQ := range objectives { 182 ε := DefObjectives[wantQ] 183 gotQ := *m.Summary.Quantile[i].Quantile 184 gotV := *m.Summary.Quantile[i].Value 185 min, max := getBounds(allVars, wantQ, ε) 186 if gotQ != wantQ { 187 t.Errorf("got quantile %f, want %f", gotQ, wantQ) 188 } 189 if gotV < min || gotV > max { 190 t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max) 191 } 192 } 193 return true 194 } 195 196 if err := quick.Check(it, nil); err != nil { 197 t.Error(err) 198 } 199} 200 201func TestSummaryVecConcurrency(t *testing.T) { 202 if testing.Short() { 203 t.Skip("Skipping test in short mode.") 204 } 205 206 rand.Seed(42) 207 208 objectives := make([]float64, 0, len(DefObjectives)) 209 for qu := range DefObjectives { 210 211 objectives = append(objectives, qu) 212 } 213 sort.Float64s(objectives) 214 215 it := func(n uint32) bool { 216 mutations := int(n%1e4 + 1e4) 217 concLevel := int(n%7 + 1) 218 vecLength := int(n%3 + 1) 219 220 var start, end sync.WaitGroup 221 start.Add(1) 222 end.Add(concLevel) 223 224 sum := NewSummaryVec( 225 SummaryOpts{ 226 Name: "test_summary", 227 Help: "helpless", 228 }, 229 []string{"label"}, 230 ) 231 232 allVars := make([][]float64, vecLength) 233 sampleSums := make([]float64, vecLength) 234 for i := 0; i < concLevel; i++ { 235 vals := make([]float64, mutations) 236 picks := make([]int, mutations) 237 for j := 0; j < mutations; j++ { 238 v := rand.NormFloat64() 239 vals[j] = v 240 pick := rand.Intn(vecLength) 241 picks[j] = pick 242 allVars[pick] = append(allVars[pick], v) 243 sampleSums[pick] += v 244 } 245 246 go func(vals []float64) { 247 start.Wait() 248 for i, v := range vals { 249 sum.WithLabelValues(string('A' + picks[i])).Observe(v) 250 } 251 end.Done() 252 }(vals) 253 } 254 for _, vars := range allVars { 255 sort.Float64s(vars) 256 } 257 start.Done() 258 end.Wait() 259 260 for i := 0; i < vecLength; i++ { 261 m := &dto.Metric{} 262 s := sum.WithLabelValues(string('A' + i)) 263 s.Write(m) 264 if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want { 265 t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want) 266 } 267 if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 { 268 t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want) 269 } 270 for j, wantQ := range objectives { 271 ε := DefObjectives[wantQ] 272 gotQ := *m.Summary.Quantile[j].Quantile 273 gotV := *m.Summary.Quantile[j].Value 274 min, max := getBounds(allVars[i], wantQ, ε) 275 if gotQ != wantQ { 276 t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ) 277 } 278 if gotV < min || gotV > max { 279 t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max) 280 } 281 } 282 } 283 return true 284 } 285 286 if err := quick.Check(it, nil); err != nil { 287 t.Error(err) 288 } 289} 290 291func TestSummaryDecay(t *testing.T) { 292 if testing.Short() { 293 t.Skip("Skipping test in short mode.") 294 // More because it depends on timing than because it is particularly long... 295 } 296 297 sum := NewSummary(SummaryOpts{ 298 Name: "test_summary", 299 Help: "helpless", 300 MaxAge: 100 * time.Millisecond, 301 Objectives: map[float64]float64{0.1: 0.001}, 302 AgeBuckets: 10, 303 }) 304 305 m := &dto.Metric{} 306 i := 0 307 tick := time.NewTicker(time.Millisecond) 308 for _ = range tick.C { 309 i++ 310 sum.Observe(float64(i)) 311 if i%10 == 0 { 312 sum.Write(m) 313 if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 { 314 t.Errorf("%d. got %f, want %f", i, got, want) 315 } 316 m.Reset() 317 } 318 if i >= 1000 { 319 break 320 } 321 } 322 tick.Stop() 323 // Wait for MaxAge without observations and make sure quantiles are NaN. 324 time.Sleep(100 * time.Millisecond) 325 sum.Write(m) 326 if got := *m.Summary.Quantile[0].Value; !math.IsNaN(got) { 327 t.Errorf("got %f, want NaN after expiration", got) 328 } 329} 330 331func getBounds(vars []float64, q, ε float64) (min, max float64) { 332 // TODO: This currently tolerates an error of up to 2*ε. The error must 333 // be at most ε, but for some reason, it's sometimes slightly 334 // higher. That's a bug. 335 n := float64(len(vars)) 336 lower := int((q - 2*ε) * n) 337 upper := int(math.Ceil((q + 2*ε) * n)) 338 min = vars[0] 339 if lower > 1 { 340 min = vars[lower-1] 341 } 342 max = vars[len(vars)-1] 343 if upper < len(vars) { 344 max = vars[upper-1] 345 } 346 return 347} 348