1// Copyright 2015 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package prometheus 15 16import ( 17 "fmt" 18 "math" 19 "runtime" 20 "sort" 21 "sync" 22 "sync/atomic" 23 "time" 24 25 "github.com/golang/protobuf/proto" 26 27 dto "github.com/prometheus/client_model/go" 28) 29 30// A Histogram counts individual observations from an event or sample stream in 31// configurable buckets. Similar to a summary, it also provides a sum of 32// observations and an observation count. 33// 34// On the Prometheus server, quantiles can be calculated from a Histogram using 35// the histogram_quantile function in the query language. 36// 37// Note that Histograms, in contrast to Summaries, can be aggregated with the 38// Prometheus query language (see the documentation for detailed 39// procedures). However, Histograms require the user to pre-define suitable 40// buckets, and they are in general less accurate. The Observe method of a 41// Histogram has a very low performance overhead in comparison with the Observe 42// method of a Summary. 43// 44// To create Histogram instances, use NewHistogram. 45type Histogram interface { 46 Metric 47 Collector 48 49 // Observe adds a single observation to the histogram. 50 Observe(float64) 51} 52 53// bucketLabel is used for the label that defines the upper bound of a 54// bucket of a histogram ("le" -> "less or equal"). 55const bucketLabel = "le" 56 57// DefBuckets are the default Histogram buckets. The default buckets are 58// tailored to broadly measure the response time (in seconds) of a network 59// service. Most likely, however, you will be required to define buckets 60// customized to your use case. 61var ( 62 DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} 63 64 errBucketLabelNotAllowed = fmt.Errorf( 65 "%q is not allowed as label name in histograms", bucketLabel, 66 ) 67) 68 69// LinearBuckets creates 'count' buckets, each 'width' wide, where the lowest 70// bucket has an upper bound of 'start'. The final +Inf bucket is not counted 71// and not included in the returned slice. The returned slice is meant to be 72// used for the Buckets field of HistogramOpts. 73// 74// The function panics if 'count' is zero or negative. 75func LinearBuckets(start, width float64, count int) []float64 { 76 if count < 1 { 77 panic("LinearBuckets needs a positive count") 78 } 79 buckets := make([]float64, count) 80 for i := range buckets { 81 buckets[i] = start 82 start += width 83 } 84 return buckets 85} 86 87// ExponentialBuckets creates 'count' buckets, where the lowest bucket has an 88// upper bound of 'start' and each following bucket's upper bound is 'factor' 89// times the previous bucket's upper bound. The final +Inf bucket is not counted 90// and not included in the returned slice. The returned slice is meant to be 91// used for the Buckets field of HistogramOpts. 92// 93// The function panics if 'count' is 0 or negative, if 'start' is 0 or negative, 94// or if 'factor' is less than or equal 1. 95func ExponentialBuckets(start, factor float64, count int) []float64 { 96 if count < 1 { 97 panic("ExponentialBuckets needs a positive count") 98 } 99 if start <= 0 { 100 panic("ExponentialBuckets needs a positive start value") 101 } 102 if factor <= 1 { 103 panic("ExponentialBuckets needs a factor greater than 1") 104 } 105 buckets := make([]float64, count) 106 for i := range buckets { 107 buckets[i] = start 108 start *= factor 109 } 110 return buckets 111} 112 113// HistogramOpts bundles the options for creating a Histogram metric. It is 114// mandatory to set Name to a non-empty string. All other fields are optional 115// and can safely be left at their zero value, although it is strongly 116// encouraged to set a Help string. 117type HistogramOpts struct { 118 // Namespace, Subsystem, and Name are components of the fully-qualified 119 // name of the Histogram (created by joining these components with 120 // "_"). Only Name is mandatory, the others merely help structuring the 121 // name. Note that the fully-qualified name of the Histogram must be a 122 // valid Prometheus metric name. 123 Namespace string 124 Subsystem string 125 Name string 126 127 // Help provides information about this Histogram. 128 // 129 // Metrics with the same fully-qualified name must have the same Help 130 // string. 131 Help string 132 133 // ConstLabels are used to attach fixed labels to this metric. Metrics 134 // with the same fully-qualified name must have the same label names in 135 // their ConstLabels. 136 // 137 // ConstLabels are only used rarely. In particular, do not use them to 138 // attach the same labels to all your metrics. Those use cases are 139 // better covered by target labels set by the scraping Prometheus 140 // server, or by one specific metric (e.g. a build_info or a 141 // machine_role metric). See also 142 // https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels 143 ConstLabels Labels 144 145 // Buckets defines the buckets into which observations are counted. Each 146 // element in the slice is the upper inclusive bound of a bucket. The 147 // values must be sorted in strictly increasing order. There is no need 148 // to add a highest bucket with +Inf bound, it will be added 149 // implicitly. The default value is DefBuckets. 150 Buckets []float64 151} 152 153// NewHistogram creates a new Histogram based on the provided HistogramOpts. It 154// panics if the buckets in HistogramOpts are not in strictly increasing order. 155// 156// The returned implementation also implements ExemplarObserver. It is safe to 157// perform the corresponding type assertion. Exemplars are tracked separately 158// for each bucket. 159func NewHistogram(opts HistogramOpts) Histogram { 160 return newHistogram( 161 NewDesc( 162 BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), 163 opts.Help, 164 nil, 165 opts.ConstLabels, 166 ), 167 opts, 168 ) 169} 170 171func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogram { 172 if len(desc.variableLabels) != len(labelValues) { 173 panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, labelValues)) 174 } 175 176 for _, n := range desc.variableLabels { 177 if n == bucketLabel { 178 panic(errBucketLabelNotAllowed) 179 } 180 } 181 for _, lp := range desc.constLabelPairs { 182 if lp.GetName() == bucketLabel { 183 panic(errBucketLabelNotAllowed) 184 } 185 } 186 187 if len(opts.Buckets) == 0 { 188 opts.Buckets = DefBuckets 189 } 190 191 h := &histogram{ 192 desc: desc, 193 upperBounds: opts.Buckets, 194 labelPairs: makeLabelPairs(desc, labelValues), 195 counts: [2]*histogramCounts{{}, {}}, 196 now: time.Now, 197 } 198 for i, upperBound := range h.upperBounds { 199 if i < len(h.upperBounds)-1 { 200 if upperBound >= h.upperBounds[i+1] { 201 panic(fmt.Errorf( 202 "histogram buckets must be in increasing order: %f >= %f", 203 upperBound, h.upperBounds[i+1], 204 )) 205 } 206 } else { 207 if math.IsInf(upperBound, +1) { 208 // The +Inf bucket is implicit. Remove it here. 209 h.upperBounds = h.upperBounds[:i] 210 } 211 } 212 } 213 // Finally we know the final length of h.upperBounds and can make buckets 214 // for both counts as well as exemplars: 215 h.counts[0].buckets = make([]uint64, len(h.upperBounds)) 216 h.counts[1].buckets = make([]uint64, len(h.upperBounds)) 217 h.exemplars = make([]atomic.Value, len(h.upperBounds)+1) 218 219 h.init(h) // Init self-collection. 220 return h 221} 222 223type histogramCounts struct { 224 // sumBits contains the bits of the float64 representing the sum of all 225 // observations. sumBits and count have to go first in the struct to 226 // guarantee alignment for atomic operations. 227 // http://golang.org/pkg/sync/atomic/#pkg-note-BUG 228 sumBits uint64 229 count uint64 230 buckets []uint64 231} 232 233type histogram struct { 234 // countAndHotIdx enables lock-free writes with use of atomic updates. 235 // The most significant bit is the hot index [0 or 1] of the count field 236 // below. Observe calls update the hot one. All remaining bits count the 237 // number of Observe calls. Observe starts by incrementing this counter, 238 // and finish by incrementing the count field in the respective 239 // histogramCounts, as a marker for completion. 240 // 241 // Calls of the Write method (which are non-mutating reads from the 242 // perspective of the histogram) swap the hot–cold under the writeMtx 243 // lock. A cooldown is awaited (while locked) by comparing the number of 244 // observations with the initiation count. Once they match, then the 245 // last observation on the now cool one has completed. All cool fields must 246 // be merged into the new hot before releasing writeMtx. 247 // 248 // Fields with atomic access first! See alignment constraint: 249 // http://golang.org/pkg/sync/atomic/#pkg-note-BUG 250 countAndHotIdx uint64 251 252 selfCollector 253 desc *Desc 254 writeMtx sync.Mutex // Only used in the Write method. 255 256 // Two counts, one is "hot" for lock-free observations, the other is 257 // "cold" for writing out a dto.Metric. It has to be an array of 258 // pointers to guarantee 64bit alignment of the histogramCounts, see 259 // http://golang.org/pkg/sync/atomic/#pkg-note-BUG. 260 counts [2]*histogramCounts 261 262 upperBounds []float64 263 labelPairs []*dto.LabelPair 264 exemplars []atomic.Value // One more than buckets (to include +Inf), each a *dto.Exemplar. 265 266 now func() time.Time // To mock out time.Now() for testing. 267} 268 269func (h *histogram) Desc() *Desc { 270 return h.desc 271} 272 273func (h *histogram) Observe(v float64) { 274 h.observe(v, h.findBucket(v)) 275} 276 277func (h *histogram) ObserveWithExemplar(v float64, e Labels) { 278 i := h.findBucket(v) 279 h.observe(v, i) 280 h.updateExemplar(v, i, e) 281} 282 283func (h *histogram) Write(out *dto.Metric) error { 284 // For simplicity, we protect this whole method by a mutex. It is not in 285 // the hot path, i.e. Observe is called much more often than Write. The 286 // complication of making Write lock-free isn't worth it, if possible at 287 // all. 288 h.writeMtx.Lock() 289 defer h.writeMtx.Unlock() 290 291 // Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0) 292 // without touching the count bits. See the struct comments for a full 293 // description of the algorithm. 294 n := atomic.AddUint64(&h.countAndHotIdx, 1<<63) 295 // count is contained unchanged in the lower 63 bits. 296 count := n & ((1 << 63) - 1) 297 // The most significant bit tells us which counts is hot. The complement 298 // is thus the cold one. 299 hotCounts := h.counts[n>>63] 300 coldCounts := h.counts[(^n)>>63] 301 302 // Await cooldown. 303 for count != atomic.LoadUint64(&coldCounts.count) { 304 runtime.Gosched() // Let observations get work done. 305 } 306 307 his := &dto.Histogram{ 308 Bucket: make([]*dto.Bucket, len(h.upperBounds)), 309 SampleCount: proto.Uint64(count), 310 SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))), 311 } 312 var cumCount uint64 313 for i, upperBound := range h.upperBounds { 314 cumCount += atomic.LoadUint64(&coldCounts.buckets[i]) 315 his.Bucket[i] = &dto.Bucket{ 316 CumulativeCount: proto.Uint64(cumCount), 317 UpperBound: proto.Float64(upperBound), 318 } 319 if e := h.exemplars[i].Load(); e != nil { 320 his.Bucket[i].Exemplar = e.(*dto.Exemplar) 321 } 322 } 323 // If there is an exemplar for the +Inf bucket, we have to add that bucket explicitly. 324 if e := h.exemplars[len(h.upperBounds)].Load(); e != nil { 325 b := &dto.Bucket{ 326 CumulativeCount: proto.Uint64(count), 327 UpperBound: proto.Float64(math.Inf(1)), 328 Exemplar: e.(*dto.Exemplar), 329 } 330 his.Bucket = append(his.Bucket, b) 331 } 332 333 out.Histogram = his 334 out.Label = h.labelPairs 335 336 // Finally add all the cold counts to the new hot counts and reset the cold counts. 337 atomic.AddUint64(&hotCounts.count, count) 338 atomic.StoreUint64(&coldCounts.count, 0) 339 for { 340 oldBits := atomic.LoadUint64(&hotCounts.sumBits) 341 newBits := math.Float64bits(math.Float64frombits(oldBits) + his.GetSampleSum()) 342 if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) { 343 atomic.StoreUint64(&coldCounts.sumBits, 0) 344 break 345 } 346 } 347 for i := range h.upperBounds { 348 atomic.AddUint64(&hotCounts.buckets[i], atomic.LoadUint64(&coldCounts.buckets[i])) 349 atomic.StoreUint64(&coldCounts.buckets[i], 0) 350 } 351 return nil 352} 353 354// findBucket returns the index of the bucket for the provided value, or 355// len(h.upperBounds) for the +Inf bucket. 356func (h *histogram) findBucket(v float64) int { 357 // TODO(beorn7): For small numbers of buckets (<30), a linear search is 358 // slightly faster than the binary search. If we really care, we could 359 // switch from one search strategy to the other depending on the number 360 // of buckets. 361 // 362 // Microbenchmarks (BenchmarkHistogramNoLabels): 363 // 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op 364 // 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op 365 // 300 buckets: 154 ns/op linear - binary 61.6 ns/op 366 return sort.SearchFloat64s(h.upperBounds, v) 367} 368 369// observe is the implementation for Observe without the findBucket part. 370func (h *histogram) observe(v float64, bucket int) { 371 // We increment h.countAndHotIdx so that the counter in the lower 372 // 63 bits gets incremented. At the same time, we get the new value 373 // back, which we can use to find the currently-hot counts. 374 n := atomic.AddUint64(&h.countAndHotIdx, 1) 375 hotCounts := h.counts[n>>63] 376 377 if bucket < len(h.upperBounds) { 378 atomic.AddUint64(&hotCounts.buckets[bucket], 1) 379 } 380 for { 381 oldBits := atomic.LoadUint64(&hotCounts.sumBits) 382 newBits := math.Float64bits(math.Float64frombits(oldBits) + v) 383 if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) { 384 break 385 } 386 } 387 // Increment count last as we take it as a signal that the observation 388 // is complete. 389 atomic.AddUint64(&hotCounts.count, 1) 390} 391 392// updateExemplar replaces the exemplar for the provided bucket. With empty 393// labels, it's a no-op. It panics if any of the labels is invalid. 394func (h *histogram) updateExemplar(v float64, bucket int, l Labels) { 395 if l == nil { 396 return 397 } 398 e, err := newExemplar(v, h.now(), l) 399 if err != nil { 400 panic(err) 401 } 402 h.exemplars[bucket].Store(e) 403} 404 405// HistogramVec is a Collector that bundles a set of Histograms that all share the 406// same Desc, but have different values for their variable labels. This is used 407// if you want to count the same thing partitioned by various dimensions 408// (e.g. HTTP request latencies, partitioned by status code and method). Create 409// instances with NewHistogramVec. 410type HistogramVec struct { 411 *metricVec 412} 413 414// NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and 415// partitioned by the given label names. 416func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { 417 desc := NewDesc( 418 BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), 419 opts.Help, 420 labelNames, 421 opts.ConstLabels, 422 ) 423 return &HistogramVec{ 424 metricVec: newMetricVec(desc, func(lvs ...string) Metric { 425 return newHistogram(desc, opts, lvs...) 426 }), 427 } 428} 429 430// GetMetricWithLabelValues returns the Histogram for the given slice of label 431// values (same order as the VariableLabels in Desc). If that combination of 432// label values is accessed for the first time, a new Histogram is created. 433// 434// It is possible to call this method without using the returned Histogram to only 435// create the new Histogram but leave it at its starting value, a Histogram without 436// any observations. 437// 438// Keeping the Histogram for later use is possible (and should be considered if 439// performance is critical), but keep in mind that Reset, DeleteLabelValues and 440// Delete can be used to delete the Histogram from the HistogramVec. In that case, the 441// Histogram will still exist, but it will not be exported anymore, even if a 442// Histogram with the same label values is created later. See also the CounterVec 443// example. 444// 445// An error is returned if the number of label values is not the same as the 446// number of VariableLabels in Desc (minus any curried labels). 447// 448// Note that for more than one label value, this method is prone to mistakes 449// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as 450// an alternative to avoid that type of mistake. For higher label numbers, the 451// latter has a much more readable (albeit more verbose) syntax, but it comes 452// with a performance overhead (for creating and processing the Labels map). 453// See also the GaugeVec example. 454func (v *HistogramVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) { 455 metric, err := v.metricVec.getMetricWithLabelValues(lvs...) 456 if metric != nil { 457 return metric.(Observer), err 458 } 459 return nil, err 460} 461 462// GetMetricWith returns the Histogram for the given Labels map (the label names 463// must match those of the VariableLabels in Desc). If that label map is 464// accessed for the first time, a new Histogram is created. Implications of 465// creating a Histogram without using it and keeping the Histogram for later use 466// are the same as for GetMetricWithLabelValues. 467// 468// An error is returned if the number and names of the Labels are inconsistent 469// with those of the VariableLabels in Desc (minus any curried labels). 470// 471// This method is used for the same purpose as 472// GetMetricWithLabelValues(...string). See there for pros and cons of the two 473// methods. 474func (v *HistogramVec) GetMetricWith(labels Labels) (Observer, error) { 475 metric, err := v.metricVec.getMetricWith(labels) 476 if metric != nil { 477 return metric.(Observer), err 478 } 479 return nil, err 480} 481 482// WithLabelValues works as GetMetricWithLabelValues, but panics where 483// GetMetricWithLabelValues would have returned an error. Not returning an 484// error allows shortcuts like 485// myVec.WithLabelValues("404", "GET").Observe(42.21) 486func (v *HistogramVec) WithLabelValues(lvs ...string) Observer { 487 h, err := v.GetMetricWithLabelValues(lvs...) 488 if err != nil { 489 panic(err) 490 } 491 return h 492} 493 494// With works as GetMetricWith but panics where GetMetricWithLabels would have 495// returned an error. Not returning an error allows shortcuts like 496// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Observe(42.21) 497func (v *HistogramVec) With(labels Labels) Observer { 498 h, err := v.GetMetricWith(labels) 499 if err != nil { 500 panic(err) 501 } 502 return h 503} 504 505// CurryWith returns a vector curried with the provided labels, i.e. the 506// returned vector has those labels pre-set for all labeled operations performed 507// on it. The cardinality of the curried vector is reduced accordingly. The 508// order of the remaining labels stays the same (just with the curried labels 509// taken out of the sequence – which is relevant for the 510// (GetMetric)WithLabelValues methods). It is possible to curry a curried 511// vector, but only with labels not yet used for currying before. 512// 513// The metrics contained in the HistogramVec are shared between the curried and 514// uncurried vectors. They are just accessed differently. Curried and uncurried 515// vectors behave identically in terms of collection. Only one must be 516// registered with a given registry (usually the uncurried version). The Reset 517// method deletes all metrics, even if called on a curried vector. 518func (v *HistogramVec) CurryWith(labels Labels) (ObserverVec, error) { 519 vec, err := v.curryWith(labels) 520 if vec != nil { 521 return &HistogramVec{vec}, err 522 } 523 return nil, err 524} 525 526// MustCurryWith works as CurryWith but panics where CurryWith would have 527// returned an error. 528func (v *HistogramVec) MustCurryWith(labels Labels) ObserverVec { 529 vec, err := v.CurryWith(labels) 530 if err != nil { 531 panic(err) 532 } 533 return vec 534} 535 536type constHistogram struct { 537 desc *Desc 538 count uint64 539 sum float64 540 buckets map[float64]uint64 541 labelPairs []*dto.LabelPair 542} 543 544func (h *constHistogram) Desc() *Desc { 545 return h.desc 546} 547 548func (h *constHistogram) Write(out *dto.Metric) error { 549 his := &dto.Histogram{} 550 buckets := make([]*dto.Bucket, 0, len(h.buckets)) 551 552 his.SampleCount = proto.Uint64(h.count) 553 his.SampleSum = proto.Float64(h.sum) 554 555 for upperBound, count := range h.buckets { 556 buckets = append(buckets, &dto.Bucket{ 557 CumulativeCount: proto.Uint64(count), 558 UpperBound: proto.Float64(upperBound), 559 }) 560 } 561 562 if len(buckets) > 0 { 563 sort.Sort(buckSort(buckets)) 564 } 565 his.Bucket = buckets 566 567 out.Histogram = his 568 out.Label = h.labelPairs 569 570 return nil 571} 572 573// NewConstHistogram returns a metric representing a Prometheus histogram with 574// fixed values for the count, sum, and bucket counts. As those parameters 575// cannot be changed, the returned value does not implement the Histogram 576// interface (but only the Metric interface). Users of this package will not 577// have much use for it in regular operations. However, when implementing custom 578// Collectors, it is useful as a throw-away metric that is generated on the fly 579// to send it to Prometheus in the Collect method. 580// 581// buckets is a map of upper bounds to cumulative counts, excluding the +Inf 582// bucket. 583// 584// NewConstHistogram returns an error if the length of labelValues is not 585// consistent with the variable labels in Desc or if Desc is invalid. 586func NewConstHistogram( 587 desc *Desc, 588 count uint64, 589 sum float64, 590 buckets map[float64]uint64, 591 labelValues ...string, 592) (Metric, error) { 593 if desc.err != nil { 594 return nil, desc.err 595 } 596 if err := validateLabelValues(labelValues, len(desc.variableLabels)); err != nil { 597 return nil, err 598 } 599 return &constHistogram{ 600 desc: desc, 601 count: count, 602 sum: sum, 603 buckets: buckets, 604 labelPairs: makeLabelPairs(desc, labelValues), 605 }, nil 606} 607 608// MustNewConstHistogram is a version of NewConstHistogram that panics where 609// NewConstMetric would have returned an error. 610func MustNewConstHistogram( 611 desc *Desc, 612 count uint64, 613 sum float64, 614 buckets map[float64]uint64, 615 labelValues ...string, 616) Metric { 617 m, err := NewConstHistogram(desc, count, sum, buckets, labelValues...) 618 if err != nil { 619 panic(err) 620 } 621 return m 622} 623 624type buckSort []*dto.Bucket 625 626func (s buckSort) Len() int { 627 return len(s) 628} 629 630func (s buckSort) Swap(i, j int) { 631 s[i], s[j] = s[j], s[i] 632} 633 634func (s buckSort) Less(i, j int) bool { 635 return s[i].GetUpperBound() < s[j].GetUpperBound() 636} 637