1// Copyright 2014 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package prometheus 15 16import ( 17 "bytes" 18 "fmt" 19 "io/ioutil" 20 "os" 21 "path/filepath" 22 "runtime" 23 "sort" 24 "strings" 25 "sync" 26 "unicode/utf8" 27 28 "github.com/golang/protobuf/proto" 29 "github.com/prometheus/common/expfmt" 30 31 dto "github.com/prometheus/client_model/go" 32 33 "github.com/prometheus/client_golang/prometheus/internal" 34) 35 36const ( 37 // Capacity for the channel to collect metrics and descriptors. 38 capMetricChan = 1000 39 capDescChan = 10 40) 41 42// DefaultRegisterer and DefaultGatherer are the implementations of the 43// Registerer and Gatherer interface a number of convenience functions in this 44// package act on. Initially, both variables point to the same Registry, which 45// has a process collector (currently on Linux only, see NewProcessCollector) 46// and a Go collector (see NewGoCollector, in particular the note about 47// stop-the-world implication with Go versions older than 1.9) already 48// registered. This approach to keep default instances as global state mirrors 49// the approach of other packages in the Go standard library. Note that there 50// are caveats. Change the variables with caution and only if you understand the 51// consequences. Users who want to avoid global state altogether should not use 52// the convenience functions and act on custom instances instead. 53var ( 54 defaultRegistry = NewRegistry() 55 DefaultRegisterer Registerer = defaultRegistry 56 DefaultGatherer Gatherer = defaultRegistry 57) 58 59func init() { 60 MustRegister(NewProcessCollector(ProcessCollectorOpts{})) 61 MustRegister(NewGoCollector()) 62} 63 64// NewRegistry creates a new vanilla Registry without any Collectors 65// pre-registered. 66func NewRegistry() *Registry { 67 return &Registry{ 68 collectorsByID: map[uint64]Collector{}, 69 descIDs: map[uint64]struct{}{}, 70 dimHashesByName: map[string]uint64{}, 71 } 72} 73 74// NewPedanticRegistry returns a registry that checks during collection if each 75// collected Metric is consistent with its reported Desc, and if the Desc has 76// actually been registered with the registry. Unchecked Collectors (those whose 77// Describe methed does not yield any descriptors) are excluded from the check. 78// 79// Usually, a Registry will be happy as long as the union of all collected 80// Metrics is consistent and valid even if some metrics are not consistent with 81// their own Desc or a Desc provided by their registered Collector. Well-behaved 82// Collectors and Metrics will only provide consistent Descs. This Registry is 83// useful to test the implementation of Collectors and Metrics. 84func NewPedanticRegistry() *Registry { 85 r := NewRegistry() 86 r.pedanticChecksEnabled = true 87 return r 88} 89 90// Registerer is the interface for the part of a registry in charge of 91// registering and unregistering. Users of custom registries should use 92// Registerer as type for registration purposes (rather than the Registry type 93// directly). In that way, they are free to use custom Registerer implementation 94// (e.g. for testing purposes). 95type Registerer interface { 96 // Register registers a new Collector to be included in metrics 97 // collection. It returns an error if the descriptors provided by the 98 // Collector are invalid or if they — in combination with descriptors of 99 // already registered Collectors — do not fulfill the consistency and 100 // uniqueness criteria described in the documentation of metric.Desc. 101 // 102 // If the provided Collector is equal to a Collector already registered 103 // (which includes the case of re-registering the same Collector), the 104 // returned error is an instance of AlreadyRegisteredError, which 105 // contains the previously registered Collector. 106 // 107 // A Collector whose Describe method does not yield any Desc is treated 108 // as unchecked. Registration will always succeed. No check for 109 // re-registering (see previous paragraph) is performed. Thus, the 110 // caller is responsible for not double-registering the same unchecked 111 // Collector, and for providing a Collector that will not cause 112 // inconsistent metrics on collection. (This would lead to scrape 113 // errors.) 114 Register(Collector) error 115 // MustRegister works like Register but registers any number of 116 // Collectors and panics upon the first registration that causes an 117 // error. 118 MustRegister(...Collector) 119 // Unregister unregisters the Collector that equals the Collector passed 120 // in as an argument. (Two Collectors are considered equal if their 121 // Describe method yields the same set of descriptors.) The function 122 // returns whether a Collector was unregistered. Note that an unchecked 123 // Collector cannot be unregistered (as its Describe method does not 124 // yield any descriptor). 125 // 126 // Note that even after unregistering, it will not be possible to 127 // register a new Collector that is inconsistent with the unregistered 128 // Collector, e.g. a Collector collecting metrics with the same name but 129 // a different help string. The rationale here is that the same registry 130 // instance must only collect consistent metrics throughout its 131 // lifetime. 132 Unregister(Collector) bool 133} 134 135// Gatherer is the interface for the part of a registry in charge of gathering 136// the collected metrics into a number of MetricFamilies. The Gatherer interface 137// comes with the same general implication as described for the Registerer 138// interface. 139type Gatherer interface { 140 // Gather calls the Collect method of the registered Collectors and then 141 // gathers the collected metrics into a lexicographically sorted slice 142 // of uniquely named MetricFamily protobufs. Gather ensures that the 143 // returned slice is valid and self-consistent so that it can be used 144 // for valid exposition. As an exception to the strict consistency 145 // requirements described for metric.Desc, Gather will tolerate 146 // different sets of label names for metrics of the same metric family. 147 // 148 // Even if an error occurs, Gather attempts to gather as many metrics as 149 // possible. Hence, if a non-nil error is returned, the returned 150 // MetricFamily slice could be nil (in case of a fatal error that 151 // prevented any meaningful metric collection) or contain a number of 152 // MetricFamily protobufs, some of which might be incomplete, and some 153 // might be missing altogether. The returned error (which might be a 154 // MultiError) explains the details. Note that this is mostly useful for 155 // debugging purposes. If the gathered protobufs are to be used for 156 // exposition in actual monitoring, it is almost always better to not 157 // expose an incomplete result and instead disregard the returned 158 // MetricFamily protobufs in case the returned error is non-nil. 159 Gather() ([]*dto.MetricFamily, error) 160} 161 162// Register registers the provided Collector with the DefaultRegisterer. 163// 164// Register is a shortcut for DefaultRegisterer.Register(c). See there for more 165// details. 166func Register(c Collector) error { 167 return DefaultRegisterer.Register(c) 168} 169 170// MustRegister registers the provided Collectors with the DefaultRegisterer and 171// panics if any error occurs. 172// 173// MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See 174// there for more details. 175func MustRegister(cs ...Collector) { 176 DefaultRegisterer.MustRegister(cs...) 177} 178 179// Unregister removes the registration of the provided Collector from the 180// DefaultRegisterer. 181// 182// Unregister is a shortcut for DefaultRegisterer.Unregister(c). See there for 183// more details. 184func Unregister(c Collector) bool { 185 return DefaultRegisterer.Unregister(c) 186} 187 188// GathererFunc turns a function into a Gatherer. 189type GathererFunc func() ([]*dto.MetricFamily, error) 190 191// Gather implements Gatherer. 192func (gf GathererFunc) Gather() ([]*dto.MetricFamily, error) { 193 return gf() 194} 195 196// AlreadyRegisteredError is returned by the Register method if the Collector to 197// be registered has already been registered before, or a different Collector 198// that collects the same metrics has been registered before. Registration fails 199// in that case, but you can detect from the kind of error what has 200// happened. The error contains fields for the existing Collector and the 201// (rejected) new Collector that equals the existing one. This can be used to 202// find out if an equal Collector has been registered before and switch over to 203// using the old one, as demonstrated in the example. 204type AlreadyRegisteredError struct { 205 ExistingCollector, NewCollector Collector 206} 207 208func (err AlreadyRegisteredError) Error() string { 209 return "duplicate metrics collector registration attempted" 210} 211 212// MultiError is a slice of errors implementing the error interface. It is used 213// by a Gatherer to report multiple errors during MetricFamily gathering. 214type MultiError []error 215 216func (errs MultiError) Error() string { 217 if len(errs) == 0 { 218 return "" 219 } 220 buf := &bytes.Buffer{} 221 fmt.Fprintf(buf, "%d error(s) occurred:", len(errs)) 222 for _, err := range errs { 223 fmt.Fprintf(buf, "\n* %s", err) 224 } 225 return buf.String() 226} 227 228// Append appends the provided error if it is not nil. 229func (errs *MultiError) Append(err error) { 230 if err != nil { 231 *errs = append(*errs, err) 232 } 233} 234 235// MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only 236// contained error as error if len(errs is 1). In all other cases, it returns 237// the MultiError directly. This is helpful for returning a MultiError in a way 238// that only uses the MultiError if needed. 239func (errs MultiError) MaybeUnwrap() error { 240 switch len(errs) { 241 case 0: 242 return nil 243 case 1: 244 return errs[0] 245 default: 246 return errs 247 } 248} 249 250// Registry registers Prometheus collectors, collects their metrics, and gathers 251// them into MetricFamilies for exposition. It implements both Registerer and 252// Gatherer. The zero value is not usable. Create instances with NewRegistry or 253// NewPedanticRegistry. 254type Registry struct { 255 mtx sync.RWMutex 256 collectorsByID map[uint64]Collector // ID is a hash of the descIDs. 257 descIDs map[uint64]struct{} 258 dimHashesByName map[string]uint64 259 uncheckedCollectors []Collector 260 pedanticChecksEnabled bool 261} 262 263// Register implements Registerer. 264func (r *Registry) Register(c Collector) error { 265 var ( 266 descChan = make(chan *Desc, capDescChan) 267 newDescIDs = map[uint64]struct{}{} 268 newDimHashesByName = map[string]uint64{} 269 collectorID uint64 // Just a sum of all desc IDs. 270 duplicateDescErr error 271 ) 272 go func() { 273 c.Describe(descChan) 274 close(descChan) 275 }() 276 r.mtx.Lock() 277 defer func() { 278 // Drain channel in case of premature return to not leak a goroutine. 279 for range descChan { 280 } 281 r.mtx.Unlock() 282 }() 283 // Conduct various tests... 284 for desc := range descChan { 285 286 // Is the descriptor valid at all? 287 if desc.err != nil { 288 return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err) 289 } 290 291 // Is the descID unique? 292 // (In other words: Is the fqName + constLabel combination unique?) 293 if _, exists := r.descIDs[desc.id]; exists { 294 duplicateDescErr = fmt.Errorf("descriptor %s already exists with the same fully-qualified name and const label values", desc) 295 } 296 // If it is not a duplicate desc in this collector, add it to 297 // the collectorID. (We allow duplicate descs within the same 298 // collector, but their existence must be a no-op.) 299 if _, exists := newDescIDs[desc.id]; !exists { 300 newDescIDs[desc.id] = struct{}{} 301 collectorID += desc.id 302 } 303 304 // Are all the label names and the help string consistent with 305 // previous descriptors of the same name? 306 // First check existing descriptors... 307 if dimHash, exists := r.dimHashesByName[desc.fqName]; exists { 308 if dimHash != desc.dimHash { 309 return fmt.Errorf("a previously registered descriptor with the same fully-qualified name as %s has different label names or a different help string", desc) 310 } 311 } else { 312 // ...then check the new descriptors already seen. 313 if dimHash, exists := newDimHashesByName[desc.fqName]; exists { 314 if dimHash != desc.dimHash { 315 return fmt.Errorf("descriptors reported by collector have inconsistent label names or help strings for the same fully-qualified name, offender is %s", desc) 316 } 317 } else { 318 newDimHashesByName[desc.fqName] = desc.dimHash 319 } 320 } 321 } 322 // A Collector yielding no Desc at all is considered unchecked. 323 if len(newDescIDs) == 0 { 324 r.uncheckedCollectors = append(r.uncheckedCollectors, c) 325 return nil 326 } 327 if existing, exists := r.collectorsByID[collectorID]; exists { 328 switch e := existing.(type) { 329 case *wrappingCollector: 330 return AlreadyRegisteredError{ 331 ExistingCollector: e.unwrapRecursively(), 332 NewCollector: c, 333 } 334 default: 335 return AlreadyRegisteredError{ 336 ExistingCollector: e, 337 NewCollector: c, 338 } 339 } 340 } 341 // If the collectorID is new, but at least one of the descs existed 342 // before, we are in trouble. 343 if duplicateDescErr != nil { 344 return duplicateDescErr 345 } 346 347 // Only after all tests have passed, actually register. 348 r.collectorsByID[collectorID] = c 349 for hash := range newDescIDs { 350 r.descIDs[hash] = struct{}{} 351 } 352 for name, dimHash := range newDimHashesByName { 353 r.dimHashesByName[name] = dimHash 354 } 355 return nil 356} 357 358// Unregister implements Registerer. 359func (r *Registry) Unregister(c Collector) bool { 360 var ( 361 descChan = make(chan *Desc, capDescChan) 362 descIDs = map[uint64]struct{}{} 363 collectorID uint64 // Just a sum of the desc IDs. 364 ) 365 go func() { 366 c.Describe(descChan) 367 close(descChan) 368 }() 369 for desc := range descChan { 370 if _, exists := descIDs[desc.id]; !exists { 371 collectorID += desc.id 372 descIDs[desc.id] = struct{}{} 373 } 374 } 375 376 r.mtx.RLock() 377 if _, exists := r.collectorsByID[collectorID]; !exists { 378 r.mtx.RUnlock() 379 return false 380 } 381 r.mtx.RUnlock() 382 383 r.mtx.Lock() 384 defer r.mtx.Unlock() 385 386 delete(r.collectorsByID, collectorID) 387 for id := range descIDs { 388 delete(r.descIDs, id) 389 } 390 // dimHashesByName is left untouched as those must be consistent 391 // throughout the lifetime of a program. 392 return true 393} 394 395// MustRegister implements Registerer. 396func (r *Registry) MustRegister(cs ...Collector) { 397 for _, c := range cs { 398 if err := r.Register(c); err != nil { 399 panic(err) 400 } 401 } 402} 403 404// Gather implements Gatherer. 405func (r *Registry) Gather() ([]*dto.MetricFamily, error) { 406 var ( 407 checkedMetricChan = make(chan Metric, capMetricChan) 408 uncheckedMetricChan = make(chan Metric, capMetricChan) 409 metricHashes = map[uint64]struct{}{} 410 wg sync.WaitGroup 411 errs MultiError // The collected errors to return in the end. 412 registeredDescIDs map[uint64]struct{} // Only used for pedantic checks 413 ) 414 415 r.mtx.RLock() 416 goroutineBudget := len(r.collectorsByID) + len(r.uncheckedCollectors) 417 metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName)) 418 checkedCollectors := make(chan Collector, len(r.collectorsByID)) 419 uncheckedCollectors := make(chan Collector, len(r.uncheckedCollectors)) 420 for _, collector := range r.collectorsByID { 421 checkedCollectors <- collector 422 } 423 for _, collector := range r.uncheckedCollectors { 424 uncheckedCollectors <- collector 425 } 426 // In case pedantic checks are enabled, we have to copy the map before 427 // giving up the RLock. 428 if r.pedanticChecksEnabled { 429 registeredDescIDs = make(map[uint64]struct{}, len(r.descIDs)) 430 for id := range r.descIDs { 431 registeredDescIDs[id] = struct{}{} 432 } 433 } 434 r.mtx.RUnlock() 435 436 wg.Add(goroutineBudget) 437 438 collectWorker := func() { 439 for { 440 select { 441 case collector := <-checkedCollectors: 442 collector.Collect(checkedMetricChan) 443 case collector := <-uncheckedCollectors: 444 collector.Collect(uncheckedMetricChan) 445 default: 446 return 447 } 448 wg.Done() 449 } 450 } 451 452 // Start the first worker now to make sure at least one is running. 453 go collectWorker() 454 goroutineBudget-- 455 456 // Close checkedMetricChan and uncheckedMetricChan once all collectors 457 // are collected. 458 go func() { 459 wg.Wait() 460 close(checkedMetricChan) 461 close(uncheckedMetricChan) 462 }() 463 464 // Drain checkedMetricChan and uncheckedMetricChan in case of premature return. 465 defer func() { 466 if checkedMetricChan != nil { 467 for range checkedMetricChan { 468 } 469 } 470 if uncheckedMetricChan != nil { 471 for range uncheckedMetricChan { 472 } 473 } 474 }() 475 476 // Copy the channel references so we can nil them out later to remove 477 // them from the select statements below. 478 cmc := checkedMetricChan 479 umc := uncheckedMetricChan 480 481 for { 482 select { 483 case metric, ok := <-cmc: 484 if !ok { 485 cmc = nil 486 break 487 } 488 errs.Append(processMetric( 489 metric, metricFamiliesByName, 490 metricHashes, 491 registeredDescIDs, 492 )) 493 case metric, ok := <-umc: 494 if !ok { 495 umc = nil 496 break 497 } 498 errs.Append(processMetric( 499 metric, metricFamiliesByName, 500 metricHashes, 501 nil, 502 )) 503 default: 504 if goroutineBudget <= 0 || len(checkedCollectors)+len(uncheckedCollectors) == 0 { 505 // All collectors are already being worked on or 506 // we have already as many goroutines started as 507 // there are collectors. Do the same as above, 508 // just without the default. 509 select { 510 case metric, ok := <-cmc: 511 if !ok { 512 cmc = nil 513 break 514 } 515 errs.Append(processMetric( 516 metric, metricFamiliesByName, 517 metricHashes, 518 registeredDescIDs, 519 )) 520 case metric, ok := <-umc: 521 if !ok { 522 umc = nil 523 break 524 } 525 errs.Append(processMetric( 526 metric, metricFamiliesByName, 527 metricHashes, 528 nil, 529 )) 530 } 531 break 532 } 533 // Start more workers. 534 go collectWorker() 535 goroutineBudget-- 536 runtime.Gosched() 537 } 538 // Once both checkedMetricChan and uncheckdMetricChan are closed 539 // and drained, the contraption above will nil out cmc and umc, 540 // and then we can leave the collect loop here. 541 if cmc == nil && umc == nil { 542 break 543 } 544 } 545 return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() 546} 547 548// WriteToTextfile calls Gather on the provided Gatherer, encodes the result in the 549// Prometheus text format, and writes it to a temporary file. Upon success, the 550// temporary file is renamed to the provided filename. 551// 552// This is intended for use with the textfile collector of the node exporter. 553// Note that the node exporter expects the filename to be suffixed with ".prom". 554func WriteToTextfile(filename string, g Gatherer) error { 555 tmp, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)) 556 if err != nil { 557 return err 558 } 559 defer os.Remove(tmp.Name()) 560 561 mfs, err := g.Gather() 562 if err != nil { 563 return err 564 } 565 for _, mf := range mfs { 566 if _, err := expfmt.MetricFamilyToText(tmp, mf); err != nil { 567 return err 568 } 569 } 570 if err := tmp.Close(); err != nil { 571 return err 572 } 573 574 if err := os.Chmod(tmp.Name(), 0644); err != nil { 575 return err 576 } 577 return os.Rename(tmp.Name(), filename) 578} 579 580// processMetric is an internal helper method only used by the Gather method. 581func processMetric( 582 metric Metric, 583 metricFamiliesByName map[string]*dto.MetricFamily, 584 metricHashes map[uint64]struct{}, 585 registeredDescIDs map[uint64]struct{}, 586) error { 587 desc := metric.Desc() 588 // Wrapped metrics collected by an unchecked Collector can have an 589 // invalid Desc. 590 if desc.err != nil { 591 return desc.err 592 } 593 dtoMetric := &dto.Metric{} 594 if err := metric.Write(dtoMetric); err != nil { 595 return fmt.Errorf("error collecting metric %v: %s", desc, err) 596 } 597 metricFamily, ok := metricFamiliesByName[desc.fqName] 598 if ok { // Existing name. 599 if metricFamily.GetHelp() != desc.help { 600 return fmt.Errorf( 601 "collected metric %s %s has help %q but should have %q", 602 desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(), 603 ) 604 } 605 // TODO(beorn7): Simplify switch once Desc has type. 606 switch metricFamily.GetType() { 607 case dto.MetricType_COUNTER: 608 if dtoMetric.Counter == nil { 609 return fmt.Errorf( 610 "collected metric %s %s should be a Counter", 611 desc.fqName, dtoMetric, 612 ) 613 } 614 case dto.MetricType_GAUGE: 615 if dtoMetric.Gauge == nil { 616 return fmt.Errorf( 617 "collected metric %s %s should be a Gauge", 618 desc.fqName, dtoMetric, 619 ) 620 } 621 case dto.MetricType_SUMMARY: 622 if dtoMetric.Summary == nil { 623 return fmt.Errorf( 624 "collected metric %s %s should be a Summary", 625 desc.fqName, dtoMetric, 626 ) 627 } 628 case dto.MetricType_UNTYPED: 629 if dtoMetric.Untyped == nil { 630 return fmt.Errorf( 631 "collected metric %s %s should be Untyped", 632 desc.fqName, dtoMetric, 633 ) 634 } 635 case dto.MetricType_HISTOGRAM: 636 if dtoMetric.Histogram == nil { 637 return fmt.Errorf( 638 "collected metric %s %s should be a Histogram", 639 desc.fqName, dtoMetric, 640 ) 641 } 642 default: 643 panic("encountered MetricFamily with invalid type") 644 } 645 } else { // New name. 646 metricFamily = &dto.MetricFamily{} 647 metricFamily.Name = proto.String(desc.fqName) 648 metricFamily.Help = proto.String(desc.help) 649 // TODO(beorn7): Simplify switch once Desc has type. 650 switch { 651 case dtoMetric.Gauge != nil: 652 metricFamily.Type = dto.MetricType_GAUGE.Enum() 653 case dtoMetric.Counter != nil: 654 metricFamily.Type = dto.MetricType_COUNTER.Enum() 655 case dtoMetric.Summary != nil: 656 metricFamily.Type = dto.MetricType_SUMMARY.Enum() 657 case dtoMetric.Untyped != nil: 658 metricFamily.Type = dto.MetricType_UNTYPED.Enum() 659 case dtoMetric.Histogram != nil: 660 metricFamily.Type = dto.MetricType_HISTOGRAM.Enum() 661 default: 662 return fmt.Errorf("empty metric collected: %s", dtoMetric) 663 } 664 if err := checkSuffixCollisions(metricFamily, metricFamiliesByName); err != nil { 665 return err 666 } 667 metricFamiliesByName[desc.fqName] = metricFamily 668 } 669 if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes); err != nil { 670 return err 671 } 672 if registeredDescIDs != nil { 673 // Is the desc registered at all? 674 if _, exist := registeredDescIDs[desc.id]; !exist { 675 return fmt.Errorf( 676 "collected metric %s %s with unregistered descriptor %s", 677 metricFamily.GetName(), dtoMetric, desc, 678 ) 679 } 680 if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil { 681 return err 682 } 683 } 684 metricFamily.Metric = append(metricFamily.Metric, dtoMetric) 685 return nil 686} 687 688// Gatherers is a slice of Gatherer instances that implements the Gatherer 689// interface itself. Its Gather method calls Gather on all Gatherers in the 690// slice in order and returns the merged results. Errors returned from the 691// Gather calls are all returned in a flattened MultiError. Duplicate and 692// inconsistent Metrics are skipped (first occurrence in slice order wins) and 693// reported in the returned error. 694// 695// Gatherers can be used to merge the Gather results from multiple 696// Registries. It also provides a way to directly inject existing MetricFamily 697// protobufs into the gathering by creating a custom Gatherer with a Gather 698// method that simply returns the existing MetricFamily protobufs. Note that no 699// registration is involved (in contrast to Collector registration), so 700// obviously registration-time checks cannot happen. Any inconsistencies between 701// the gathered MetricFamilies are reported as errors by the Gather method, and 702// inconsistent Metrics are dropped. Invalid parts of the MetricFamilies 703// (e.g. syntactically invalid metric or label names) will go undetected. 704type Gatherers []Gatherer 705 706// Gather implements Gatherer. 707func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) { 708 var ( 709 metricFamiliesByName = map[string]*dto.MetricFamily{} 710 metricHashes = map[uint64]struct{}{} 711 errs MultiError // The collected errors to return in the end. 712 ) 713 714 for i, g := range gs { 715 mfs, err := g.Gather() 716 if err != nil { 717 if multiErr, ok := err.(MultiError); ok { 718 for _, err := range multiErr { 719 errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) 720 } 721 } else { 722 errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) 723 } 724 } 725 for _, mf := range mfs { 726 existingMF, exists := metricFamiliesByName[mf.GetName()] 727 if exists { 728 if existingMF.GetHelp() != mf.GetHelp() { 729 errs = append(errs, fmt.Errorf( 730 "gathered metric family %s has help %q but should have %q", 731 mf.GetName(), mf.GetHelp(), existingMF.GetHelp(), 732 )) 733 continue 734 } 735 if existingMF.GetType() != mf.GetType() { 736 errs = append(errs, fmt.Errorf( 737 "gathered metric family %s has type %s but should have %s", 738 mf.GetName(), mf.GetType(), existingMF.GetType(), 739 )) 740 continue 741 } 742 } else { 743 existingMF = &dto.MetricFamily{} 744 existingMF.Name = mf.Name 745 existingMF.Help = mf.Help 746 existingMF.Type = mf.Type 747 if err := checkSuffixCollisions(existingMF, metricFamiliesByName); err != nil { 748 errs = append(errs, err) 749 continue 750 } 751 metricFamiliesByName[mf.GetName()] = existingMF 752 } 753 for _, m := range mf.Metric { 754 if err := checkMetricConsistency(existingMF, m, metricHashes); err != nil { 755 errs = append(errs, err) 756 continue 757 } 758 existingMF.Metric = append(existingMF.Metric, m) 759 } 760 } 761 } 762 return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() 763} 764 765// checkSuffixCollisions checks for collisions with the “magic” suffixes the 766// Prometheus text format and the internal metric representation of the 767// Prometheus server add while flattening Summaries and Histograms. 768func checkSuffixCollisions(mf *dto.MetricFamily, mfs map[string]*dto.MetricFamily) error { 769 var ( 770 newName = mf.GetName() 771 newType = mf.GetType() 772 newNameWithoutSuffix = "" 773 ) 774 switch { 775 case strings.HasSuffix(newName, "_count"): 776 newNameWithoutSuffix = newName[:len(newName)-6] 777 case strings.HasSuffix(newName, "_sum"): 778 newNameWithoutSuffix = newName[:len(newName)-4] 779 case strings.HasSuffix(newName, "_bucket"): 780 newNameWithoutSuffix = newName[:len(newName)-7] 781 } 782 if newNameWithoutSuffix != "" { 783 if existingMF, ok := mfs[newNameWithoutSuffix]; ok { 784 switch existingMF.GetType() { 785 case dto.MetricType_SUMMARY: 786 if !strings.HasSuffix(newName, "_bucket") { 787 return fmt.Errorf( 788 "collected metric named %q collides with previously collected summary named %q", 789 newName, newNameWithoutSuffix, 790 ) 791 } 792 case dto.MetricType_HISTOGRAM: 793 return fmt.Errorf( 794 "collected metric named %q collides with previously collected histogram named %q", 795 newName, newNameWithoutSuffix, 796 ) 797 } 798 } 799 } 800 if newType == dto.MetricType_SUMMARY || newType == dto.MetricType_HISTOGRAM { 801 if _, ok := mfs[newName+"_count"]; ok { 802 return fmt.Errorf( 803 "collected histogram or summary named %q collides with previously collected metric named %q", 804 newName, newName+"_count", 805 ) 806 } 807 if _, ok := mfs[newName+"_sum"]; ok { 808 return fmt.Errorf( 809 "collected histogram or summary named %q collides with previously collected metric named %q", 810 newName, newName+"_sum", 811 ) 812 } 813 } 814 if newType == dto.MetricType_HISTOGRAM { 815 if _, ok := mfs[newName+"_bucket"]; ok { 816 return fmt.Errorf( 817 "collected histogram named %q collides with previously collected metric named %q", 818 newName, newName+"_bucket", 819 ) 820 } 821 } 822 return nil 823} 824 825// checkMetricConsistency checks if the provided Metric is consistent with the 826// provided MetricFamily. It also hashes the Metric labels and the MetricFamily 827// name. If the resulting hash is already in the provided metricHashes, an error 828// is returned. If not, it is added to metricHashes. 829func checkMetricConsistency( 830 metricFamily *dto.MetricFamily, 831 dtoMetric *dto.Metric, 832 metricHashes map[uint64]struct{}, 833) error { 834 name := metricFamily.GetName() 835 836 // Type consistency with metric family. 837 if metricFamily.GetType() == dto.MetricType_GAUGE && dtoMetric.Gauge == nil || 838 metricFamily.GetType() == dto.MetricType_COUNTER && dtoMetric.Counter == nil || 839 metricFamily.GetType() == dto.MetricType_SUMMARY && dtoMetric.Summary == nil || 840 metricFamily.GetType() == dto.MetricType_HISTOGRAM && dtoMetric.Histogram == nil || 841 metricFamily.GetType() == dto.MetricType_UNTYPED && dtoMetric.Untyped == nil { 842 return fmt.Errorf( 843 "collected metric %q { %s} is not a %s", 844 name, dtoMetric, metricFamily.GetType(), 845 ) 846 } 847 848 previousLabelName := "" 849 for _, labelPair := range dtoMetric.GetLabel() { 850 labelName := labelPair.GetName() 851 if labelName == previousLabelName { 852 return fmt.Errorf( 853 "collected metric %q { %s} has two or more labels with the same name: %s", 854 name, dtoMetric, labelName, 855 ) 856 } 857 if !checkLabelName(labelName) { 858 return fmt.Errorf( 859 "collected metric %q { %s} has a label with an invalid name: %s", 860 name, dtoMetric, labelName, 861 ) 862 } 863 if dtoMetric.Summary != nil && labelName == quantileLabel { 864 return fmt.Errorf( 865 "collected metric %q { %s} must not have an explicit %q label", 866 name, dtoMetric, quantileLabel, 867 ) 868 } 869 if !utf8.ValidString(labelPair.GetValue()) { 870 return fmt.Errorf( 871 "collected metric %q { %s} has a label named %q whose value is not utf8: %#v", 872 name, dtoMetric, labelName, labelPair.GetValue()) 873 } 874 previousLabelName = labelName 875 } 876 877 // Is the metric unique (i.e. no other metric with the same name and the same labels)? 878 h := hashNew() 879 h = hashAdd(h, name) 880 h = hashAddByte(h, separatorByte) 881 // Make sure label pairs are sorted. We depend on it for the consistency 882 // check. 883 if !sort.IsSorted(labelPairSorter(dtoMetric.Label)) { 884 // We cannot sort dtoMetric.Label in place as it is immutable by contract. 885 copiedLabels := make([]*dto.LabelPair, len(dtoMetric.Label)) 886 copy(copiedLabels, dtoMetric.Label) 887 sort.Sort(labelPairSorter(copiedLabels)) 888 dtoMetric.Label = copiedLabels 889 } 890 for _, lp := range dtoMetric.Label { 891 h = hashAdd(h, lp.GetName()) 892 h = hashAddByte(h, separatorByte) 893 h = hashAdd(h, lp.GetValue()) 894 h = hashAddByte(h, separatorByte) 895 } 896 if _, exists := metricHashes[h]; exists { 897 return fmt.Errorf( 898 "collected metric %q { %s} was collected before with the same name and label values", 899 name, dtoMetric, 900 ) 901 } 902 metricHashes[h] = struct{}{} 903 return nil 904} 905 906func checkDescConsistency( 907 metricFamily *dto.MetricFamily, 908 dtoMetric *dto.Metric, 909 desc *Desc, 910) error { 911 // Desc help consistency with metric family help. 912 if metricFamily.GetHelp() != desc.help { 913 return fmt.Errorf( 914 "collected metric %s %s has help %q but should have %q", 915 metricFamily.GetName(), dtoMetric, metricFamily.GetHelp(), desc.help, 916 ) 917 } 918 919 // Is the desc consistent with the content of the metric? 920 lpsFromDesc := make([]*dto.LabelPair, len(desc.constLabelPairs), len(dtoMetric.Label)) 921 copy(lpsFromDesc, desc.constLabelPairs) 922 for _, l := range desc.variableLabels { 923 lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ 924 Name: proto.String(l), 925 }) 926 } 927 if len(lpsFromDesc) != len(dtoMetric.Label) { 928 return fmt.Errorf( 929 "labels in collected metric %s %s are inconsistent with descriptor %s", 930 metricFamily.GetName(), dtoMetric, desc, 931 ) 932 } 933 sort.Sort(labelPairSorter(lpsFromDesc)) 934 for i, lpFromDesc := range lpsFromDesc { 935 lpFromMetric := dtoMetric.Label[i] 936 if lpFromDesc.GetName() != lpFromMetric.GetName() || 937 lpFromDesc.Value != nil && lpFromDesc.GetValue() != lpFromMetric.GetValue() { 938 return fmt.Errorf( 939 "labels in collected metric %s %s are inconsistent with descriptor %s", 940 metricFamily.GetName(), dtoMetric, desc, 941 ) 942 } 943 } 944 return nil 945} 946