1// Copyright 2014 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package prometheus 15 16import ( 17 "bytes" 18 "fmt" 19 "io/ioutil" 20 "os" 21 "path/filepath" 22 "runtime" 23 "sort" 24 "strings" 25 "sync" 26 "unicode/utf8" 27 28 "github.com/cespare/xxhash/v2" 29 "github.com/golang/protobuf/proto" 30 "github.com/prometheus/common/expfmt" 31 32 dto "github.com/prometheus/client_model/go" 33 34 "github.com/prometheus/client_golang/prometheus/internal" 35) 36 37const ( 38 // Capacity for the channel to collect metrics and descriptors. 39 capMetricChan = 1000 40 capDescChan = 10 41) 42 43// DefaultRegisterer and DefaultGatherer are the implementations of the 44// Registerer and Gatherer interface a number of convenience functions in this 45// package act on. Initially, both variables point to the same Registry, which 46// has a process collector (currently on Linux only, see NewProcessCollector) 47// and a Go collector (see NewGoCollector, in particular the note about 48// stop-the-world implication with Go versions older than 1.9) already 49// registered. This approach to keep default instances as global state mirrors 50// the approach of other packages in the Go standard library. Note that there 51// are caveats. Change the variables with caution and only if you understand the 52// consequences. Users who want to avoid global state altogether should not use 53// the convenience functions and act on custom instances instead. 54var ( 55 defaultRegistry = NewRegistry() 56 DefaultRegisterer Registerer = defaultRegistry 57 DefaultGatherer Gatherer = defaultRegistry 58) 59 60func init() { 61 MustRegister(NewProcessCollector(ProcessCollectorOpts{})) 62 MustRegister(NewGoCollector()) 63} 64 65// NewRegistry creates a new vanilla Registry without any Collectors 66// pre-registered. 67func NewRegistry() *Registry { 68 return &Registry{ 69 collectorsByID: map[uint64]Collector{}, 70 descIDs: map[uint64]struct{}{}, 71 dimHashesByName: map[string]uint64{}, 72 } 73} 74 75// NewPedanticRegistry returns a registry that checks during collection if each 76// collected Metric is consistent with its reported Desc, and if the Desc has 77// actually been registered with the registry. Unchecked Collectors (those whose 78// Describe method does not yield any descriptors) are excluded from the check. 79// 80// Usually, a Registry will be happy as long as the union of all collected 81// Metrics is consistent and valid even if some metrics are not consistent with 82// their own Desc or a Desc provided by their registered Collector. Well-behaved 83// Collectors and Metrics will only provide consistent Descs. This Registry is 84// useful to test the implementation of Collectors and Metrics. 85func NewPedanticRegistry() *Registry { 86 r := NewRegistry() 87 r.pedanticChecksEnabled = true 88 return r 89} 90 91// Registerer is the interface for the part of a registry in charge of 92// registering and unregistering. Users of custom registries should use 93// Registerer as type for registration purposes (rather than the Registry type 94// directly). In that way, they are free to use custom Registerer implementation 95// (e.g. for testing purposes). 96type Registerer interface { 97 // Register registers a new Collector to be included in metrics 98 // collection. It returns an error if the descriptors provided by the 99 // Collector are invalid or if they — in combination with descriptors of 100 // already registered Collectors — do not fulfill the consistency and 101 // uniqueness criteria described in the documentation of metric.Desc. 102 // 103 // If the provided Collector is equal to a Collector already registered 104 // (which includes the case of re-registering the same Collector), the 105 // returned error is an instance of AlreadyRegisteredError, which 106 // contains the previously registered Collector. 107 // 108 // A Collector whose Describe method does not yield any Desc is treated 109 // as unchecked. Registration will always succeed. No check for 110 // re-registering (see previous paragraph) is performed. Thus, the 111 // caller is responsible for not double-registering the same unchecked 112 // Collector, and for providing a Collector that will not cause 113 // inconsistent metrics on collection. (This would lead to scrape 114 // errors.) 115 Register(Collector) error 116 // MustRegister works like Register but registers any number of 117 // Collectors and panics upon the first registration that causes an 118 // error. 119 MustRegister(...Collector) 120 // Unregister unregisters the Collector that equals the Collector passed 121 // in as an argument. (Two Collectors are considered equal if their 122 // Describe method yields the same set of descriptors.) The function 123 // returns whether a Collector was unregistered. Note that an unchecked 124 // Collector cannot be unregistered (as its Describe method does not 125 // yield any descriptor). 126 // 127 // Note that even after unregistering, it will not be possible to 128 // register a new Collector that is inconsistent with the unregistered 129 // Collector, e.g. a Collector collecting metrics with the same name but 130 // a different help string. The rationale here is that the same registry 131 // instance must only collect consistent metrics throughout its 132 // lifetime. 133 Unregister(Collector) bool 134} 135 136// Gatherer is the interface for the part of a registry in charge of gathering 137// the collected metrics into a number of MetricFamilies. The Gatherer interface 138// comes with the same general implication as described for the Registerer 139// interface. 140type Gatherer interface { 141 // Gather calls the Collect method of the registered Collectors and then 142 // gathers the collected metrics into a lexicographically sorted slice 143 // of uniquely named MetricFamily protobufs. Gather ensures that the 144 // returned slice is valid and self-consistent so that it can be used 145 // for valid exposition. As an exception to the strict consistency 146 // requirements described for metric.Desc, Gather will tolerate 147 // different sets of label names for metrics of the same metric family. 148 // 149 // Even if an error occurs, Gather attempts to gather as many metrics as 150 // possible. Hence, if a non-nil error is returned, the returned 151 // MetricFamily slice could be nil (in case of a fatal error that 152 // prevented any meaningful metric collection) or contain a number of 153 // MetricFamily protobufs, some of which might be incomplete, and some 154 // might be missing altogether. The returned error (which might be a 155 // MultiError) explains the details. Note that this is mostly useful for 156 // debugging purposes. If the gathered protobufs are to be used for 157 // exposition in actual monitoring, it is almost always better to not 158 // expose an incomplete result and instead disregard the returned 159 // MetricFamily protobufs in case the returned error is non-nil. 160 Gather() ([]*dto.MetricFamily, error) 161} 162 163// Register registers the provided Collector with the DefaultRegisterer. 164// 165// Register is a shortcut for DefaultRegisterer.Register(c). See there for more 166// details. 167func Register(c Collector) error { 168 return DefaultRegisterer.Register(c) 169} 170 171// MustRegister registers the provided Collectors with the DefaultRegisterer and 172// panics if any error occurs. 173// 174// MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See 175// there for more details. 176func MustRegister(cs ...Collector) { 177 DefaultRegisterer.MustRegister(cs...) 178} 179 180// Unregister removes the registration of the provided Collector from the 181// DefaultRegisterer. 182// 183// Unregister is a shortcut for DefaultRegisterer.Unregister(c). See there for 184// more details. 185func Unregister(c Collector) bool { 186 return DefaultRegisterer.Unregister(c) 187} 188 189// GathererFunc turns a function into a Gatherer. 190type GathererFunc func() ([]*dto.MetricFamily, error) 191 192// Gather implements Gatherer. 193func (gf GathererFunc) Gather() ([]*dto.MetricFamily, error) { 194 return gf() 195} 196 197// AlreadyRegisteredError is returned by the Register method if the Collector to 198// be registered has already been registered before, or a different Collector 199// that collects the same metrics has been registered before. Registration fails 200// in that case, but you can detect from the kind of error what has 201// happened. The error contains fields for the existing Collector and the 202// (rejected) new Collector that equals the existing one. This can be used to 203// find out if an equal Collector has been registered before and switch over to 204// using the old one, as demonstrated in the example. 205type AlreadyRegisteredError struct { 206 ExistingCollector, NewCollector Collector 207} 208 209func (err AlreadyRegisteredError) Error() string { 210 return "duplicate metrics collector registration attempted" 211} 212 213// MultiError is a slice of errors implementing the error interface. It is used 214// by a Gatherer to report multiple errors during MetricFamily gathering. 215type MultiError []error 216 217func (errs MultiError) Error() string { 218 if len(errs) == 0 { 219 return "" 220 } 221 buf := &bytes.Buffer{} 222 fmt.Fprintf(buf, "%d error(s) occurred:", len(errs)) 223 for _, err := range errs { 224 fmt.Fprintf(buf, "\n* %s", err) 225 } 226 return buf.String() 227} 228 229// Append appends the provided error if it is not nil. 230func (errs *MultiError) Append(err error) { 231 if err != nil { 232 *errs = append(*errs, err) 233 } 234} 235 236// MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only 237// contained error as error if len(errs is 1). In all other cases, it returns 238// the MultiError directly. This is helpful for returning a MultiError in a way 239// that only uses the MultiError if needed. 240func (errs MultiError) MaybeUnwrap() error { 241 switch len(errs) { 242 case 0: 243 return nil 244 case 1: 245 return errs[0] 246 default: 247 return errs 248 } 249} 250 251// Registry registers Prometheus collectors, collects their metrics, and gathers 252// them into MetricFamilies for exposition. It implements both Registerer and 253// Gatherer. The zero value is not usable. Create instances with NewRegistry or 254// NewPedanticRegistry. 255type Registry struct { 256 mtx sync.RWMutex 257 collectorsByID map[uint64]Collector // ID is a hash of the descIDs. 258 descIDs map[uint64]struct{} 259 dimHashesByName map[string]uint64 260 uncheckedCollectors []Collector 261 pedanticChecksEnabled bool 262} 263 264// Register implements Registerer. 265func (r *Registry) Register(c Collector) error { 266 var ( 267 descChan = make(chan *Desc, capDescChan) 268 newDescIDs = map[uint64]struct{}{} 269 newDimHashesByName = map[string]uint64{} 270 collectorID uint64 // All desc IDs XOR'd together. 271 duplicateDescErr error 272 ) 273 go func() { 274 c.Describe(descChan) 275 close(descChan) 276 }() 277 r.mtx.Lock() 278 defer func() { 279 // Drain channel in case of premature return to not leak a goroutine. 280 for range descChan { 281 } 282 r.mtx.Unlock() 283 }() 284 // Conduct various tests... 285 for desc := range descChan { 286 287 // Is the descriptor valid at all? 288 if desc.err != nil { 289 return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err) 290 } 291 292 // Is the descID unique? 293 // (In other words: Is the fqName + constLabel combination unique?) 294 if _, exists := r.descIDs[desc.id]; exists { 295 duplicateDescErr = fmt.Errorf("descriptor %s already exists with the same fully-qualified name and const label values", desc) 296 } 297 // If it is not a duplicate desc in this collector, XOR it to 298 // the collectorID. (We allow duplicate descs within the same 299 // collector, but their existence must be a no-op.) 300 if _, exists := newDescIDs[desc.id]; !exists { 301 newDescIDs[desc.id] = struct{}{} 302 collectorID ^= desc.id 303 } 304 305 // Are all the label names and the help string consistent with 306 // previous descriptors of the same name? 307 // First check existing descriptors... 308 if dimHash, exists := r.dimHashesByName[desc.fqName]; exists { 309 if dimHash != desc.dimHash { 310 return fmt.Errorf("a previously registered descriptor with the same fully-qualified name as %s has different label names or a different help string", desc) 311 } 312 } else { 313 // ...then check the new descriptors already seen. 314 if dimHash, exists := newDimHashesByName[desc.fqName]; exists { 315 if dimHash != desc.dimHash { 316 return fmt.Errorf("descriptors reported by collector have inconsistent label names or help strings for the same fully-qualified name, offender is %s", desc) 317 } 318 } else { 319 newDimHashesByName[desc.fqName] = desc.dimHash 320 } 321 } 322 } 323 // A Collector yielding no Desc at all is considered unchecked. 324 if len(newDescIDs) == 0 { 325 r.uncheckedCollectors = append(r.uncheckedCollectors, c) 326 return nil 327 } 328 if existing, exists := r.collectorsByID[collectorID]; exists { 329 switch e := existing.(type) { 330 case *wrappingCollector: 331 return AlreadyRegisteredError{ 332 ExistingCollector: e.unwrapRecursively(), 333 NewCollector: c, 334 } 335 default: 336 return AlreadyRegisteredError{ 337 ExistingCollector: e, 338 NewCollector: c, 339 } 340 } 341 } 342 // If the collectorID is new, but at least one of the descs existed 343 // before, we are in trouble. 344 if duplicateDescErr != nil { 345 return duplicateDescErr 346 } 347 348 // Only after all tests have passed, actually register. 349 r.collectorsByID[collectorID] = c 350 for hash := range newDescIDs { 351 r.descIDs[hash] = struct{}{} 352 } 353 for name, dimHash := range newDimHashesByName { 354 r.dimHashesByName[name] = dimHash 355 } 356 return nil 357} 358 359// Unregister implements Registerer. 360func (r *Registry) Unregister(c Collector) bool { 361 var ( 362 descChan = make(chan *Desc, capDescChan) 363 descIDs = map[uint64]struct{}{} 364 collectorID uint64 // All desc IDs XOR'd together. 365 ) 366 go func() { 367 c.Describe(descChan) 368 close(descChan) 369 }() 370 for desc := range descChan { 371 if _, exists := descIDs[desc.id]; !exists { 372 collectorID ^= desc.id 373 descIDs[desc.id] = struct{}{} 374 } 375 } 376 377 r.mtx.RLock() 378 if _, exists := r.collectorsByID[collectorID]; !exists { 379 r.mtx.RUnlock() 380 return false 381 } 382 r.mtx.RUnlock() 383 384 r.mtx.Lock() 385 defer r.mtx.Unlock() 386 387 delete(r.collectorsByID, collectorID) 388 for id := range descIDs { 389 delete(r.descIDs, id) 390 } 391 // dimHashesByName is left untouched as those must be consistent 392 // throughout the lifetime of a program. 393 return true 394} 395 396// MustRegister implements Registerer. 397func (r *Registry) MustRegister(cs ...Collector) { 398 for _, c := range cs { 399 if err := r.Register(c); err != nil { 400 panic(err) 401 } 402 } 403} 404 405// Gather implements Gatherer. 406func (r *Registry) Gather() ([]*dto.MetricFamily, error) { 407 var ( 408 checkedMetricChan = make(chan Metric, capMetricChan) 409 uncheckedMetricChan = make(chan Metric, capMetricChan) 410 metricHashes = map[uint64]struct{}{} 411 wg sync.WaitGroup 412 errs MultiError // The collected errors to return in the end. 413 registeredDescIDs map[uint64]struct{} // Only used for pedantic checks 414 ) 415 416 r.mtx.RLock() 417 goroutineBudget := len(r.collectorsByID) + len(r.uncheckedCollectors) 418 metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName)) 419 checkedCollectors := make(chan Collector, len(r.collectorsByID)) 420 uncheckedCollectors := make(chan Collector, len(r.uncheckedCollectors)) 421 for _, collector := range r.collectorsByID { 422 checkedCollectors <- collector 423 } 424 for _, collector := range r.uncheckedCollectors { 425 uncheckedCollectors <- collector 426 } 427 // In case pedantic checks are enabled, we have to copy the map before 428 // giving up the RLock. 429 if r.pedanticChecksEnabled { 430 registeredDescIDs = make(map[uint64]struct{}, len(r.descIDs)) 431 for id := range r.descIDs { 432 registeredDescIDs[id] = struct{}{} 433 } 434 } 435 r.mtx.RUnlock() 436 437 wg.Add(goroutineBudget) 438 439 collectWorker := func() { 440 for { 441 select { 442 case collector := <-checkedCollectors: 443 collector.Collect(checkedMetricChan) 444 case collector := <-uncheckedCollectors: 445 collector.Collect(uncheckedMetricChan) 446 default: 447 return 448 } 449 wg.Done() 450 } 451 } 452 453 // Start the first worker now to make sure at least one is running. 454 go collectWorker() 455 goroutineBudget-- 456 457 // Close checkedMetricChan and uncheckedMetricChan once all collectors 458 // are collected. 459 go func() { 460 wg.Wait() 461 close(checkedMetricChan) 462 close(uncheckedMetricChan) 463 }() 464 465 // Drain checkedMetricChan and uncheckedMetricChan in case of premature return. 466 defer func() { 467 if checkedMetricChan != nil { 468 for range checkedMetricChan { 469 } 470 } 471 if uncheckedMetricChan != nil { 472 for range uncheckedMetricChan { 473 } 474 } 475 }() 476 477 // Copy the channel references so we can nil them out later to remove 478 // them from the select statements below. 479 cmc := checkedMetricChan 480 umc := uncheckedMetricChan 481 482 for { 483 select { 484 case metric, ok := <-cmc: 485 if !ok { 486 cmc = nil 487 break 488 } 489 errs.Append(processMetric( 490 metric, metricFamiliesByName, 491 metricHashes, 492 registeredDescIDs, 493 )) 494 case metric, ok := <-umc: 495 if !ok { 496 umc = nil 497 break 498 } 499 errs.Append(processMetric( 500 metric, metricFamiliesByName, 501 metricHashes, 502 nil, 503 )) 504 default: 505 if goroutineBudget <= 0 || len(checkedCollectors)+len(uncheckedCollectors) == 0 { 506 // All collectors are already being worked on or 507 // we have already as many goroutines started as 508 // there are collectors. Do the same as above, 509 // just without the default. 510 select { 511 case metric, ok := <-cmc: 512 if !ok { 513 cmc = nil 514 break 515 } 516 errs.Append(processMetric( 517 metric, metricFamiliesByName, 518 metricHashes, 519 registeredDescIDs, 520 )) 521 case metric, ok := <-umc: 522 if !ok { 523 umc = nil 524 break 525 } 526 errs.Append(processMetric( 527 metric, metricFamiliesByName, 528 metricHashes, 529 nil, 530 )) 531 } 532 break 533 } 534 // Start more workers. 535 go collectWorker() 536 goroutineBudget-- 537 runtime.Gosched() 538 } 539 // Once both checkedMetricChan and uncheckdMetricChan are closed 540 // and drained, the contraption above will nil out cmc and umc, 541 // and then we can leave the collect loop here. 542 if cmc == nil && umc == nil { 543 break 544 } 545 } 546 return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() 547} 548 549// WriteToTextfile calls Gather on the provided Gatherer, encodes the result in the 550// Prometheus text format, and writes it to a temporary file. Upon success, the 551// temporary file is renamed to the provided filename. 552// 553// This is intended for use with the textfile collector of the node exporter. 554// Note that the node exporter expects the filename to be suffixed with ".prom". 555func WriteToTextfile(filename string, g Gatherer) error { 556 tmp, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)) 557 if err != nil { 558 return err 559 } 560 defer os.Remove(tmp.Name()) 561 562 mfs, err := g.Gather() 563 if err != nil { 564 return err 565 } 566 for _, mf := range mfs { 567 if _, err := expfmt.MetricFamilyToText(tmp, mf); err != nil { 568 return err 569 } 570 } 571 if err := tmp.Close(); err != nil { 572 return err 573 } 574 575 if err := os.Chmod(tmp.Name(), 0644); err != nil { 576 return err 577 } 578 return os.Rename(tmp.Name(), filename) 579} 580 581// processMetric is an internal helper method only used by the Gather method. 582func processMetric( 583 metric Metric, 584 metricFamiliesByName map[string]*dto.MetricFamily, 585 metricHashes map[uint64]struct{}, 586 registeredDescIDs map[uint64]struct{}, 587) error { 588 desc := metric.Desc() 589 // Wrapped metrics collected by an unchecked Collector can have an 590 // invalid Desc. 591 if desc.err != nil { 592 return desc.err 593 } 594 dtoMetric := &dto.Metric{} 595 if err := metric.Write(dtoMetric); err != nil { 596 return fmt.Errorf("error collecting metric %v: %s", desc, err) 597 } 598 metricFamily, ok := metricFamiliesByName[desc.fqName] 599 if ok { // Existing name. 600 if metricFamily.GetHelp() != desc.help { 601 return fmt.Errorf( 602 "collected metric %s %s has help %q but should have %q", 603 desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(), 604 ) 605 } 606 // TODO(beorn7): Simplify switch once Desc has type. 607 switch metricFamily.GetType() { 608 case dto.MetricType_COUNTER: 609 if dtoMetric.Counter == nil { 610 return fmt.Errorf( 611 "collected metric %s %s should be a Counter", 612 desc.fqName, dtoMetric, 613 ) 614 } 615 case dto.MetricType_GAUGE: 616 if dtoMetric.Gauge == nil { 617 return fmt.Errorf( 618 "collected metric %s %s should be a Gauge", 619 desc.fqName, dtoMetric, 620 ) 621 } 622 case dto.MetricType_SUMMARY: 623 if dtoMetric.Summary == nil { 624 return fmt.Errorf( 625 "collected metric %s %s should be a Summary", 626 desc.fqName, dtoMetric, 627 ) 628 } 629 case dto.MetricType_UNTYPED: 630 if dtoMetric.Untyped == nil { 631 return fmt.Errorf( 632 "collected metric %s %s should be Untyped", 633 desc.fqName, dtoMetric, 634 ) 635 } 636 case dto.MetricType_HISTOGRAM: 637 if dtoMetric.Histogram == nil { 638 return fmt.Errorf( 639 "collected metric %s %s should be a Histogram", 640 desc.fqName, dtoMetric, 641 ) 642 } 643 default: 644 panic("encountered MetricFamily with invalid type") 645 } 646 } else { // New name. 647 metricFamily = &dto.MetricFamily{} 648 metricFamily.Name = proto.String(desc.fqName) 649 metricFamily.Help = proto.String(desc.help) 650 // TODO(beorn7): Simplify switch once Desc has type. 651 switch { 652 case dtoMetric.Gauge != nil: 653 metricFamily.Type = dto.MetricType_GAUGE.Enum() 654 case dtoMetric.Counter != nil: 655 metricFamily.Type = dto.MetricType_COUNTER.Enum() 656 case dtoMetric.Summary != nil: 657 metricFamily.Type = dto.MetricType_SUMMARY.Enum() 658 case dtoMetric.Untyped != nil: 659 metricFamily.Type = dto.MetricType_UNTYPED.Enum() 660 case dtoMetric.Histogram != nil: 661 metricFamily.Type = dto.MetricType_HISTOGRAM.Enum() 662 default: 663 return fmt.Errorf("empty metric collected: %s", dtoMetric) 664 } 665 if err := checkSuffixCollisions(metricFamily, metricFamiliesByName); err != nil { 666 return err 667 } 668 metricFamiliesByName[desc.fqName] = metricFamily 669 } 670 if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes); err != nil { 671 return err 672 } 673 if registeredDescIDs != nil { 674 // Is the desc registered at all? 675 if _, exist := registeredDescIDs[desc.id]; !exist { 676 return fmt.Errorf( 677 "collected metric %s %s with unregistered descriptor %s", 678 metricFamily.GetName(), dtoMetric, desc, 679 ) 680 } 681 if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil { 682 return err 683 } 684 } 685 metricFamily.Metric = append(metricFamily.Metric, dtoMetric) 686 return nil 687} 688 689// Gatherers is a slice of Gatherer instances that implements the Gatherer 690// interface itself. Its Gather method calls Gather on all Gatherers in the 691// slice in order and returns the merged results. Errors returned from the 692// Gather calls are all returned in a flattened MultiError. Duplicate and 693// inconsistent Metrics are skipped (first occurrence in slice order wins) and 694// reported in the returned error. 695// 696// Gatherers can be used to merge the Gather results from multiple 697// Registries. It also provides a way to directly inject existing MetricFamily 698// protobufs into the gathering by creating a custom Gatherer with a Gather 699// method that simply returns the existing MetricFamily protobufs. Note that no 700// registration is involved (in contrast to Collector registration), so 701// obviously registration-time checks cannot happen. Any inconsistencies between 702// the gathered MetricFamilies are reported as errors by the Gather method, and 703// inconsistent Metrics are dropped. Invalid parts of the MetricFamilies 704// (e.g. syntactically invalid metric or label names) will go undetected. 705type Gatherers []Gatherer 706 707// Gather implements Gatherer. 708func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) { 709 var ( 710 metricFamiliesByName = map[string]*dto.MetricFamily{} 711 metricHashes = map[uint64]struct{}{} 712 errs MultiError // The collected errors to return in the end. 713 ) 714 715 for i, g := range gs { 716 mfs, err := g.Gather() 717 if err != nil { 718 if multiErr, ok := err.(MultiError); ok { 719 for _, err := range multiErr { 720 errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) 721 } 722 } else { 723 errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) 724 } 725 } 726 for _, mf := range mfs { 727 existingMF, exists := metricFamiliesByName[mf.GetName()] 728 if exists { 729 if existingMF.GetHelp() != mf.GetHelp() { 730 errs = append(errs, fmt.Errorf( 731 "gathered metric family %s has help %q but should have %q", 732 mf.GetName(), mf.GetHelp(), existingMF.GetHelp(), 733 )) 734 continue 735 } 736 if existingMF.GetType() != mf.GetType() { 737 errs = append(errs, fmt.Errorf( 738 "gathered metric family %s has type %s but should have %s", 739 mf.GetName(), mf.GetType(), existingMF.GetType(), 740 )) 741 continue 742 } 743 } else { 744 existingMF = &dto.MetricFamily{} 745 existingMF.Name = mf.Name 746 existingMF.Help = mf.Help 747 existingMF.Type = mf.Type 748 if err := checkSuffixCollisions(existingMF, metricFamiliesByName); err != nil { 749 errs = append(errs, err) 750 continue 751 } 752 metricFamiliesByName[mf.GetName()] = existingMF 753 } 754 for _, m := range mf.Metric { 755 if err := checkMetricConsistency(existingMF, m, metricHashes); err != nil { 756 errs = append(errs, err) 757 continue 758 } 759 existingMF.Metric = append(existingMF.Metric, m) 760 } 761 } 762 } 763 return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() 764} 765 766// checkSuffixCollisions checks for collisions with the “magic” suffixes the 767// Prometheus text format and the internal metric representation of the 768// Prometheus server add while flattening Summaries and Histograms. 769func checkSuffixCollisions(mf *dto.MetricFamily, mfs map[string]*dto.MetricFamily) error { 770 var ( 771 newName = mf.GetName() 772 newType = mf.GetType() 773 newNameWithoutSuffix = "" 774 ) 775 switch { 776 case strings.HasSuffix(newName, "_count"): 777 newNameWithoutSuffix = newName[:len(newName)-6] 778 case strings.HasSuffix(newName, "_sum"): 779 newNameWithoutSuffix = newName[:len(newName)-4] 780 case strings.HasSuffix(newName, "_bucket"): 781 newNameWithoutSuffix = newName[:len(newName)-7] 782 } 783 if newNameWithoutSuffix != "" { 784 if existingMF, ok := mfs[newNameWithoutSuffix]; ok { 785 switch existingMF.GetType() { 786 case dto.MetricType_SUMMARY: 787 if !strings.HasSuffix(newName, "_bucket") { 788 return fmt.Errorf( 789 "collected metric named %q collides with previously collected summary named %q", 790 newName, newNameWithoutSuffix, 791 ) 792 } 793 case dto.MetricType_HISTOGRAM: 794 return fmt.Errorf( 795 "collected metric named %q collides with previously collected histogram named %q", 796 newName, newNameWithoutSuffix, 797 ) 798 } 799 } 800 } 801 if newType == dto.MetricType_SUMMARY || newType == dto.MetricType_HISTOGRAM { 802 if _, ok := mfs[newName+"_count"]; ok { 803 return fmt.Errorf( 804 "collected histogram or summary named %q collides with previously collected metric named %q", 805 newName, newName+"_count", 806 ) 807 } 808 if _, ok := mfs[newName+"_sum"]; ok { 809 return fmt.Errorf( 810 "collected histogram or summary named %q collides with previously collected metric named %q", 811 newName, newName+"_sum", 812 ) 813 } 814 } 815 if newType == dto.MetricType_HISTOGRAM { 816 if _, ok := mfs[newName+"_bucket"]; ok { 817 return fmt.Errorf( 818 "collected histogram named %q collides with previously collected metric named %q", 819 newName, newName+"_bucket", 820 ) 821 } 822 } 823 return nil 824} 825 826// checkMetricConsistency checks if the provided Metric is consistent with the 827// provided MetricFamily. It also hashes the Metric labels and the MetricFamily 828// name. If the resulting hash is already in the provided metricHashes, an error 829// is returned. If not, it is added to metricHashes. 830func checkMetricConsistency( 831 metricFamily *dto.MetricFamily, 832 dtoMetric *dto.Metric, 833 metricHashes map[uint64]struct{}, 834) error { 835 name := metricFamily.GetName() 836 837 // Type consistency with metric family. 838 if metricFamily.GetType() == dto.MetricType_GAUGE && dtoMetric.Gauge == nil || 839 metricFamily.GetType() == dto.MetricType_COUNTER && dtoMetric.Counter == nil || 840 metricFamily.GetType() == dto.MetricType_SUMMARY && dtoMetric.Summary == nil || 841 metricFamily.GetType() == dto.MetricType_HISTOGRAM && dtoMetric.Histogram == nil || 842 metricFamily.GetType() == dto.MetricType_UNTYPED && dtoMetric.Untyped == nil { 843 return fmt.Errorf( 844 "collected metric %q { %s} is not a %s", 845 name, dtoMetric, metricFamily.GetType(), 846 ) 847 } 848 849 previousLabelName := "" 850 for _, labelPair := range dtoMetric.GetLabel() { 851 labelName := labelPair.GetName() 852 if labelName == previousLabelName { 853 return fmt.Errorf( 854 "collected metric %q { %s} has two or more labels with the same name: %s", 855 name, dtoMetric, labelName, 856 ) 857 } 858 if !checkLabelName(labelName) { 859 return fmt.Errorf( 860 "collected metric %q { %s} has a label with an invalid name: %s", 861 name, dtoMetric, labelName, 862 ) 863 } 864 if dtoMetric.Summary != nil && labelName == quantileLabel { 865 return fmt.Errorf( 866 "collected metric %q { %s} must not have an explicit %q label", 867 name, dtoMetric, quantileLabel, 868 ) 869 } 870 if !utf8.ValidString(labelPair.GetValue()) { 871 return fmt.Errorf( 872 "collected metric %q { %s} has a label named %q whose value is not utf8: %#v", 873 name, dtoMetric, labelName, labelPair.GetValue()) 874 } 875 previousLabelName = labelName 876 } 877 878 // Is the metric unique (i.e. no other metric with the same name and the same labels)? 879 h := xxhash.New() 880 h.WriteString(name) 881 h.Write(separatorByteSlice) 882 // Make sure label pairs are sorted. We depend on it for the consistency 883 // check. 884 if !sort.IsSorted(labelPairSorter(dtoMetric.Label)) { 885 // We cannot sort dtoMetric.Label in place as it is immutable by contract. 886 copiedLabels := make([]*dto.LabelPair, len(dtoMetric.Label)) 887 copy(copiedLabels, dtoMetric.Label) 888 sort.Sort(labelPairSorter(copiedLabels)) 889 dtoMetric.Label = copiedLabels 890 } 891 for _, lp := range dtoMetric.Label { 892 h.WriteString(lp.GetName()) 893 h.Write(separatorByteSlice) 894 h.WriteString(lp.GetValue()) 895 h.Write(separatorByteSlice) 896 } 897 hSum := h.Sum64() 898 if _, exists := metricHashes[hSum]; exists { 899 return fmt.Errorf( 900 "collected metric %q { %s} was collected before with the same name and label values", 901 name, dtoMetric, 902 ) 903 } 904 metricHashes[hSum] = struct{}{} 905 return nil 906} 907 908func checkDescConsistency( 909 metricFamily *dto.MetricFamily, 910 dtoMetric *dto.Metric, 911 desc *Desc, 912) error { 913 // Desc help consistency with metric family help. 914 if metricFamily.GetHelp() != desc.help { 915 return fmt.Errorf( 916 "collected metric %s %s has help %q but should have %q", 917 metricFamily.GetName(), dtoMetric, metricFamily.GetHelp(), desc.help, 918 ) 919 } 920 921 // Is the desc consistent with the content of the metric? 922 lpsFromDesc := make([]*dto.LabelPair, len(desc.constLabelPairs), len(dtoMetric.Label)) 923 copy(lpsFromDesc, desc.constLabelPairs) 924 for _, l := range desc.variableLabels { 925 lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ 926 Name: proto.String(l), 927 }) 928 } 929 if len(lpsFromDesc) != len(dtoMetric.Label) { 930 return fmt.Errorf( 931 "labels in collected metric %s %s are inconsistent with descriptor %s", 932 metricFamily.GetName(), dtoMetric, desc, 933 ) 934 } 935 sort.Sort(labelPairSorter(lpsFromDesc)) 936 for i, lpFromDesc := range lpsFromDesc { 937 lpFromMetric := dtoMetric.Label[i] 938 if lpFromDesc.GetName() != lpFromMetric.GetName() || 939 lpFromDesc.Value != nil && lpFromDesc.GetValue() != lpFromMetric.GetValue() { 940 return fmt.Errorf( 941 "labels in collected metric %s %s are inconsistent with descriptor %s", 942 metricFamily.GetName(), dtoMetric, desc, 943 ) 944 } 945 } 946 return nil 947} 948