1// Copyright 2014 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14package prometheus 15 16import ( 17 "bytes" 18 "fmt" 19 "io/ioutil" 20 "os" 21 "path/filepath" 22 "runtime" 23 "sort" 24 "strings" 25 "sync" 26 "unicode/utf8" 27 28 "github.com/golang/protobuf/proto" 29 "github.com/prometheus/common/expfmt" 30 31 dto "github.com/prometheus/client_model/go" 32 33 "github.com/prometheus/client_golang/prometheus/internal" 34) 35 36const ( 37 // Capacity for the channel to collect metrics and descriptors. 38 capMetricChan = 1000 39 capDescChan = 10 40) 41 42// DefaultRegisterer and DefaultGatherer are the implementations of the 43// Registerer and Gatherer interface a number of convenience functions in this 44// package act on. Initially, both variables point to the same Registry, which 45// has a process collector (currently on Linux only, see NewProcessCollector) 46// and a Go collector (see NewGoCollector, in particular the note about 47// stop-the-world implication with Go versions older than 1.9) already 48// registered. This approach to keep default instances as global state mirrors 49// the approach of other packages in the Go standard library. Note that there 50// are caveats. Change the variables with caution and only if you understand the 51// consequences. Users who want to avoid global state altogether should not use 52// the convenience functions and act on custom instances instead. 53var ( 54 defaultRegistry = NewRegistry() 55 DefaultRegisterer Registerer = defaultRegistry 56 DefaultGatherer Gatherer = defaultRegistry 57) 58 59func init() { 60 MustRegister(NewProcessCollector(ProcessCollectorOpts{})) 61 MustRegister(NewGoCollector()) 62} 63 64// NewRegistry creates a new vanilla Registry without any Collectors 65// pre-registered. 66func NewRegistry() *Registry { 67 return &Registry{ 68 collectorsByID: map[uint64]Collector{}, 69 descIDs: map[uint64]struct{}{}, 70 dimHashesByName: map[string]uint64{}, 71 } 72} 73 74// NewPedanticRegistry returns a registry that checks during collection if each 75// collected Metric is consistent with its reported Desc, and if the Desc has 76// actually been registered with the registry. Unchecked Collectors (those whose 77// Describe methed does not yield any descriptors) are excluded from the check. 78// 79// Usually, a Registry will be happy as long as the union of all collected 80// Metrics is consistent and valid even if some metrics are not consistent with 81// their own Desc or a Desc provided by their registered Collector. Well-behaved 82// Collectors and Metrics will only provide consistent Descs. This Registry is 83// useful to test the implementation of Collectors and Metrics. 84func NewPedanticRegistry() *Registry { 85 r := NewRegistry() 86 r.pedanticChecksEnabled = true 87 return r 88} 89 90// Registerer is the interface for the part of a registry in charge of 91// registering and unregistering. Users of custom registries should use 92// Registerer as type for registration purposes (rather than the Registry type 93// directly). In that way, they are free to use custom Registerer implementation 94// (e.g. for testing purposes). 95type Registerer interface { 96 // Register registers a new Collector to be included in metrics 97 // collection. It returns an error if the descriptors provided by the 98 // Collector are invalid or if they — in combination with descriptors of 99 // already registered Collectors — do not fulfill the consistency and 100 // uniqueness criteria described in the documentation of metric.Desc. 101 // 102 // If the provided Collector is equal to a Collector already registered 103 // (which includes the case of re-registering the same Collector), the 104 // returned error is an instance of AlreadyRegisteredError, which 105 // contains the previously registered Collector. 106 // 107 // A Collector whose Describe method does not yield any Desc is treated 108 // as unchecked. Registration will always succeed. No check for 109 // re-registering (see previous paragraph) is performed. Thus, the 110 // caller is responsible for not double-registering the same unchecked 111 // Collector, and for providing a Collector that will not cause 112 // inconsistent metrics on collection. (This would lead to scrape 113 // errors.) 114 Register(Collector) error 115 // MustRegister works like Register but registers any number of 116 // Collectors and panics upon the first registration that causes an 117 // error. 118 MustRegister(...Collector) 119 // Unregister unregisters the Collector that equals the Collector passed 120 // in as an argument. (Two Collectors are considered equal if their 121 // Describe method yields the same set of descriptors.) The function 122 // returns whether a Collector was unregistered. Note that an unchecked 123 // Collector cannot be unregistered (as its Describe method does not 124 // yield any descriptor). 125 // 126 // Note that even after unregistering, it will not be possible to 127 // register a new Collector that is inconsistent with the unregistered 128 // Collector, e.g. a Collector collecting metrics with the same name but 129 // a different help string. The rationale here is that the same registry 130 // instance must only collect consistent metrics throughout its 131 // lifetime. 132 Unregister(Collector) bool 133} 134 135// Gatherer is the interface for the part of a registry in charge of gathering 136// the collected metrics into a number of MetricFamilies. The Gatherer interface 137// comes with the same general implication as described for the Registerer 138// interface. 139type Gatherer interface { 140 // Gather calls the Collect method of the registered Collectors and then 141 // gathers the collected metrics into a lexicographically sorted slice 142 // of uniquely named MetricFamily protobufs. Gather ensures that the 143 // returned slice is valid and self-consistent so that it can be used 144 // for valid exposition. As an exception to the strict consistency 145 // requirements described for metric.Desc, Gather will tolerate 146 // different sets of label names for metrics of the same metric family. 147 // 148 // Even if an error occurs, Gather attempts to gather as many metrics as 149 // possible. Hence, if a non-nil error is returned, the returned 150 // MetricFamily slice could be nil (in case of a fatal error that 151 // prevented any meaningful metric collection) or contain a number of 152 // MetricFamily protobufs, some of which might be incomplete, and some 153 // might be missing altogether. The returned error (which might be a 154 // MultiError) explains the details. Note that this is mostly useful for 155 // debugging purposes. If the gathered protobufs are to be used for 156 // exposition in actual monitoring, it is almost always better to not 157 // expose an incomplete result and instead disregard the returned 158 // MetricFamily protobufs in case the returned error is non-nil. 159 Gather() ([]*dto.MetricFamily, error) 160} 161 162// Register registers the provided Collector with the DefaultRegisterer. 163// 164// Register is a shortcut for DefaultRegisterer.Register(c). See there for more 165// details. 166func Register(c Collector) error { 167 return DefaultRegisterer.Register(c) 168} 169 170// MustRegister registers the provided Collectors with the DefaultRegisterer and 171// panics if any error occurs. 172// 173// MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See 174// there for more details. 175func MustRegister(cs ...Collector) { 176 DefaultRegisterer.MustRegister(cs...) 177} 178 179// Unregister removes the registration of the provided Collector from the 180// DefaultRegisterer. 181// 182// Unregister is a shortcut for DefaultRegisterer.Unregister(c). See there for 183// more details. 184func Unregister(c Collector) bool { 185 return DefaultRegisterer.Unregister(c) 186} 187 188// GathererFunc turns a function into a Gatherer. 189type GathererFunc func() ([]*dto.MetricFamily, error) 190 191// Gather implements Gatherer. 192func (gf GathererFunc) Gather() ([]*dto.MetricFamily, error) { 193 return gf() 194} 195 196// AlreadyRegisteredError is returned by the Register method if the Collector to 197// be registered has already been registered before, or a different Collector 198// that collects the same metrics has been registered before. Registration fails 199// in that case, but you can detect from the kind of error what has 200// happened. The error contains fields for the existing Collector and the 201// (rejected) new Collector that equals the existing one. This can be used to 202// find out if an equal Collector has been registered before and switch over to 203// using the old one, as demonstrated in the example. 204type AlreadyRegisteredError struct { 205 ExistingCollector, NewCollector Collector 206} 207 208func (err AlreadyRegisteredError) Error() string { 209 return "duplicate metrics collector registration attempted" 210} 211 212// MultiError is a slice of errors implementing the error interface. It is used 213// by a Gatherer to report multiple errors during MetricFamily gathering. 214type MultiError []error 215 216func (errs MultiError) Error() string { 217 if len(errs) == 0 { 218 return "" 219 } 220 buf := &bytes.Buffer{} 221 fmt.Fprintf(buf, "%d error(s) occurred:", len(errs)) 222 for _, err := range errs { 223 fmt.Fprintf(buf, "\n* %s", err) 224 } 225 return buf.String() 226} 227 228// Append appends the provided error if it is not nil. 229func (errs *MultiError) Append(err error) { 230 if err != nil { 231 *errs = append(*errs, err) 232 } 233} 234 235// MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only 236// contained error as error if len(errs is 1). In all other cases, it returns 237// the MultiError directly. This is helpful for returning a MultiError in a way 238// that only uses the MultiError if needed. 239func (errs MultiError) MaybeUnwrap() error { 240 switch len(errs) { 241 case 0: 242 return nil 243 case 1: 244 return errs[0] 245 default: 246 return errs 247 } 248} 249 250// Registry registers Prometheus collectors, collects their metrics, and gathers 251// them into MetricFamilies for exposition. It implements both Registerer and 252// Gatherer. The zero value is not usable. Create instances with NewRegistry or 253// NewPedanticRegistry. 254type Registry struct { 255 mtx sync.RWMutex 256 collectorsByID map[uint64]Collector // ID is a hash of the descIDs. 257 descIDs map[uint64]struct{} 258 dimHashesByName map[string]uint64 259 uncheckedCollectors []Collector 260 pedanticChecksEnabled bool 261} 262 263// Register implements Registerer. 264func (r *Registry) Register(c Collector) error { 265 var ( 266 descChan = make(chan *Desc, capDescChan) 267 newDescIDs = map[uint64]struct{}{} 268 newDimHashesByName = map[string]uint64{} 269 collectorID uint64 // Just a sum of all desc IDs. 270 duplicateDescErr error 271 ) 272 go func() { 273 c.Describe(descChan) 274 close(descChan) 275 }() 276 r.mtx.Lock() 277 defer func() { 278 // Drain channel in case of premature return to not leak a goroutine. 279 for range descChan { 280 } 281 r.mtx.Unlock() 282 }() 283 // Conduct various tests... 284 for desc := range descChan { 285 286 // Is the descriptor valid at all? 287 if desc.err != nil { 288 return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err) 289 } 290 291 // Is the descID unique? 292 // (In other words: Is the fqName + constLabel combination unique?) 293 if _, exists := r.descIDs[desc.id]; exists { 294 duplicateDescErr = fmt.Errorf("descriptor %s already exists with the same fully-qualified name and const label values", desc) 295 } 296 // If it is not a duplicate desc in this collector, add it to 297 // the collectorID. (We allow duplicate descs within the same 298 // collector, but their existence must be a no-op.) 299 if _, exists := newDescIDs[desc.id]; !exists { 300 newDescIDs[desc.id] = struct{}{} 301 collectorID += desc.id 302 } 303 304 // Are all the label names and the help string consistent with 305 // previous descriptors of the same name? 306 // First check existing descriptors... 307 if dimHash, exists := r.dimHashesByName[desc.fqName]; exists { 308 if dimHash != desc.dimHash { 309 return fmt.Errorf("a previously registered descriptor with the same fully-qualified name as %s has different label names or a different help string", desc) 310 } 311 } else { 312 // ...then check the new descriptors already seen. 313 if dimHash, exists := newDimHashesByName[desc.fqName]; exists { 314 if dimHash != desc.dimHash { 315 return fmt.Errorf("descriptors reported by collector have inconsistent label names or help strings for the same fully-qualified name, offender is %s", desc) 316 } 317 } else { 318 newDimHashesByName[desc.fqName] = desc.dimHash 319 } 320 } 321 } 322 // A Collector yielding no Desc at all is considered unchecked. 323 if len(newDescIDs) == 0 { 324 r.uncheckedCollectors = append(r.uncheckedCollectors, c) 325 return nil 326 } 327 if existing, exists := r.collectorsByID[collectorID]; exists { 328 return AlreadyRegisteredError{ 329 ExistingCollector: existing, 330 NewCollector: c, 331 } 332 } 333 // If the collectorID is new, but at least one of the descs existed 334 // before, we are in trouble. 335 if duplicateDescErr != nil { 336 return duplicateDescErr 337 } 338 339 // Only after all tests have passed, actually register. 340 r.collectorsByID[collectorID] = c 341 for hash := range newDescIDs { 342 r.descIDs[hash] = struct{}{} 343 } 344 for name, dimHash := range newDimHashesByName { 345 r.dimHashesByName[name] = dimHash 346 } 347 return nil 348} 349 350// Unregister implements Registerer. 351func (r *Registry) Unregister(c Collector) bool { 352 var ( 353 descChan = make(chan *Desc, capDescChan) 354 descIDs = map[uint64]struct{}{} 355 collectorID uint64 // Just a sum of the desc IDs. 356 ) 357 go func() { 358 c.Describe(descChan) 359 close(descChan) 360 }() 361 for desc := range descChan { 362 if _, exists := descIDs[desc.id]; !exists { 363 collectorID += desc.id 364 descIDs[desc.id] = struct{}{} 365 } 366 } 367 368 r.mtx.RLock() 369 if _, exists := r.collectorsByID[collectorID]; !exists { 370 r.mtx.RUnlock() 371 return false 372 } 373 r.mtx.RUnlock() 374 375 r.mtx.Lock() 376 defer r.mtx.Unlock() 377 378 delete(r.collectorsByID, collectorID) 379 for id := range descIDs { 380 delete(r.descIDs, id) 381 } 382 // dimHashesByName is left untouched as those must be consistent 383 // throughout the lifetime of a program. 384 return true 385} 386 387// MustRegister implements Registerer. 388func (r *Registry) MustRegister(cs ...Collector) { 389 for _, c := range cs { 390 if err := r.Register(c); err != nil { 391 panic(err) 392 } 393 } 394} 395 396// Gather implements Gatherer. 397func (r *Registry) Gather() ([]*dto.MetricFamily, error) { 398 var ( 399 checkedMetricChan = make(chan Metric, capMetricChan) 400 uncheckedMetricChan = make(chan Metric, capMetricChan) 401 metricHashes = map[uint64]struct{}{} 402 wg sync.WaitGroup 403 errs MultiError // The collected errors to return in the end. 404 registeredDescIDs map[uint64]struct{} // Only used for pedantic checks 405 ) 406 407 r.mtx.RLock() 408 goroutineBudget := len(r.collectorsByID) + len(r.uncheckedCollectors) 409 metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName)) 410 checkedCollectors := make(chan Collector, len(r.collectorsByID)) 411 uncheckedCollectors := make(chan Collector, len(r.uncheckedCollectors)) 412 for _, collector := range r.collectorsByID { 413 checkedCollectors <- collector 414 } 415 for _, collector := range r.uncheckedCollectors { 416 uncheckedCollectors <- collector 417 } 418 // In case pedantic checks are enabled, we have to copy the map before 419 // giving up the RLock. 420 if r.pedanticChecksEnabled { 421 registeredDescIDs = make(map[uint64]struct{}, len(r.descIDs)) 422 for id := range r.descIDs { 423 registeredDescIDs[id] = struct{}{} 424 } 425 } 426 r.mtx.RUnlock() 427 428 wg.Add(goroutineBudget) 429 430 collectWorker := func() { 431 for { 432 select { 433 case collector := <-checkedCollectors: 434 collector.Collect(checkedMetricChan) 435 case collector := <-uncheckedCollectors: 436 collector.Collect(uncheckedMetricChan) 437 default: 438 return 439 } 440 wg.Done() 441 } 442 } 443 444 // Start the first worker now to make sure at least one is running. 445 go collectWorker() 446 goroutineBudget-- 447 448 // Close checkedMetricChan and uncheckedMetricChan once all collectors 449 // are collected. 450 go func() { 451 wg.Wait() 452 close(checkedMetricChan) 453 close(uncheckedMetricChan) 454 }() 455 456 // Drain checkedMetricChan and uncheckedMetricChan in case of premature return. 457 defer func() { 458 if checkedMetricChan != nil { 459 for range checkedMetricChan { 460 } 461 } 462 if uncheckedMetricChan != nil { 463 for range uncheckedMetricChan { 464 } 465 } 466 }() 467 468 // Copy the channel references so we can nil them out later to remove 469 // them from the select statements below. 470 cmc := checkedMetricChan 471 umc := uncheckedMetricChan 472 473 for { 474 select { 475 case metric, ok := <-cmc: 476 if !ok { 477 cmc = nil 478 break 479 } 480 errs.Append(processMetric( 481 metric, metricFamiliesByName, 482 metricHashes, 483 registeredDescIDs, 484 )) 485 case metric, ok := <-umc: 486 if !ok { 487 umc = nil 488 break 489 } 490 errs.Append(processMetric( 491 metric, metricFamiliesByName, 492 metricHashes, 493 nil, 494 )) 495 default: 496 if goroutineBudget <= 0 || len(checkedCollectors)+len(uncheckedCollectors) == 0 { 497 // All collectors are already being worked on or 498 // we have already as many goroutines started as 499 // there are collectors. Do the same as above, 500 // just without the default. 501 select { 502 case metric, ok := <-cmc: 503 if !ok { 504 cmc = nil 505 break 506 } 507 errs.Append(processMetric( 508 metric, metricFamiliesByName, 509 metricHashes, 510 registeredDescIDs, 511 )) 512 case metric, ok := <-umc: 513 if !ok { 514 umc = nil 515 break 516 } 517 errs.Append(processMetric( 518 metric, metricFamiliesByName, 519 metricHashes, 520 nil, 521 )) 522 } 523 break 524 } 525 // Start more workers. 526 go collectWorker() 527 goroutineBudget-- 528 runtime.Gosched() 529 } 530 // Once both checkedMetricChan and uncheckdMetricChan are closed 531 // and drained, the contraption above will nil out cmc and umc, 532 // and then we can leave the collect loop here. 533 if cmc == nil && umc == nil { 534 break 535 } 536 } 537 return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() 538} 539 540// WriteToTextfile calls Gather on the provided Gatherer, encodes the result in the 541// Prometheus text format, and writes it to a temporary file. Upon success, the 542// temporary file is renamed to the provided filename. 543// 544// This is intended for use with the textfile collector of the node exporter. 545// Note that the node exporter expects the filename to be suffixed with ".prom". 546func WriteToTextfile(filename string, g Gatherer) error { 547 tmp, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)) 548 if err != nil { 549 return err 550 } 551 defer os.Remove(tmp.Name()) 552 553 mfs, err := g.Gather() 554 if err != nil { 555 return err 556 } 557 for _, mf := range mfs { 558 if _, err := expfmt.MetricFamilyToText(tmp, mf); err != nil { 559 return err 560 } 561 } 562 if err := tmp.Close(); err != nil { 563 return err 564 } 565 566 if err := os.Chmod(tmp.Name(), 0644); err != nil { 567 return err 568 } 569 return os.Rename(tmp.Name(), filename) 570} 571 572// processMetric is an internal helper method only used by the Gather method. 573func processMetric( 574 metric Metric, 575 metricFamiliesByName map[string]*dto.MetricFamily, 576 metricHashes map[uint64]struct{}, 577 registeredDescIDs map[uint64]struct{}, 578) error { 579 desc := metric.Desc() 580 // Wrapped metrics collected by an unchecked Collector can have an 581 // invalid Desc. 582 if desc.err != nil { 583 return desc.err 584 } 585 dtoMetric := &dto.Metric{} 586 if err := metric.Write(dtoMetric); err != nil { 587 return fmt.Errorf("error collecting metric %v: %s", desc, err) 588 } 589 metricFamily, ok := metricFamiliesByName[desc.fqName] 590 if ok { // Existing name. 591 if metricFamily.GetHelp() != desc.help { 592 return fmt.Errorf( 593 "collected metric %s %s has help %q but should have %q", 594 desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(), 595 ) 596 } 597 // TODO(beorn7): Simplify switch once Desc has type. 598 switch metricFamily.GetType() { 599 case dto.MetricType_COUNTER: 600 if dtoMetric.Counter == nil { 601 return fmt.Errorf( 602 "collected metric %s %s should be a Counter", 603 desc.fqName, dtoMetric, 604 ) 605 } 606 case dto.MetricType_GAUGE: 607 if dtoMetric.Gauge == nil { 608 return fmt.Errorf( 609 "collected metric %s %s should be a Gauge", 610 desc.fqName, dtoMetric, 611 ) 612 } 613 case dto.MetricType_SUMMARY: 614 if dtoMetric.Summary == nil { 615 return fmt.Errorf( 616 "collected metric %s %s should be a Summary", 617 desc.fqName, dtoMetric, 618 ) 619 } 620 case dto.MetricType_UNTYPED: 621 if dtoMetric.Untyped == nil { 622 return fmt.Errorf( 623 "collected metric %s %s should be Untyped", 624 desc.fqName, dtoMetric, 625 ) 626 } 627 case dto.MetricType_HISTOGRAM: 628 if dtoMetric.Histogram == nil { 629 return fmt.Errorf( 630 "collected metric %s %s should be a Histogram", 631 desc.fqName, dtoMetric, 632 ) 633 } 634 default: 635 panic("encountered MetricFamily with invalid type") 636 } 637 } else { // New name. 638 metricFamily = &dto.MetricFamily{} 639 metricFamily.Name = proto.String(desc.fqName) 640 metricFamily.Help = proto.String(desc.help) 641 // TODO(beorn7): Simplify switch once Desc has type. 642 switch { 643 case dtoMetric.Gauge != nil: 644 metricFamily.Type = dto.MetricType_GAUGE.Enum() 645 case dtoMetric.Counter != nil: 646 metricFamily.Type = dto.MetricType_COUNTER.Enum() 647 case dtoMetric.Summary != nil: 648 metricFamily.Type = dto.MetricType_SUMMARY.Enum() 649 case dtoMetric.Untyped != nil: 650 metricFamily.Type = dto.MetricType_UNTYPED.Enum() 651 case dtoMetric.Histogram != nil: 652 metricFamily.Type = dto.MetricType_HISTOGRAM.Enum() 653 default: 654 return fmt.Errorf("empty metric collected: %s", dtoMetric) 655 } 656 if err := checkSuffixCollisions(metricFamily, metricFamiliesByName); err != nil { 657 return err 658 } 659 metricFamiliesByName[desc.fqName] = metricFamily 660 } 661 if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes); err != nil { 662 return err 663 } 664 if registeredDescIDs != nil { 665 // Is the desc registered at all? 666 if _, exist := registeredDescIDs[desc.id]; !exist { 667 return fmt.Errorf( 668 "collected metric %s %s with unregistered descriptor %s", 669 metricFamily.GetName(), dtoMetric, desc, 670 ) 671 } 672 if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil { 673 return err 674 } 675 } 676 metricFamily.Metric = append(metricFamily.Metric, dtoMetric) 677 return nil 678} 679 680// Gatherers is a slice of Gatherer instances that implements the Gatherer 681// interface itself. Its Gather method calls Gather on all Gatherers in the 682// slice in order and returns the merged results. Errors returned from the 683// Gather calles are all returned in a flattened MultiError. Duplicate and 684// inconsistent Metrics are skipped (first occurrence in slice order wins) and 685// reported in the returned error. 686// 687// Gatherers can be used to merge the Gather results from multiple 688// Registries. It also provides a way to directly inject existing MetricFamily 689// protobufs into the gathering by creating a custom Gatherer with a Gather 690// method that simply returns the existing MetricFamily protobufs. Note that no 691// registration is involved (in contrast to Collector registration), so 692// obviously registration-time checks cannot happen. Any inconsistencies between 693// the gathered MetricFamilies are reported as errors by the Gather method, and 694// inconsistent Metrics are dropped. Invalid parts of the MetricFamilies 695// (e.g. syntactically invalid metric or label names) will go undetected. 696type Gatherers []Gatherer 697 698// Gather implements Gatherer. 699func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) { 700 var ( 701 metricFamiliesByName = map[string]*dto.MetricFamily{} 702 metricHashes = map[uint64]struct{}{} 703 errs MultiError // The collected errors to return in the end. 704 ) 705 706 for i, g := range gs { 707 mfs, err := g.Gather() 708 if err != nil { 709 if multiErr, ok := err.(MultiError); ok { 710 for _, err := range multiErr { 711 errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) 712 } 713 } else { 714 errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err)) 715 } 716 } 717 for _, mf := range mfs { 718 existingMF, exists := metricFamiliesByName[mf.GetName()] 719 if exists { 720 if existingMF.GetHelp() != mf.GetHelp() { 721 errs = append(errs, fmt.Errorf( 722 "gathered metric family %s has help %q but should have %q", 723 mf.GetName(), mf.GetHelp(), existingMF.GetHelp(), 724 )) 725 continue 726 } 727 if existingMF.GetType() != mf.GetType() { 728 errs = append(errs, fmt.Errorf( 729 "gathered metric family %s has type %s but should have %s", 730 mf.GetName(), mf.GetType(), existingMF.GetType(), 731 )) 732 continue 733 } 734 } else { 735 existingMF = &dto.MetricFamily{} 736 existingMF.Name = mf.Name 737 existingMF.Help = mf.Help 738 existingMF.Type = mf.Type 739 if err := checkSuffixCollisions(existingMF, metricFamiliesByName); err != nil { 740 errs = append(errs, err) 741 continue 742 } 743 metricFamiliesByName[mf.GetName()] = existingMF 744 } 745 for _, m := range mf.Metric { 746 if err := checkMetricConsistency(existingMF, m, metricHashes); err != nil { 747 errs = append(errs, err) 748 continue 749 } 750 existingMF.Metric = append(existingMF.Metric, m) 751 } 752 } 753 } 754 return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() 755} 756 757// checkSuffixCollisions checks for collisions with the “magic” suffixes the 758// Prometheus text format and the internal metric representation of the 759// Prometheus server add while flattening Summaries and Histograms. 760func checkSuffixCollisions(mf *dto.MetricFamily, mfs map[string]*dto.MetricFamily) error { 761 var ( 762 newName = mf.GetName() 763 newType = mf.GetType() 764 newNameWithoutSuffix = "" 765 ) 766 switch { 767 case strings.HasSuffix(newName, "_count"): 768 newNameWithoutSuffix = newName[:len(newName)-6] 769 case strings.HasSuffix(newName, "_sum"): 770 newNameWithoutSuffix = newName[:len(newName)-4] 771 case strings.HasSuffix(newName, "_bucket"): 772 newNameWithoutSuffix = newName[:len(newName)-7] 773 } 774 if newNameWithoutSuffix != "" { 775 if existingMF, ok := mfs[newNameWithoutSuffix]; ok { 776 switch existingMF.GetType() { 777 case dto.MetricType_SUMMARY: 778 if !strings.HasSuffix(newName, "_bucket") { 779 return fmt.Errorf( 780 "collected metric named %q collides with previously collected summary named %q", 781 newName, newNameWithoutSuffix, 782 ) 783 } 784 case dto.MetricType_HISTOGRAM: 785 return fmt.Errorf( 786 "collected metric named %q collides with previously collected histogram named %q", 787 newName, newNameWithoutSuffix, 788 ) 789 } 790 } 791 } 792 if newType == dto.MetricType_SUMMARY || newType == dto.MetricType_HISTOGRAM { 793 if _, ok := mfs[newName+"_count"]; ok { 794 return fmt.Errorf( 795 "collected histogram or summary named %q collides with previously collected metric named %q", 796 newName, newName+"_count", 797 ) 798 } 799 if _, ok := mfs[newName+"_sum"]; ok { 800 return fmt.Errorf( 801 "collected histogram or summary named %q collides with previously collected metric named %q", 802 newName, newName+"_sum", 803 ) 804 } 805 } 806 if newType == dto.MetricType_HISTOGRAM { 807 if _, ok := mfs[newName+"_bucket"]; ok { 808 return fmt.Errorf( 809 "collected histogram named %q collides with previously collected metric named %q", 810 newName, newName+"_bucket", 811 ) 812 } 813 } 814 return nil 815} 816 817// checkMetricConsistency checks if the provided Metric is consistent with the 818// provided MetricFamily. It also hashes the Metric labels and the MetricFamily 819// name. If the resulting hash is already in the provided metricHashes, an error 820// is returned. If not, it is added to metricHashes. 821func checkMetricConsistency( 822 metricFamily *dto.MetricFamily, 823 dtoMetric *dto.Metric, 824 metricHashes map[uint64]struct{}, 825) error { 826 name := metricFamily.GetName() 827 828 // Type consistency with metric family. 829 if metricFamily.GetType() == dto.MetricType_GAUGE && dtoMetric.Gauge == nil || 830 metricFamily.GetType() == dto.MetricType_COUNTER && dtoMetric.Counter == nil || 831 metricFamily.GetType() == dto.MetricType_SUMMARY && dtoMetric.Summary == nil || 832 metricFamily.GetType() == dto.MetricType_HISTOGRAM && dtoMetric.Histogram == nil || 833 metricFamily.GetType() == dto.MetricType_UNTYPED && dtoMetric.Untyped == nil { 834 return fmt.Errorf( 835 "collected metric %q { %s} is not a %s", 836 name, dtoMetric, metricFamily.GetType(), 837 ) 838 } 839 840 previousLabelName := "" 841 for _, labelPair := range dtoMetric.GetLabel() { 842 labelName := labelPair.GetName() 843 if labelName == previousLabelName { 844 return fmt.Errorf( 845 "collected metric %q { %s} has two or more labels with the same name: %s", 846 name, dtoMetric, labelName, 847 ) 848 } 849 if !checkLabelName(labelName) { 850 return fmt.Errorf( 851 "collected metric %q { %s} has a label with an invalid name: %s", 852 name, dtoMetric, labelName, 853 ) 854 } 855 if dtoMetric.Summary != nil && labelName == quantileLabel { 856 return fmt.Errorf( 857 "collected metric %q { %s} must not have an explicit %q label", 858 name, dtoMetric, quantileLabel, 859 ) 860 } 861 if !utf8.ValidString(labelPair.GetValue()) { 862 return fmt.Errorf( 863 "collected metric %q { %s} has a label named %q whose value is not utf8: %#v", 864 name, dtoMetric, labelName, labelPair.GetValue()) 865 } 866 previousLabelName = labelName 867 } 868 869 // Is the metric unique (i.e. no other metric with the same name and the same labels)? 870 h := hashNew() 871 h = hashAdd(h, name) 872 h = hashAddByte(h, separatorByte) 873 // Make sure label pairs are sorted. We depend on it for the consistency 874 // check. 875 sort.Sort(labelPairSorter(dtoMetric.Label)) 876 for _, lp := range dtoMetric.Label { 877 h = hashAdd(h, lp.GetName()) 878 h = hashAddByte(h, separatorByte) 879 h = hashAdd(h, lp.GetValue()) 880 h = hashAddByte(h, separatorByte) 881 } 882 if _, exists := metricHashes[h]; exists { 883 return fmt.Errorf( 884 "collected metric %q { %s} was collected before with the same name and label values", 885 name, dtoMetric, 886 ) 887 } 888 metricHashes[h] = struct{}{} 889 return nil 890} 891 892func checkDescConsistency( 893 metricFamily *dto.MetricFamily, 894 dtoMetric *dto.Metric, 895 desc *Desc, 896) error { 897 // Desc help consistency with metric family help. 898 if metricFamily.GetHelp() != desc.help { 899 return fmt.Errorf( 900 "collected metric %s %s has help %q but should have %q", 901 metricFamily.GetName(), dtoMetric, metricFamily.GetHelp(), desc.help, 902 ) 903 } 904 905 // Is the desc consistent with the content of the metric? 906 lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label)) 907 lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...) 908 for _, l := range desc.variableLabels { 909 lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ 910 Name: proto.String(l), 911 }) 912 } 913 if len(lpsFromDesc) != len(dtoMetric.Label) { 914 return fmt.Errorf( 915 "labels in collected metric %s %s are inconsistent with descriptor %s", 916 metricFamily.GetName(), dtoMetric, desc, 917 ) 918 } 919 sort.Sort(labelPairSorter(lpsFromDesc)) 920 for i, lpFromDesc := range lpsFromDesc { 921 lpFromMetric := dtoMetric.Label[i] 922 if lpFromDesc.GetName() != lpFromMetric.GetName() || 923 lpFromDesc.Value != nil && lpFromDesc.GetValue() != lpFromMetric.GetValue() { 924 return fmt.Errorf( 925 "labels in collected metric %s %s are inconsistent with descriptor %s", 926 metricFamily.GetName(), dtoMetric, desc, 927 ) 928 } 929 } 930 return nil 931} 932