1/* 2Copyright 2019 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package testutil 18 19import ( 20 "fmt" 21 "io" 22 "math" 23 "reflect" 24 "sort" 25 "strings" 26 27 dto "github.com/prometheus/client_model/go" 28 "github.com/prometheus/common/expfmt" 29 "github.com/prometheus/common/model" 30 31 "k8s.io/component-base/metrics" 32) 33 34var ( 35 // MetricNameLabel is label under which model.Sample stores metric name 36 MetricNameLabel model.LabelName = model.MetricNameLabel 37 // QuantileLabel is label under which model.Sample stores latency quantile value 38 QuantileLabel model.LabelName = model.QuantileLabel 39) 40 41// Metrics is generic metrics for other specific metrics 42type Metrics map[string]model.Samples 43 44// Equal returns true if all metrics are the same as the arguments. 45func (m *Metrics) Equal(o Metrics) bool { 46 var leftKeySet []string 47 var rightKeySet []string 48 for k := range *m { 49 leftKeySet = append(leftKeySet, k) 50 } 51 for k := range o { 52 rightKeySet = append(rightKeySet, k) 53 } 54 if !reflect.DeepEqual(leftKeySet, rightKeySet) { 55 return false 56 } 57 for _, k := range leftKeySet { 58 if !(*m)[k].Equal(o[k]) { 59 return false 60 } 61 } 62 return true 63} 64 65// NewMetrics returns new metrics which are initialized. 66func NewMetrics() Metrics { 67 result := make(Metrics) 68 return result 69} 70 71// ParseMetrics parses Metrics from data returned from prometheus endpoint 72func ParseMetrics(data string, output *Metrics) error { 73 dec := expfmt.NewDecoder(strings.NewReader(data), expfmt.FmtText) 74 decoder := expfmt.SampleDecoder{ 75 Dec: dec, 76 Opts: &expfmt.DecodeOptions{}, 77 } 78 79 for { 80 var v model.Vector 81 if err := decoder.Decode(&v); err != nil { 82 if err == io.EOF { 83 // Expected loop termination condition. 84 return nil 85 } 86 continue 87 } 88 for _, metric := range v { 89 name := string(metric.Metric[MetricNameLabel]) 90 (*output)[name] = append((*output)[name], metric) 91 } 92 } 93} 94 95// TextToMetricFamilies reads 'in' as the simple and flat text-based exchange 96// format and creates MetricFamily proto messages. It returns the MetricFamily 97// proto messages in a map where the metric names are the keys, along with any 98// error encountered. 99func TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) { 100 var textParser expfmt.TextParser 101 return textParser.TextToMetricFamilies(in) 102} 103 104// PrintSample returns formatted representation of metric Sample 105func PrintSample(sample *model.Sample) string { 106 buf := make([]string, 0) 107 // Id is a VERY special label. For 'normal' container it's useless, but it's necessary 108 // for 'system' containers (e.g. /docker-daemon, /kubelet, etc.). We know if that's the 109 // case by checking if there's a label "kubernetes_container_name" present. It's hacky 110 // but it works... 111 _, normalContainer := sample.Metric["kubernetes_container_name"] 112 for k, v := range sample.Metric { 113 if strings.HasPrefix(string(k), "__") { 114 continue 115 } 116 117 if string(k) == "id" && normalContainer { 118 continue 119 } 120 buf = append(buf, fmt.Sprintf("%v=%v", string(k), v)) 121 } 122 return fmt.Sprintf("[%v] = %v", strings.Join(buf, ","), sample.Value) 123} 124 125// ComputeHistogramDelta computes the change in histogram metric for a selected label. 126// Results are stored in after samples 127func ComputeHistogramDelta(before, after model.Samples, label model.LabelName) { 128 beforeSamplesMap := make(map[string]*model.Sample) 129 for _, bSample := range before { 130 beforeSamplesMap[makeKey(bSample.Metric[label], bSample.Metric["le"])] = bSample 131 } 132 for _, aSample := range after { 133 if bSample, found := beforeSamplesMap[makeKey(aSample.Metric[label], aSample.Metric["le"])]; found { 134 aSample.Value = aSample.Value - bSample.Value 135 } 136 } 137} 138 139func makeKey(a, b model.LabelValue) string { 140 return string(a) + "___" + string(b) 141} 142 143// GetMetricValuesForLabel returns value of metric for a given dimension 144func GetMetricValuesForLabel(ms Metrics, metricName, label string) map[string]int64 { 145 samples, found := ms[metricName] 146 result := make(map[string]int64, len(samples)) 147 if !found { 148 return result 149 } 150 for _, sample := range samples { 151 count := int64(sample.Value) 152 dimensionName := string(sample.Metric[model.LabelName(label)]) 153 result[dimensionName] = count 154 } 155 return result 156} 157 158// ValidateMetrics verifies if every sample of metric has all expected labels 159func ValidateMetrics(metrics Metrics, metricName string, expectedLabels ...string) error { 160 samples, ok := metrics[metricName] 161 if !ok { 162 return fmt.Errorf("metric %q was not found in metrics", metricName) 163 } 164 for _, sample := range samples { 165 for _, l := range expectedLabels { 166 if _, ok := sample.Metric[model.LabelName(l)]; !ok { 167 return fmt.Errorf("metric %q is missing label %q, sample: %q", metricName, l, sample.String()) 168 } 169 } 170 } 171 return nil 172} 173 174// Histogram wraps prometheus histogram DTO (data transfer object) 175type Histogram struct { 176 *dto.Histogram 177} 178 179// GetHistogramFromGatherer collects a metric from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface. 180// Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint). 181func GetHistogramFromGatherer(gatherer metrics.Gatherer, metricName string) (Histogram, error) { 182 var metricFamily *dto.MetricFamily 183 m, err := gatherer.Gather() 184 if err != nil { 185 return Histogram{}, err 186 } 187 for _, mFamily := range m { 188 if mFamily.GetName() == metricName { 189 metricFamily = mFamily 190 break 191 } 192 } 193 194 if metricFamily == nil { 195 return Histogram{}, fmt.Errorf("metric %q not found", metricName) 196 } 197 198 if metricFamily.GetMetric() == nil { 199 return Histogram{}, fmt.Errorf("metric %q is empty", metricName) 200 } 201 202 if len(metricFamily.GetMetric()) == 0 { 203 return Histogram{}, fmt.Errorf("metric %q is empty", metricName) 204 } 205 206 return Histogram{ 207 // Histograms are stored under the first index (based on observation). 208 // Given there's only one histogram registered per each metric name, accessing 209 // the first index is sufficient. 210 metricFamily.GetMetric()[0].GetHistogram(), 211 }, nil 212} 213 214func uint64Ptr(u uint64) *uint64 { 215 return &u 216} 217 218// Bucket of a histogram 219type bucket struct { 220 upperBound float64 221 count float64 222} 223 224func bucketQuantile(q float64, buckets []bucket) float64 { 225 if q < 0 { 226 return math.Inf(-1) 227 } 228 if q > 1 { 229 return math.Inf(+1) 230 } 231 232 if len(buckets) < 2 { 233 return math.NaN() 234 } 235 236 rank := q * buckets[len(buckets)-1].count 237 b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank }) 238 239 if b == 0 { 240 return buckets[0].upperBound * (rank / buckets[0].count) 241 } 242 243 if b == len(buckets)-1 && math.IsInf(buckets[b].upperBound, 1) { 244 return buckets[len(buckets)-2].upperBound 245 } 246 247 // linear approximation of b-th bucket 248 brank := rank - buckets[b-1].count 249 bSize := buckets[b].upperBound - buckets[b-1].upperBound 250 bCount := buckets[b].count - buckets[b-1].count 251 252 return buckets[b-1].upperBound + bSize*(brank/bCount) 253} 254 255// Quantile computes q-th quantile of a cumulative histogram. 256// It's expected the histogram is valid (by calling Validate) 257func (hist *Histogram) Quantile(q float64) float64 { 258 var buckets []bucket 259 260 for _, bckt := range hist.Bucket { 261 buckets = append(buckets, bucket{ 262 count: float64(bckt.GetCumulativeCount()), 263 upperBound: bckt.GetUpperBound(), 264 }) 265 } 266 267 if len(buckets) == 0 || buckets[len(buckets)-1].upperBound != math.Inf(+1) { 268 // The list of buckets in dto.Histogram doesn't include the final +Inf bucket, so we 269 // add it here for the reset of the samples. 270 buckets = append(buckets, bucket{ 271 count: float64(hist.GetSampleCount()), 272 upperBound: math.Inf(+1), 273 }) 274 } 275 276 return bucketQuantile(q, buckets) 277} 278 279// Average computes histogram's average value 280func (hist *Histogram) Average() float64 { 281 return hist.GetSampleSum() / float64(hist.GetSampleCount()) 282} 283 284// Validate makes sure the wrapped histogram has all necessary fields set and with valid values. 285func (hist *Histogram) Validate() error { 286 if hist.SampleCount == nil || hist.GetSampleCount() == 0 { 287 return fmt.Errorf("nil or empty histogram SampleCount") 288 } 289 290 if hist.SampleSum == nil || hist.GetSampleSum() == 0 { 291 return fmt.Errorf("nil or empty histogram SampleSum") 292 } 293 294 for _, bckt := range hist.Bucket { 295 if bckt == nil { 296 return fmt.Errorf("empty histogram bucket") 297 } 298 if bckt.UpperBound == nil || bckt.GetUpperBound() < 0 { 299 return fmt.Errorf("nil or negative histogram bucket UpperBound") 300 } 301 } 302 303 return nil 304} 305 306// GetGaugeMetricValue extracts metric value from GaugeMetric 307func GetGaugeMetricValue(m metrics.GaugeMetric) (float64, error) { 308 metricProto := &dto.Metric{} 309 if err := m.Write(metricProto); err != nil { 310 return 0, fmt.Errorf("error writing m: %v", err) 311 } 312 return metricProto.Gauge.GetValue(), nil 313} 314 315// GetCounterMetricValue extracts metric value from CounterMetric 316func GetCounterMetricValue(m metrics.CounterMetric) (float64, error) { 317 metricProto := &dto.Metric{} 318 if err := m.(metrics.Metric).Write(metricProto); err != nil { 319 return 0, fmt.Errorf("error writing m: %v", err) 320 } 321 return metricProto.Counter.GetValue(), nil 322} 323 324// GetHistogramMetricValue extracts sum of all samples from ObserverMetric 325func GetHistogramMetricValue(m metrics.ObserverMetric) (float64, error) { 326 metricProto := &dto.Metric{} 327 if err := m.(metrics.Metric).Write(metricProto); err != nil { 328 return 0, fmt.Errorf("error writing m: %v", err) 329 } 330 return metricProto.Histogram.GetSampleSum(), nil 331} 332 333// GetHistogramMetricCount extracts count of all samples from ObserverMetric 334func GetHistogramMetricCount(m metrics.ObserverMetric) (uint64, error) { 335 metricProto := &dto.Metric{} 336 if err := m.(metrics.Metric).Write(metricProto); err != nil { 337 return 0, fmt.Errorf("error writing m: %v", err) 338 } 339 return metricProto.Histogram.GetSampleCount(), nil 340} 341 342// LabelsMatch returns true if metric has all expected labels otherwise false 343func LabelsMatch(metric *dto.Metric, labelFilter map[string]string) bool { 344 metricLabels := map[string]string{} 345 346 for _, labelPair := range metric.Label { 347 metricLabels[labelPair.GetName()] = labelPair.GetValue() 348 } 349 350 // length comparison then match key to values in the maps 351 if len(labelFilter) > len(metricLabels) { 352 return false 353 } 354 355 for labelName, labelValue := range labelFilter { 356 if value, ok := metricLabels[labelName]; !ok || value != labelValue { 357 return false 358 } 359 } 360 361 return true 362} 363