1{ 2 local thanos = self, 3 4 // We build alerts for the presence of all these jobs. 5 jobs:: { 6 ThanosQuery: thanos.query.selector, 7 ThanosStore: thanos.store.selector, 8 ThanosReceive: thanos.receive.selector, 9 ThanosRule: thanos.rule.selector, 10 ThanosCompact: thanos.compact.selector, 11 ThanosSidecar: thanos.sidecar.selector, 12 }, 13 14 prometheusAlerts+:: { 15 groups+: [ 16 { 17 name: 'thanos-component-absent.rules', 18 rules: [ 19 { 20 alert: '%sIsDown' % name, 21 expr: ||| 22 absent(up{%s} == 1) 23 ||| % thanos.jobs[name], 24 'for': '5m', 25 labels: { 26 severity: 'critical', 27 }, 28 annotations: { 29 message: '%s has disappeared from Prometheus target discovery.' % name, 30 }, 31 } 32 for name in std.objectFields(thanos.jobs) 33 ], 34 }, 35 ], 36 }, 37} 38