1package sender 2 3import ( 4 "context" 5 "net/url" 6 "strings" 7 "sync" 8 "time" 9 10 "github.com/grafana/grafana/pkg/infra/log" 11 apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" 12 "github.com/grafana/grafana/pkg/services/ngalert/logging" 13 "github.com/grafana/grafana/pkg/services/ngalert/metrics" 14 ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" 15 16 gokit_log "github.com/go-kit/kit/log" 17 "github.com/prometheus/alertmanager/api/v2/models" 18 "github.com/prometheus/client_golang/prometheus" 19 common_config "github.com/prometheus/common/config" 20 "github.com/prometheus/common/model" 21 "github.com/prometheus/prometheus/config" 22 "github.com/prometheus/prometheus/discovery" 23 "github.com/prometheus/prometheus/notifier" 24 "github.com/prometheus/prometheus/pkg/labels" 25) 26 27const ( 28 defaultMaxQueueCapacity = 10000 29 defaultTimeout = 10 * time.Second 30) 31 32// Sender is responsible for dispatching alert notifications to an external Alertmanager service. 33type Sender struct { 34 logger log.Logger 35 gokitLogger gokit_log.Logger 36 wg sync.WaitGroup 37 38 manager *notifier.Manager 39 40 sdCancel context.CancelFunc 41 sdManager *discovery.Manager 42} 43 44func New(_ *metrics.Scheduler) (*Sender, error) { 45 l := log.New("sender") 46 sdCtx, sdCancel := context.WithCancel(context.Background()) 47 s := &Sender{ 48 logger: l, 49 gokitLogger: gokit_log.NewLogfmtLogger(logging.NewWrapper(l)), 50 sdCancel: sdCancel, 51 } 52 53 s.manager = notifier.NewManager( 54 // Injecting a new registry here means these metrics are not exported. 55 // Once we fix the individual Alertmanager metrics we should fix this scenario too. 56 ¬ifier.Options{QueueCapacity: defaultMaxQueueCapacity, Registerer: prometheus.NewRegistry()}, 57 s.gokitLogger, 58 ) 59 60 s.sdManager = discovery.NewManager(sdCtx, s.gokitLogger) 61 62 return s, nil 63} 64 65// ApplyConfig syncs a configuration with the sender. 66func (s *Sender) ApplyConfig(cfg *ngmodels.AdminConfiguration) error { 67 notifierCfg, err := buildNotifierConfig(cfg) 68 if err != nil { 69 return err 70 } 71 72 if err := s.manager.ApplyConfig(notifierCfg); err != nil { 73 return err 74 } 75 76 sdCfgs := make(map[string]discovery.Configs) 77 for k, v := range notifierCfg.AlertingConfig.AlertmanagerConfigs.ToMap() { 78 sdCfgs[k] = v.ServiceDiscoveryConfigs 79 } 80 81 return s.sdManager.ApplyConfig(sdCfgs) 82} 83 84func (s *Sender) Run() { 85 s.wg.Add(2) 86 87 go func() { 88 if err := s.sdManager.Run(); err != nil { 89 s.logger.Error("failed to start the sender service discovery manager", "err", err) 90 } 91 s.wg.Done() 92 }() 93 94 go func() { 95 s.manager.Run(s.sdManager.SyncCh()) 96 s.wg.Done() 97 }() 98} 99 100// SendAlerts sends a set of alerts to the configured Alertmanager(s). 101func (s *Sender) SendAlerts(alerts apimodels.PostableAlerts) { 102 if len(alerts.PostableAlerts) == 0 { 103 s.logger.Debug("no alerts to send to external Alertmanager(s)") 104 return 105 } 106 as := make([]*notifier.Alert, 0, len(alerts.PostableAlerts)) 107 for _, a := range alerts.PostableAlerts { 108 na := alertToNotifierAlert(a) 109 as = append(as, na) 110 } 111 112 s.logger.Debug("sending alerts to the external Alertmanager(s)", "am_count", len(s.manager.Alertmanagers()), "alert_count", len(as)) 113 s.manager.Send(as...) 114} 115 116// Stop shuts down the sender. 117func (s *Sender) Stop() { 118 s.sdCancel() 119 s.manager.Stop() 120 s.wg.Wait() 121} 122 123// Alertmanagers returns a list of the discovered Alertmanager(s). 124func (s *Sender) Alertmanagers() []*url.URL { 125 return s.manager.Alertmanagers() 126} 127 128// DroppedAlertmanagers returns a list of Alertmanager(s) we no longer send alerts to. 129func (s *Sender) DroppedAlertmanagers() []*url.URL { 130 return s.manager.DroppedAlertmanagers() 131} 132 133func buildNotifierConfig(cfg *ngmodels.AdminConfiguration) (*config.Config, error) { 134 amConfigs := make([]*config.AlertmanagerConfig, 0, len(cfg.Alertmanagers)) 135 for _, amURL := range cfg.Alertmanagers { 136 u, err := url.Parse(amURL) 137 if err != nil { 138 return nil, err 139 } 140 141 sdConfig := discovery.Configs{ 142 discovery.StaticConfig{ 143 { 144 Targets: []model.LabelSet{{model.AddressLabel: model.LabelValue(u.Host)}}, 145 }, 146 }, 147 } 148 149 amConfig := &config.AlertmanagerConfig{ 150 APIVersion: config.AlertmanagerAPIVersionV2, 151 Scheme: u.Scheme, 152 PathPrefix: u.Path, 153 Timeout: model.Duration(defaultTimeout), 154 ServiceDiscoveryConfigs: sdConfig, 155 } 156 157 // Check the URL for basic authentication information first 158 if u.User != nil { 159 amConfig.HTTPClientConfig.BasicAuth = &common_config.BasicAuth{ 160 Username: u.User.Username(), 161 } 162 163 if password, isSet := u.User.Password(); isSet { 164 amConfig.HTTPClientConfig.BasicAuth.Password = common_config.Secret(password) 165 } 166 } 167 amConfigs = append(amConfigs, amConfig) 168 } 169 170 notifierConfig := &config.Config{ 171 AlertingConfig: config.AlertingConfig{ 172 AlertmanagerConfigs: amConfigs, 173 }, 174 } 175 176 return notifierConfig, nil 177} 178 179func alertToNotifierAlert(alert models.PostableAlert) *notifier.Alert { 180 ls := make(labels.Labels, 0, len(alert.Alert.Labels)) 181 a := make(labels.Labels, 0, len(alert.Annotations)) 182 183 // Prometheus does not allow spaces in labels or annotations while Grafana does, we need to make sure we 184 // remove them before sending the alerts. 185 for k, v := range alert.Alert.Labels { 186 ls = append(ls, labels.Label{Name: removeSpaces(k), Value: v}) 187 } 188 189 for k, v := range alert.Annotations { 190 a = append(a, labels.Label{Name: removeSpaces(k), Value: v}) 191 } 192 193 return ¬ifier.Alert{ 194 Labels: ls, 195 Annotations: a, 196 StartsAt: time.Time(alert.StartsAt), 197 EndsAt: time.Time(alert.EndsAt), 198 GeneratorURL: alert.Alert.GeneratorURL.String(), 199 } 200} 201 202func removeSpaces(labelName string) string { 203 return strings.Join(strings.Fields(labelName), "") 204} 205