1// Copyright 2016 The Prometheus Authors 2// Licensed under the Apache License, Version 2.0 (the "License"); 3// you may not use this file except in compliance with the License. 4// You may obtain a copy of the License at 5// 6// http://www.apache.org/licenses/LICENSE-2.0 7// 8// Unless required by applicable law or agreed to in writing, software 9// distributed under the License is distributed on an "AS IS" BASIS, 10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11// See the License for the specific language governing permissions and 12// limitations under the License. 13 14// +build go1.8 15 16package config 17 18import ( 19 "bytes" 20 "crypto/md5" 21 "crypto/tls" 22 "crypto/x509" 23 "fmt" 24 "io/ioutil" 25 "net/http" 26 "net/url" 27 "strings" 28 "sync" 29 "time" 30 31 "github.com/mwitkow/go-conntrack" 32 "gopkg.in/yaml.v2" 33) 34 35type closeIdler interface { 36 CloseIdleConnections() 37} 38 39// BasicAuth contains basic HTTP authentication credentials. 40type BasicAuth struct { 41 Username string `yaml:"username"` 42 Password Secret `yaml:"password,omitempty"` 43 PasswordFile string `yaml:"password_file,omitempty"` 44} 45 46// URL is a custom URL type that allows validation at configuration load time. 47type URL struct { 48 *url.URL 49} 50 51// UnmarshalYAML implements the yaml.Unmarshaler interface for URLs. 52func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error { 53 var s string 54 if err := unmarshal(&s); err != nil { 55 return err 56 } 57 58 urlp, err := url.Parse(s) 59 if err != nil { 60 return err 61 } 62 u.URL = urlp 63 return nil 64} 65 66// MarshalYAML implements the yaml.Marshaler interface for URLs. 67func (u URL) MarshalYAML() (interface{}, error) { 68 if u.URL != nil { 69 return u.String(), nil 70 } 71 return nil, nil 72} 73 74// HTTPClientConfig configures an HTTP client. 75type HTTPClientConfig struct { 76 // The HTTP basic authentication credentials for the targets. 77 BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"` 78 // The bearer token for the targets. 79 BearerToken Secret `yaml:"bearer_token,omitempty"` 80 // The bearer token file for the targets. 81 BearerTokenFile string `yaml:"bearer_token_file,omitempty"` 82 // HTTP proxy server to use to connect to the targets. 83 ProxyURL URL `yaml:"proxy_url,omitempty"` 84 // TLSConfig to use to connect to the targets. 85 TLSConfig TLSConfig `yaml:"tls_config,omitempty"` 86} 87 88// Validate validates the HTTPClientConfig to check only one of BearerToken, 89// BasicAuth and BearerTokenFile is configured. 90func (c *HTTPClientConfig) Validate() error { 91 if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 { 92 return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured") 93 } 94 if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { 95 return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured") 96 } 97 if c.BasicAuth != nil && (string(c.BasicAuth.Password) != "" && c.BasicAuth.PasswordFile != "") { 98 return fmt.Errorf("at most one of basic_auth password & password_file must be configured") 99 } 100 return nil 101} 102 103// UnmarshalYAML implements the yaml.Unmarshaler interface 104func (c *HTTPClientConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { 105 type plain HTTPClientConfig 106 if err := unmarshal((*plain)(c)); err != nil { 107 return err 108 } 109 return c.Validate() 110} 111 112// UnmarshalYAML implements the yaml.Unmarshaler interface. 113func (a *BasicAuth) UnmarshalYAML(unmarshal func(interface{}) error) error { 114 type plain BasicAuth 115 return unmarshal((*plain)(a)) 116} 117 118// NewClient returns a http.Client using the specified http.RoundTripper. 119func newClient(rt http.RoundTripper) *http.Client { 120 return &http.Client{Transport: rt} 121} 122 123// NewClientFromConfig returns a new HTTP client configured for the 124// given config.HTTPClientConfig. The name is used as go-conntrack metric label. 125func NewClientFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives bool) (*http.Client, error) { 126 rt, err := NewRoundTripperFromConfig(cfg, name, disableKeepAlives) 127 if err != nil { 128 return nil, err 129 } 130 return newClient(rt), nil 131} 132 133// NewRoundTripperFromConfig returns a new HTTP RoundTripper configured for the 134// given config.HTTPClientConfig. The name is used as go-conntrack metric label. 135func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, disableKeepAlives bool) (http.RoundTripper, error) { 136 newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) { 137 // The only timeout we care about is the configured scrape timeout. 138 // It is applied on request. So we leave out any timings here. 139 var rt http.RoundTripper = &http.Transport{ 140 Proxy: http.ProxyURL(cfg.ProxyURL.URL), 141 MaxIdleConns: 20000, 142 MaxIdleConnsPerHost: 1000, // see https://github.com/golang/go/issues/13801 143 DisableKeepAlives: disableKeepAlives, 144 TLSClientConfig: tlsConfig, 145 DisableCompression: true, 146 // 5 minutes is typically above the maximum sane scrape interval. So we can 147 // use keepalive for all configurations. 148 IdleConnTimeout: 5 * time.Minute, 149 TLSHandshakeTimeout: 10 * time.Second, 150 ExpectContinueTimeout: 1 * time.Second, 151 DialContext: conntrack.NewDialContextFunc( 152 conntrack.DialWithTracing(), 153 conntrack.DialWithName(name), 154 ), 155 } 156 157 // If a bearer token is provided, create a round tripper that will set the 158 // Authorization header correctly on each request. 159 if len(cfg.BearerToken) > 0 { 160 rt = NewBearerAuthRoundTripper(cfg.BearerToken, rt) 161 } else if len(cfg.BearerTokenFile) > 0 { 162 rt = NewBearerAuthFileRoundTripper(cfg.BearerTokenFile, rt) 163 } 164 165 if cfg.BasicAuth != nil { 166 rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.PasswordFile, rt) 167 } 168 // Return a new configured RoundTripper. 169 return rt, nil 170 } 171 172 tlsConfig, err := NewTLSConfig(&cfg.TLSConfig) 173 if err != nil { 174 return nil, err 175 } 176 177 if len(cfg.TLSConfig.CAFile) == 0 { 178 // No need for a RoundTripper that reloads the CA file automatically. 179 return newRT(tlsConfig) 180 } 181 182 return newTLSRoundTripper(tlsConfig, cfg.TLSConfig.CAFile, newRT) 183} 184 185type bearerAuthRoundTripper struct { 186 bearerToken Secret 187 rt http.RoundTripper 188} 189 190// NewBearerAuthRoundTripper adds the provided bearer token to a request unless the authorization 191// header has already been set. 192func NewBearerAuthRoundTripper(token Secret, rt http.RoundTripper) http.RoundTripper { 193 return &bearerAuthRoundTripper{token, rt} 194} 195 196func (rt *bearerAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 197 if len(req.Header.Get("Authorization")) == 0 { 198 req = cloneRequest(req) 199 req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(rt.bearerToken))) 200 } 201 return rt.rt.RoundTrip(req) 202} 203 204func (rt *bearerAuthRoundTripper) CloseIdleConnections() { 205 if ci, ok := rt.rt.(closeIdler); ok { 206 ci.CloseIdleConnections() 207 } 208} 209 210type bearerAuthFileRoundTripper struct { 211 bearerFile string 212 rt http.RoundTripper 213} 214 215// NewBearerAuthFileRoundTripper adds the bearer token read from the provided file to a request unless 216// the authorization header has already been set. This file is read for every request. 217func NewBearerAuthFileRoundTripper(bearerFile string, rt http.RoundTripper) http.RoundTripper { 218 return &bearerAuthFileRoundTripper{bearerFile, rt} 219} 220 221func (rt *bearerAuthFileRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 222 if len(req.Header.Get("Authorization")) == 0 { 223 b, err := ioutil.ReadFile(rt.bearerFile) 224 if err != nil { 225 return nil, fmt.Errorf("unable to read bearer token file %s: %s", rt.bearerFile, err) 226 } 227 bearerToken := strings.TrimSpace(string(b)) 228 229 req = cloneRequest(req) 230 req.Header.Set("Authorization", "Bearer "+bearerToken) 231 } 232 233 return rt.rt.RoundTrip(req) 234} 235 236func (rt *bearerAuthFileRoundTripper) CloseIdleConnections() { 237 if ci, ok := rt.rt.(closeIdler); ok { 238 ci.CloseIdleConnections() 239 } 240} 241 242type basicAuthRoundTripper struct { 243 username string 244 password Secret 245 passwordFile string 246 rt http.RoundTripper 247} 248 249// NewBasicAuthRoundTripper will apply a BASIC auth authorization header to a request unless it has 250// already been set. 251func NewBasicAuthRoundTripper(username string, password Secret, passwordFile string, rt http.RoundTripper) http.RoundTripper { 252 return &basicAuthRoundTripper{username, password, passwordFile, rt} 253} 254 255func (rt *basicAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 256 if len(req.Header.Get("Authorization")) != 0 { 257 return rt.rt.RoundTrip(req) 258 } 259 req = cloneRequest(req) 260 if rt.passwordFile != "" { 261 bs, err := ioutil.ReadFile(rt.passwordFile) 262 if err != nil { 263 return nil, fmt.Errorf("unable to read basic auth password file %s: %s", rt.passwordFile, err) 264 } 265 req.SetBasicAuth(rt.username, strings.TrimSpace(string(bs))) 266 } else { 267 req.SetBasicAuth(rt.username, strings.TrimSpace(string(rt.password))) 268 } 269 return rt.rt.RoundTrip(req) 270} 271 272func (rt *basicAuthRoundTripper) CloseIdleConnections() { 273 if ci, ok := rt.rt.(closeIdler); ok { 274 ci.CloseIdleConnections() 275 } 276} 277 278// cloneRequest returns a clone of the provided *http.Request. 279// The clone is a shallow copy of the struct and its Header map. 280func cloneRequest(r *http.Request) *http.Request { 281 // Shallow copy of the struct. 282 r2 := new(http.Request) 283 *r2 = *r 284 // Deep copy of the Header. 285 r2.Header = make(http.Header) 286 for k, s := range r.Header { 287 r2.Header[k] = s 288 } 289 return r2 290} 291 292// NewTLSConfig creates a new tls.Config from the given TLSConfig. 293func NewTLSConfig(cfg *TLSConfig) (*tls.Config, error) { 294 tlsConfig := &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify} 295 296 // If a CA cert is provided then let's read it in so we can validate the 297 // scrape target's certificate properly. 298 if len(cfg.CAFile) > 0 { 299 b, err := readCAFile(cfg.CAFile) 300 if err != nil { 301 return nil, err 302 } 303 if !updateRootCA(tlsConfig, b) { 304 return nil, fmt.Errorf("unable to use specified CA cert %s", cfg.CAFile) 305 } 306 } 307 308 if len(cfg.ServerName) > 0 { 309 tlsConfig.ServerName = cfg.ServerName 310 } 311 // If a client cert & key is provided then configure TLS config accordingly. 312 if len(cfg.CertFile) > 0 && len(cfg.KeyFile) == 0 { 313 return nil, fmt.Errorf("client cert file %q specified without client key file", cfg.CertFile) 314 } else if len(cfg.KeyFile) > 0 && len(cfg.CertFile) == 0 { 315 return nil, fmt.Errorf("client key file %q specified without client cert file", cfg.KeyFile) 316 } else if len(cfg.CertFile) > 0 && len(cfg.KeyFile) > 0 { 317 // Verify that client cert and key are valid. 318 if _, err := cfg.getClientCertificate(nil); err != nil { 319 return nil, err 320 } 321 tlsConfig.GetClientCertificate = cfg.getClientCertificate 322 } 323 324 return tlsConfig, nil 325} 326 327// TLSConfig configures the options for TLS connections. 328type TLSConfig struct { 329 // The CA cert to use for the targets. 330 CAFile string `yaml:"ca_file,omitempty"` 331 // The client cert file for the targets. 332 CertFile string `yaml:"cert_file,omitempty"` 333 // The client key file for the targets. 334 KeyFile string `yaml:"key_file,omitempty"` 335 // Used to verify the hostname for the targets. 336 ServerName string `yaml:"server_name,omitempty"` 337 // Disable target certificate validation. 338 InsecureSkipVerify bool `yaml:"insecure_skip_verify"` 339} 340 341// UnmarshalYAML implements the yaml.Unmarshaler interface. 342func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { 343 type plain TLSConfig 344 return unmarshal((*plain)(c)) 345} 346 347// getClientCertificate reads the pair of client cert and key from disk and returns a tls.Certificate. 348func (c *TLSConfig) getClientCertificate(*tls.CertificateRequestInfo) (*tls.Certificate, error) { 349 cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) 350 if err != nil { 351 return nil, fmt.Errorf("unable to use specified client cert (%s) & key (%s): %s", c.CertFile, c.KeyFile, err) 352 } 353 return &cert, nil 354} 355 356// readCAFile reads the CA cert file from disk. 357func readCAFile(f string) ([]byte, error) { 358 data, err := ioutil.ReadFile(f) 359 if err != nil { 360 return nil, fmt.Errorf("unable to load specified CA cert %s: %s", f, err) 361 } 362 return data, nil 363} 364 365// updateRootCA parses the given byte slice as a series of PEM encoded certificates and updates tls.Config.RootCAs. 366func updateRootCA(cfg *tls.Config, b []byte) bool { 367 caCertPool := x509.NewCertPool() 368 if !caCertPool.AppendCertsFromPEM(b) { 369 return false 370 } 371 cfg.RootCAs = caCertPool 372 return true 373} 374 375// tlsRoundTripper is a RoundTripper that updates automatically its TLS 376// configuration whenever the content of the CA file changes. 377type tlsRoundTripper struct { 378 caFile string 379 // newRT returns a new RoundTripper. 380 newRT func(*tls.Config) (http.RoundTripper, error) 381 382 mtx sync.RWMutex 383 rt http.RoundTripper 384 hashCAFile []byte 385 tlsConfig *tls.Config 386} 387 388func newTLSRoundTripper( 389 cfg *tls.Config, 390 caFile string, 391 newRT func(*tls.Config) (http.RoundTripper, error), 392) (http.RoundTripper, error) { 393 t := &tlsRoundTripper{ 394 caFile: caFile, 395 newRT: newRT, 396 tlsConfig: cfg, 397 } 398 399 rt, err := t.newRT(t.tlsConfig) 400 if err != nil { 401 return nil, err 402 } 403 t.rt = rt 404 405 _, t.hashCAFile, err = t.getCAWithHash() 406 if err != nil { 407 return nil, err 408 } 409 410 return t, nil 411} 412 413func (t *tlsRoundTripper) getCAWithHash() ([]byte, []byte, error) { 414 b, err := readCAFile(t.caFile) 415 if err != nil { 416 return nil, nil, err 417 } 418 h := md5.Sum(b) 419 return b, h[:], nil 420 421} 422 423// RoundTrip implements the http.RoundTrip interface. 424func (t *tlsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 425 b, h, err := t.getCAWithHash() 426 if err != nil { 427 return nil, err 428 } 429 430 t.mtx.RLock() 431 equal := bytes.Equal(h[:], t.hashCAFile) 432 rt := t.rt 433 t.mtx.RUnlock() 434 if equal { 435 // The CA cert hasn't changed, use the existing RoundTripper. 436 return rt.RoundTrip(req) 437 } 438 439 // Create a new RoundTripper. 440 tlsConfig := t.tlsConfig.Clone() 441 if !updateRootCA(tlsConfig, b) { 442 return nil, fmt.Errorf("unable to use specified CA cert %s", t.caFile) 443 } 444 rt, err = t.newRT(tlsConfig) 445 if err != nil { 446 return nil, err 447 } 448 t.CloseIdleConnections() 449 450 t.mtx.Lock() 451 t.rt = rt 452 t.hashCAFile = h[:] 453 t.mtx.Unlock() 454 455 return rt.RoundTrip(req) 456} 457 458func (t *tlsRoundTripper) CloseIdleConnections() { 459 t.mtx.RLock() 460 defer t.mtx.RUnlock() 461 if ci, ok := t.rt.(closeIdler); ok { 462 ci.CloseIdleConnections() 463 } 464} 465 466func (c HTTPClientConfig) String() string { 467 b, err := yaml.Marshal(c) 468 if err != nil { 469 return fmt.Sprintf("<error creating http client config string: %s>", err) 470 } 471 return string(b) 472} 473