1// Copyright 2016 The etcd Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package embed 16 17import ( 18 "fmt" 19 "io/ioutil" 20 "net" 21 "net/http" 22 "net/url" 23 "os" 24 "path/filepath" 25 "strings" 26 "sync" 27 "time" 28 29 "go.etcd.io/etcd/etcdserver" 30 "go.etcd.io/etcd/etcdserver/api/v3compactor" 31 "go.etcd.io/etcd/pkg/flags" 32 "go.etcd.io/etcd/pkg/logutil" 33 "go.etcd.io/etcd/pkg/netutil" 34 "go.etcd.io/etcd/pkg/srv" 35 "go.etcd.io/etcd/pkg/tlsutil" 36 "go.etcd.io/etcd/pkg/transport" 37 "go.etcd.io/etcd/pkg/types" 38 39 bolt "go.etcd.io/bbolt" 40 "go.uber.org/zap" 41 "go.uber.org/zap/zapcore" 42 "golang.org/x/crypto/bcrypt" 43 "google.golang.org/grpc" 44 "sigs.k8s.io/yaml" 45) 46 47const ( 48 ClusterStateFlagNew = "new" 49 ClusterStateFlagExisting = "existing" 50 51 DefaultName = "default" 52 DefaultMaxSnapshots = 5 53 DefaultMaxWALs = 5 54 DefaultMaxTxnOps = uint(128) 55 DefaultMaxRequestBytes = 1.5 * 1024 * 1024 56 DefaultGRPCKeepAliveMinTime = 5 * time.Second 57 DefaultGRPCKeepAliveInterval = 2 * time.Hour 58 DefaultGRPCKeepAliveTimeout = 20 * time.Second 59 60 DefaultListenPeerURLs = "http://localhost:2380" 61 DefaultListenClientURLs = "http://localhost:2379" 62 63 DefaultLogOutput = "default" 64 JournalLogOutput = "systemd/journal" 65 StdErrLogOutput = "stderr" 66 StdOutLogOutput = "stdout" 67 68 // DefaultStrictReconfigCheck is the default value for "--strict-reconfig-check" flag. 69 // It's enabled by default. 70 DefaultStrictReconfigCheck = true 71 // DefaultEnableV2 is the default value for "--enable-v2" flag. 72 // v2 API is disabled by default. 73 DefaultEnableV2 = false 74 75 // maxElectionMs specifies the maximum value of election timeout. 76 // More details are listed in ../Documentation/tuning.md#time-parameters. 77 maxElectionMs = 50000 78 // backend freelist map type 79 freelistArrayType = "array" 80) 81 82var ( 83 ErrConflictBootstrapFlags = fmt.Errorf("multiple discovery or bootstrap flags are set. " + 84 "Choose one of \"initial-cluster\", \"discovery\" or \"discovery-srv\"") 85 ErrUnsetAdvertiseClientURLsFlag = fmt.Errorf("--advertise-client-urls is required when --listen-client-urls is set explicitly") 86 87 DefaultInitialAdvertisePeerURLs = "http://localhost:2380" 88 DefaultAdvertiseClientURLs = "http://localhost:2379" 89 90 defaultHostname string 91 defaultHostStatus error 92) 93 94var ( 95 // CompactorModePeriodic is periodic compaction mode 96 // for "Config.AutoCompactionMode" field. 97 // If "AutoCompactionMode" is CompactorModePeriodic and 98 // "AutoCompactionRetention" is "1h", it automatically compacts 99 // compacts storage every hour. 100 CompactorModePeriodic = v3compactor.ModePeriodic 101 102 // CompactorModeRevision is revision-based compaction mode 103 // for "Config.AutoCompactionMode" field. 104 // If "AutoCompactionMode" is CompactorModeRevision and 105 // "AutoCompactionRetention" is "1000", it compacts log on 106 // revision 5000 when the current revision is 6000. 107 // This runs every 5-minute if enough of logs have proceeded. 108 CompactorModeRevision = v3compactor.ModeRevision 109) 110 111func init() { 112 defaultHostname, defaultHostStatus = netutil.GetDefaultHost() 113} 114 115// Config holds the arguments for configuring an etcd server. 116type Config struct { 117 Name string `json:"name"` 118 Dir string `json:"data-dir"` 119 WalDir string `json:"wal-dir"` 120 121 SnapshotCount uint64 `json:"snapshot-count"` 122 123 // SnapshotCatchUpEntries is the number of entries for a slow follower 124 // to catch-up after compacting the raft storage entries. 125 // We expect the follower has a millisecond level latency with the leader. 126 // The max throughput is around 10K. Keep a 5K entries is enough for helping 127 // follower to catch up. 128 // WARNING: only change this for tests. 129 // Always use "DefaultSnapshotCatchUpEntries" 130 SnapshotCatchUpEntries uint64 131 132 MaxSnapFiles uint `json:"max-snapshots"` 133 MaxWalFiles uint `json:"max-wals"` 134 135 // TickMs is the number of milliseconds between heartbeat ticks. 136 // TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1). 137 // make ticks a cluster wide configuration. 138 TickMs uint `json:"heartbeat-interval"` 139 ElectionMs uint `json:"election-timeout"` 140 141 // InitialElectionTickAdvance is true, then local member fast-forwards 142 // election ticks to speed up "initial" leader election trigger. This 143 // benefits the case of larger election ticks. For instance, cross 144 // datacenter deployment may require longer election timeout of 10-second. 145 // If true, local node does not need wait up to 10-second. Instead, 146 // forwards its election ticks to 8-second, and have only 2-second left 147 // before leader election. 148 // 149 // Major assumptions are that: 150 // - cluster has no active leader thus advancing ticks enables faster 151 // leader election, or 152 // - cluster already has an established leader, and rejoining follower 153 // is likely to receive heartbeats from the leader after tick advance 154 // and before election timeout. 155 // 156 // However, when network from leader to rejoining follower is congested, 157 // and the follower does not receive leader heartbeat within left election 158 // ticks, disruptive election has to happen thus affecting cluster 159 // availabilities. 160 // 161 // Disabling this would slow down initial bootstrap process for cross 162 // datacenter deployments. Make your own tradeoffs by configuring 163 // --initial-election-tick-advance at the cost of slow initial bootstrap. 164 // 165 // If single-node, it advances ticks regardless. 166 // 167 // See https://github.com/etcd-io/etcd/issues/9333 for more detail. 168 InitialElectionTickAdvance bool `json:"initial-election-tick-advance"` 169 170 // BackendBatchInterval is the maximum time before commit the backend transaction. 171 BackendBatchInterval time.Duration `json:"backend-batch-interval"` 172 // BackendBatchLimit is the maximum operations before commit the backend transaction. 173 BackendBatchLimit int `json:"backend-batch-limit"` 174 // BackendFreelistType specifies the type of freelist that boltdb backend uses (array and map are supported types). 175 BackendFreelistType string `json:"backend-bbolt-freelist-type"` 176 QuotaBackendBytes int64 `json:"quota-backend-bytes"` 177 MaxTxnOps uint `json:"max-txn-ops"` 178 MaxRequestBytes uint `json:"max-request-bytes"` 179 180 LPUrls, LCUrls []url.URL 181 APUrls, ACUrls []url.URL 182 ClientTLSInfo transport.TLSInfo 183 ClientAutoTLS bool 184 PeerTLSInfo transport.TLSInfo 185 PeerAutoTLS bool 186 187 // CipherSuites is a list of supported TLS cipher suites between 188 // client/server and peers. If empty, Go auto-populates the list. 189 // Note that cipher suites are prioritized in the given order. 190 CipherSuites []string `json:"cipher-suites"` 191 192 ClusterState string `json:"initial-cluster-state"` 193 DNSCluster string `json:"discovery-srv"` 194 DNSClusterServiceName string `json:"discovery-srv-name"` 195 Dproxy string `json:"discovery-proxy"` 196 Durl string `json:"discovery"` 197 InitialCluster string `json:"initial-cluster"` 198 InitialClusterToken string `json:"initial-cluster-token"` 199 StrictReconfigCheck bool `json:"strict-reconfig-check"` 200 EnableV2 bool `json:"enable-v2"` 201 202 // AutoCompactionMode is either 'periodic' or 'revision'. 203 AutoCompactionMode string `json:"auto-compaction-mode"` 204 // AutoCompactionRetention is either duration string with time unit 205 // (e.g. '5m' for 5-minute), or revision unit (e.g. '5000'). 206 // If no time unit is provided and compaction mode is 'periodic', 207 // the unit defaults to hour. For example, '5' translates into 5-hour. 208 AutoCompactionRetention string `json:"auto-compaction-retention"` 209 210 // GRPCKeepAliveMinTime is the minimum interval that a client should 211 // wait before pinging server. When client pings "too fast", server 212 // sends goaway and closes the connection (errors: too_many_pings, 213 // http2.ErrCodeEnhanceYourCalm). When too slow, nothing happens. 214 // Server expects client pings only when there is any active streams 215 // (PermitWithoutStream is set false). 216 GRPCKeepAliveMinTime time.Duration `json:"grpc-keepalive-min-time"` 217 // GRPCKeepAliveInterval is the frequency of server-to-client ping 218 // to check if a connection is alive. Close a non-responsive connection 219 // after an additional duration of Timeout. 0 to disable. 220 GRPCKeepAliveInterval time.Duration `json:"grpc-keepalive-interval"` 221 // GRPCKeepAliveTimeout is the additional duration of wait 222 // before closing a non-responsive connection. 0 to disable. 223 GRPCKeepAliveTimeout time.Duration `json:"grpc-keepalive-timeout"` 224 225 // PreVote is true to enable Raft Pre-Vote. 226 // If enabled, Raft runs an additional election phase 227 // to check whether it would get enough votes to win 228 // an election, thus minimizing disruptions. 229 // TODO: enable by default in 3.5. 230 PreVote bool `json:"pre-vote"` 231 232 CORS map[string]struct{} 233 234 // HostWhitelist lists acceptable hostnames from HTTP client requests. 235 // Client origin policy protects against "DNS Rebinding" attacks 236 // to insecure etcd servers. That is, any website can simply create 237 // an authorized DNS name, and direct DNS to "localhost" (or any 238 // other address). Then, all HTTP endpoints of etcd server listening 239 // on "localhost" becomes accessible, thus vulnerable to DNS rebinding 240 // attacks. See "CVE-2018-5702" for more detail. 241 // 242 // 1. If client connection is secure via HTTPS, allow any hostnames. 243 // 2. If client connection is not secure and "HostWhitelist" is not empty, 244 // only allow HTTP requests whose Host field is listed in whitelist. 245 // 246 // Note that the client origin policy is enforced whether authentication 247 // is enabled or not, for tighter controls. 248 // 249 // By default, "HostWhitelist" is "*", which allows any hostnames. 250 // Note that when specifying hostnames, loopback addresses are not added 251 // automatically. To allow loopback interfaces, leave it empty or set it "*", 252 // or add them to whitelist manually (e.g. "localhost", "127.0.0.1", etc.). 253 // 254 // CVE-2018-5702 reference: 255 // - https://bugs.chromium.org/p/project-zero/issues/detail?id=1447#c2 256 // - https://github.com/transmission/transmission/pull/468 257 // - https://github.com/etcd-io/etcd/issues/9353 258 HostWhitelist map[string]struct{} 259 260 // UserHandlers is for registering users handlers and only used for 261 // embedding etcd into other applications. 262 // The map key is the route path for the handler, and 263 // you must ensure it can't be conflicted with etcd's. 264 UserHandlers map[string]http.Handler `json:"-"` 265 // ServiceRegister is for registering users' gRPC services. A simple usage example: 266 // cfg := embed.NewConfig() 267 // cfg.ServerRegister = func(s *grpc.Server) { 268 // pb.RegisterFooServer(s, &fooServer{}) 269 // pb.RegisterBarServer(s, &barServer{}) 270 // } 271 // embed.StartEtcd(cfg) 272 ServiceRegister func(*grpc.Server) `json:"-"` 273 274 AuthToken string `json:"auth-token"` 275 BcryptCost uint `json:"bcrypt-cost"` 276 277 ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` 278 ExperimentalCorruptCheckTime time.Duration `json:"experimental-corrupt-check-time"` 279 ExperimentalEnableV2V3 string `json:"experimental-enable-v2v3"` 280 // ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. 281 ExperimentalEnableLeaseCheckpoint bool `json:"experimental-enable-lease-checkpoint"` 282 ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"` 283 284 // ForceNewCluster starts a new cluster even if previously started; unsafe. 285 ForceNewCluster bool `json:"force-new-cluster"` 286 287 EnablePprof bool `json:"enable-pprof"` 288 Metrics string `json:"metrics"` 289 ListenMetricsUrls []url.URL 290 ListenMetricsUrlsJSON string `json:"listen-metrics-urls"` 291 292 // Logger is logger options: currently only supports "zap". 293 // "capnslog" is removed in v3.5. 294 Logger string `json:"logger"` 295 // LogLevel configures log level. Only supports debug, info, warn, error, panic, or fatal. Default 'info'. 296 LogLevel string `json:"log-level"` 297 // LogOutputs is either: 298 // - "default" as os.Stderr, 299 // - "stderr" as os.Stderr, 300 // - "stdout" as os.Stdout, 301 // - file path to append server logs to. 302 // It can be multiple when "Logger" is zap. 303 LogOutputs []string `json:"log-outputs"` 304 305 // ZapLoggerBuilder is used to build the zap logger. 306 ZapLoggerBuilder func(*Config) error 307 308 // logger logs server-side operations. The default is nil, 309 // and "setupLogging" must be called before starting server. 310 // Do not set logger directly. 311 loggerMu *sync.RWMutex 312 logger *zap.Logger 313 314 // loggerConfig is server logger configuration for Raft logger. 315 // Must be either: "loggerConfig != nil" or "loggerCore != nil && loggerWriteSyncer != nil". 316 loggerConfig *zap.Config 317 // loggerCore is "zapcore.Core" for raft logger. 318 // Must be either: "loggerConfig != nil" or "loggerCore != nil && loggerWriteSyncer != nil". 319 loggerCore zapcore.Core 320 loggerWriteSyncer zapcore.WriteSyncer 321 322 // EnableGRPCGateway is false to disable grpc gateway. 323 EnableGRPCGateway bool `json:"enable-grpc-gateway"` 324} 325 326// configYAML holds the config suitable for yaml parsing 327type configYAML struct { 328 Config 329 configJSON 330} 331 332// configJSON has file options that are translated into Config options 333type configJSON struct { 334 LPUrlsJSON string `json:"listen-peer-urls"` 335 LCUrlsJSON string `json:"listen-client-urls"` 336 APUrlsJSON string `json:"initial-advertise-peer-urls"` 337 ACUrlsJSON string `json:"advertise-client-urls"` 338 339 CORSJSON string `json:"cors"` 340 HostWhitelistJSON string `json:"host-whitelist"` 341 342 ClientSecurityJSON securityConfig `json:"client-transport-security"` 343 PeerSecurityJSON securityConfig `json:"peer-transport-security"` 344} 345 346type securityConfig struct { 347 CertFile string `json:"cert-file"` 348 KeyFile string `json:"key-file"` 349 CertAuth bool `json:"client-cert-auth"` 350 TrustedCAFile string `json:"trusted-ca-file"` 351 AutoTLS bool `json:"auto-tls"` 352} 353 354// NewConfig creates a new Config populated with default values. 355func NewConfig() *Config { 356 lpurl, _ := url.Parse(DefaultListenPeerURLs) 357 apurl, _ := url.Parse(DefaultInitialAdvertisePeerURLs) 358 lcurl, _ := url.Parse(DefaultListenClientURLs) 359 acurl, _ := url.Parse(DefaultAdvertiseClientURLs) 360 cfg := &Config{ 361 MaxSnapFiles: DefaultMaxSnapshots, 362 MaxWalFiles: DefaultMaxWALs, 363 364 Name: DefaultName, 365 366 SnapshotCount: etcdserver.DefaultSnapshotCount, 367 SnapshotCatchUpEntries: etcdserver.DefaultSnapshotCatchUpEntries, 368 369 MaxTxnOps: DefaultMaxTxnOps, 370 MaxRequestBytes: DefaultMaxRequestBytes, 371 372 GRPCKeepAliveMinTime: DefaultGRPCKeepAliveMinTime, 373 GRPCKeepAliveInterval: DefaultGRPCKeepAliveInterval, 374 GRPCKeepAliveTimeout: DefaultGRPCKeepAliveTimeout, 375 376 TickMs: 100, 377 ElectionMs: 1000, 378 InitialElectionTickAdvance: true, 379 380 LPUrls: []url.URL{*lpurl}, 381 LCUrls: []url.URL{*lcurl}, 382 APUrls: []url.URL{*apurl}, 383 ACUrls: []url.URL{*acurl}, 384 385 ClusterState: ClusterStateFlagNew, 386 InitialClusterToken: "etcd-cluster", 387 388 StrictReconfigCheck: DefaultStrictReconfigCheck, 389 Metrics: "basic", 390 EnableV2: DefaultEnableV2, 391 392 CORS: map[string]struct{}{"*": {}}, 393 HostWhitelist: map[string]struct{}{"*": {}}, 394 395 AuthToken: "simple", 396 BcryptCost: uint(bcrypt.DefaultCost), 397 398 PreVote: false, // TODO: enable by default in v3.5 399 400 loggerMu: new(sync.RWMutex), 401 logger: nil, 402 Logger: "zap", 403 LogOutputs: []string{DefaultLogOutput}, 404 LogLevel: logutil.DefaultLogLevel, 405 } 406 cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) 407 return cfg 408} 409 410func ConfigFromFile(path string) (*Config, error) { 411 cfg := &configYAML{Config: *NewConfig()} 412 if err := cfg.configFromFile(path); err != nil { 413 return nil, err 414 } 415 return &cfg.Config, nil 416} 417 418func (cfg *configYAML) configFromFile(path string) error { 419 b, err := ioutil.ReadFile(path) 420 if err != nil { 421 return err 422 } 423 424 defaultInitialCluster := cfg.InitialCluster 425 426 err = yaml.Unmarshal(b, cfg) 427 if err != nil { 428 return err 429 } 430 431 if cfg.LPUrlsJSON != "" { 432 u, err := types.NewURLs(strings.Split(cfg.LPUrlsJSON, ",")) 433 if err != nil { 434 fmt.Fprintf(os.Stderr, "unexpected error setting up listen-peer-urls: %v\n", err) 435 os.Exit(1) 436 } 437 cfg.LPUrls = []url.URL(u) 438 } 439 440 if cfg.LCUrlsJSON != "" { 441 u, err := types.NewURLs(strings.Split(cfg.LCUrlsJSON, ",")) 442 if err != nil { 443 fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-urls: %v\n", err) 444 os.Exit(1) 445 } 446 cfg.LCUrls = []url.URL(u) 447 } 448 449 if cfg.APUrlsJSON != "" { 450 u, err := types.NewURLs(strings.Split(cfg.APUrlsJSON, ",")) 451 if err != nil { 452 fmt.Fprintf(os.Stderr, "unexpected error setting up initial-advertise-peer-urls: %v\n", err) 453 os.Exit(1) 454 } 455 cfg.APUrls = []url.URL(u) 456 } 457 458 if cfg.ACUrlsJSON != "" { 459 u, err := types.NewURLs(strings.Split(cfg.ACUrlsJSON, ",")) 460 if err != nil { 461 fmt.Fprintf(os.Stderr, "unexpected error setting up advertise-peer-urls: %v\n", err) 462 os.Exit(1) 463 } 464 cfg.ACUrls = []url.URL(u) 465 } 466 467 if cfg.ListenMetricsUrlsJSON != "" { 468 u, err := types.NewURLs(strings.Split(cfg.ListenMetricsUrlsJSON, ",")) 469 if err != nil { 470 fmt.Fprintf(os.Stderr, "unexpected error setting up listen-metrics-urls: %v\n", err) 471 os.Exit(1) 472 } 473 cfg.ListenMetricsUrls = []url.URL(u) 474 } 475 476 if cfg.CORSJSON != "" { 477 uv := flags.NewUniqueURLsWithExceptions(cfg.CORSJSON, "*") 478 cfg.CORS = uv.Values 479 } 480 481 if cfg.HostWhitelistJSON != "" { 482 uv := flags.NewUniqueStringsValue(cfg.HostWhitelistJSON) 483 cfg.HostWhitelist = uv.Values 484 } 485 486 // If a discovery flag is set, clear default initial cluster set by InitialClusterFromName 487 if (cfg.Durl != "" || cfg.DNSCluster != "") && cfg.InitialCluster == defaultInitialCluster { 488 cfg.InitialCluster = "" 489 } 490 if cfg.ClusterState == "" { 491 cfg.ClusterState = ClusterStateFlagNew 492 } 493 494 copySecurityDetails := func(tls *transport.TLSInfo, ysc *securityConfig) { 495 tls.CertFile = ysc.CertFile 496 tls.KeyFile = ysc.KeyFile 497 tls.ClientCertAuth = ysc.CertAuth 498 tls.TrustedCAFile = ysc.TrustedCAFile 499 } 500 copySecurityDetails(&cfg.ClientTLSInfo, &cfg.ClientSecurityJSON) 501 copySecurityDetails(&cfg.PeerTLSInfo, &cfg.PeerSecurityJSON) 502 cfg.ClientAutoTLS = cfg.ClientSecurityJSON.AutoTLS 503 cfg.PeerAutoTLS = cfg.PeerSecurityJSON.AutoTLS 504 505 return cfg.Validate() 506} 507 508func updateCipherSuites(tls *transport.TLSInfo, ss []string) error { 509 if len(tls.CipherSuites) > 0 && len(ss) > 0 { 510 return fmt.Errorf("TLSInfo.CipherSuites is already specified (given %v)", ss) 511 } 512 if len(ss) > 0 { 513 cs := make([]uint16, len(ss)) 514 for i, s := range ss { 515 var ok bool 516 cs[i], ok = tlsutil.GetCipherSuite(s) 517 if !ok { 518 return fmt.Errorf("unexpected TLS cipher suite %q", s) 519 } 520 } 521 tls.CipherSuites = cs 522 } 523 return nil 524} 525 526// Validate ensures that '*embed.Config' fields are properly configured. 527func (cfg *Config) Validate() error { 528 if err := cfg.setupLogging(); err != nil { 529 return err 530 } 531 if err := checkBindURLs(cfg.LPUrls); err != nil { 532 return err 533 } 534 if err := checkBindURLs(cfg.LCUrls); err != nil { 535 return err 536 } 537 if err := checkBindURLs(cfg.ListenMetricsUrls); err != nil { 538 return err 539 } 540 if err := checkHostURLs(cfg.APUrls); err != nil { 541 addrs := cfg.getAPURLs() 542 return fmt.Errorf(`--initial-advertise-peer-urls %q must be "host:port" (%v)`, strings.Join(addrs, ","), err) 543 } 544 if err := checkHostURLs(cfg.ACUrls); err != nil { 545 addrs := cfg.getACURLs() 546 return fmt.Errorf(`--advertise-client-urls %q must be "host:port" (%v)`, strings.Join(addrs, ","), err) 547 } 548 // Check if conflicting flags are passed. 549 nSet := 0 550 for _, v := range []bool{cfg.Durl != "", cfg.InitialCluster != "", cfg.DNSCluster != ""} { 551 if v { 552 nSet++ 553 } 554 } 555 556 if cfg.ClusterState != ClusterStateFlagNew && cfg.ClusterState != ClusterStateFlagExisting { 557 return fmt.Errorf("unexpected clusterState %q", cfg.ClusterState) 558 } 559 560 if nSet > 1 { 561 return ErrConflictBootstrapFlags 562 } 563 564 if cfg.TickMs == 0 { 565 return fmt.Errorf("--heartbeat-interval must be >0 (set to %dms)", cfg.TickMs) 566 } 567 if cfg.ElectionMs == 0 { 568 return fmt.Errorf("--election-timeout must be >0 (set to %dms)", cfg.ElectionMs) 569 } 570 if 5*cfg.TickMs > cfg.ElectionMs { 571 return fmt.Errorf("--election-timeout[%vms] should be at least as 5 times as --heartbeat-interval[%vms]", cfg.ElectionMs, cfg.TickMs) 572 } 573 if cfg.ElectionMs > maxElectionMs { 574 return fmt.Errorf("--election-timeout[%vms] is too long, and should be set less than %vms", cfg.ElectionMs, maxElectionMs) 575 } 576 577 // check this last since proxying in etcdmain may make this OK 578 if cfg.LCUrls != nil && cfg.ACUrls == nil { 579 return ErrUnsetAdvertiseClientURLsFlag 580 } 581 582 switch cfg.AutoCompactionMode { 583 case "": 584 case CompactorModeRevision, CompactorModePeriodic: 585 default: 586 return fmt.Errorf("unknown auto-compaction-mode %q", cfg.AutoCompactionMode) 587 } 588 589 return nil 590} 591 592// PeerURLsMapAndToken sets up an initial peer URLsMap and cluster token for bootstrap or discovery. 593func (cfg *Config) PeerURLsMapAndToken(which string) (urlsmap types.URLsMap, token string, err error) { 594 token = cfg.InitialClusterToken 595 switch { 596 case cfg.Durl != "": 597 urlsmap = types.URLsMap{} 598 // If using discovery, generate a temporary cluster based on 599 // self's advertised peer URLs 600 urlsmap[cfg.Name] = cfg.APUrls 601 token = cfg.Durl 602 603 case cfg.DNSCluster != "": 604 clusterStrs, cerr := cfg.GetDNSClusterNames() 605 lg := cfg.logger 606 if cerr != nil { 607 lg.Warn("failed to resolve during SRV discovery", zap.Error(cerr)) 608 return nil, "", cerr 609 } 610 for _, s := range clusterStrs { 611 lg.Info("got bootstrap from DNS for etcd-server", zap.String("node", s)) 612 } 613 clusterStr := strings.Join(clusterStrs, ",") 614 if strings.Contains(clusterStr, "https://") && cfg.PeerTLSInfo.TrustedCAFile == "" { 615 cfg.PeerTLSInfo.ServerName = cfg.DNSCluster 616 } 617 urlsmap, err = types.NewURLsMap(clusterStr) 618 // only etcd member must belong to the discovered cluster. 619 // proxy does not need to belong to the discovered cluster. 620 if which == "etcd" { 621 if _, ok := urlsmap[cfg.Name]; !ok { 622 return nil, "", fmt.Errorf("cannot find local etcd member %q in SRV records", cfg.Name) 623 } 624 } 625 626 default: 627 // We're statically configured, and cluster has appropriately been set. 628 urlsmap, err = types.NewURLsMap(cfg.InitialCluster) 629 } 630 return urlsmap, token, err 631} 632 633// GetDNSClusterNames uses DNS SRV records to get a list of initial nodes for cluster bootstrapping. 634func (cfg *Config) GetDNSClusterNames() ([]string, error) { 635 var ( 636 clusterStrs []string 637 cerr error 638 serviceNameSuffix string 639 ) 640 if cfg.DNSClusterServiceName != "" { 641 serviceNameSuffix = "-" + cfg.DNSClusterServiceName 642 } 643 644 lg := cfg.GetLogger() 645 646 // Use both etcd-server-ssl and etcd-server for discovery. 647 // Combine the results if both are available. 648 clusterStrs, cerr = srv.GetCluster("https", "etcd-server-ssl"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.APUrls) 649 if cerr != nil { 650 clusterStrs = make([]string, 0) 651 } 652 lg.Info( 653 "get cluster for etcd-server-ssl SRV", 654 zap.String("service-scheme", "https"), 655 zap.String("service-name", "etcd-server-ssl"+serviceNameSuffix), 656 zap.String("server-name", cfg.Name), 657 zap.String("discovery-srv", cfg.DNSCluster), 658 zap.Strings("advertise-peer-urls", cfg.getAPURLs()), 659 zap.Strings("found-cluster", clusterStrs), 660 zap.Error(cerr), 661 ) 662 663 defaultHTTPClusterStrs, httpCerr := srv.GetCluster("http", "etcd-server"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.APUrls) 664 if httpCerr != nil { 665 clusterStrs = append(clusterStrs, defaultHTTPClusterStrs...) 666 } 667 lg.Info( 668 "get cluster for etcd-server SRV", 669 zap.String("service-scheme", "http"), 670 zap.String("service-name", "etcd-server"+serviceNameSuffix), 671 zap.String("server-name", cfg.Name), 672 zap.String("discovery-srv", cfg.DNSCluster), 673 zap.Strings("advertise-peer-urls", cfg.getAPURLs()), 674 zap.Strings("found-cluster", clusterStrs), 675 zap.Error(httpCerr), 676 ) 677 678 return clusterStrs, cerr 679} 680 681func (cfg Config) InitialClusterFromName(name string) (ret string) { 682 if len(cfg.APUrls) == 0 { 683 return "" 684 } 685 n := name 686 if name == "" { 687 n = DefaultName 688 } 689 for i := range cfg.APUrls { 690 ret = ret + "," + n + "=" + cfg.APUrls[i].String() 691 } 692 return ret[1:] 693} 694 695func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } 696func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } 697 698func (cfg Config) defaultPeerHost() bool { 699 return len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs 700} 701 702func (cfg Config) defaultClientHost() bool { 703 return len(cfg.ACUrls) == 1 && cfg.ACUrls[0].String() == DefaultAdvertiseClientURLs 704} 705 706func (cfg *Config) ClientSelfCert() (err error) { 707 if !cfg.ClientAutoTLS { 708 return nil 709 } 710 if !cfg.ClientTLSInfo.Empty() { 711 cfg.logger.Warn("ignoring client auto TLS since certs given") 712 return nil 713 } 714 chosts := make([]string, len(cfg.LCUrls)) 715 for i, u := range cfg.LCUrls { 716 chosts[i] = u.Host 717 } 718 cfg.ClientTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "client"), chosts) 719 if err != nil { 720 return err 721 } 722 return updateCipherSuites(&cfg.ClientTLSInfo, cfg.CipherSuites) 723} 724 725func (cfg *Config) PeerSelfCert() (err error) { 726 if !cfg.PeerAutoTLS { 727 return nil 728 } 729 if !cfg.PeerTLSInfo.Empty() { 730 cfg.logger.Warn("ignoring peer auto TLS since certs given") 731 return nil 732 } 733 phosts := make([]string, len(cfg.LPUrls)) 734 for i, u := range cfg.LPUrls { 735 phosts[i] = u.Host 736 } 737 cfg.PeerTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "peer"), phosts) 738 if err != nil { 739 return err 740 } 741 return updateCipherSuites(&cfg.PeerTLSInfo, cfg.CipherSuites) 742} 743 744// UpdateDefaultClusterFromName updates cluster advertise URLs with, if available, default host, 745// if advertise URLs are default values(localhost:2379,2380) AND if listen URL is 0.0.0.0. 746// e.g. advertise peer URL localhost:2380 or listen peer URL 0.0.0.0:2380 747// then the advertise peer host would be updated with machine's default host, 748// while keeping the listen URL's port. 749// User can work around this by explicitly setting URL with 127.0.0.1. 750// It returns the default hostname, if used, and the error, if any, from getting the machine's default host. 751// TODO: check whether fields are set instead of whether fields have default value 752func (cfg *Config) UpdateDefaultClusterFromName(defaultInitialCluster string) (string, error) { 753 if defaultHostname == "" || defaultHostStatus != nil { 754 // update 'initial-cluster' when only the name is specified (e.g. 'etcd --name=abc') 755 if cfg.Name != DefaultName && cfg.InitialCluster == defaultInitialCluster { 756 cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) 757 } 758 return "", defaultHostStatus 759 } 760 761 used := false 762 pip, pport := cfg.LPUrls[0].Hostname(), cfg.LPUrls[0].Port() 763 if cfg.defaultPeerHost() && pip == "0.0.0.0" { 764 cfg.APUrls[0] = url.URL{Scheme: cfg.APUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, pport)} 765 used = true 766 } 767 // update 'initial-cluster' when only the name is specified (e.g. 'etcd --name=abc') 768 if cfg.Name != DefaultName && cfg.InitialCluster == defaultInitialCluster { 769 cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) 770 } 771 772 cip, cport := cfg.LCUrls[0].Hostname(), cfg.LCUrls[0].Port() 773 if cfg.defaultClientHost() && cip == "0.0.0.0" { 774 cfg.ACUrls[0] = url.URL{Scheme: cfg.ACUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, cport)} 775 used = true 776 } 777 dhost := defaultHostname 778 if !used { 779 dhost = "" 780 } 781 return dhost, defaultHostStatus 782} 783 784// checkBindURLs returns an error if any URL uses a domain name. 785func checkBindURLs(urls []url.URL) error { 786 for _, url := range urls { 787 if url.Scheme == "unix" || url.Scheme == "unixs" { 788 continue 789 } 790 host, _, err := net.SplitHostPort(url.Host) 791 if err != nil { 792 return err 793 } 794 if host == "localhost" { 795 // special case for local address 796 // TODO: support /etc/hosts ? 797 continue 798 } 799 if net.ParseIP(host) == nil { 800 return fmt.Errorf("expected IP in URL for binding (%s)", url.String()) 801 } 802 } 803 return nil 804} 805 806func checkHostURLs(urls []url.URL) error { 807 for _, url := range urls { 808 host, _, err := net.SplitHostPort(url.Host) 809 if err != nil { 810 return err 811 } 812 if host == "" { 813 return fmt.Errorf("unexpected empty host (%s)", url.String()) 814 } 815 } 816 return nil 817} 818 819func (cfg *Config) getAPURLs() (ss []string) { 820 ss = make([]string, len(cfg.APUrls)) 821 for i := range cfg.APUrls { 822 ss[i] = cfg.APUrls[i].String() 823 } 824 return ss 825} 826 827func (cfg *Config) getLPURLs() (ss []string) { 828 ss = make([]string, len(cfg.LPUrls)) 829 for i := range cfg.LPUrls { 830 ss[i] = cfg.LPUrls[i].String() 831 } 832 return ss 833} 834 835func (cfg *Config) getACURLs() (ss []string) { 836 ss = make([]string, len(cfg.ACUrls)) 837 for i := range cfg.ACUrls { 838 ss[i] = cfg.ACUrls[i].String() 839 } 840 return ss 841} 842 843func (cfg *Config) getLCURLs() (ss []string) { 844 ss = make([]string, len(cfg.LCUrls)) 845 for i := range cfg.LCUrls { 846 ss[i] = cfg.LCUrls[i].String() 847 } 848 return ss 849} 850 851func (cfg *Config) getMetricsURLs() (ss []string) { 852 ss = make([]string, len(cfg.ListenMetricsUrls)) 853 for i := range cfg.ListenMetricsUrls { 854 ss[i] = cfg.ListenMetricsUrls[i].String() 855 } 856 return ss 857} 858 859func parseBackendFreelistType(freelistType string) bolt.FreelistType { 860 if freelistType == freelistArrayType { 861 return bolt.FreelistArrayType 862 } 863 864 return bolt.FreelistMapType 865} 866