1package envoy 2 3import ( 4 "errors" 5 "flag" 6 "fmt" 7 "io/ioutil" 8 "net" 9 "os" 10 "os/exec" 11 "strings" 12 13 "github.com/mitchellh/cli" 14 "github.com/mitchellh/mapstructure" 15 16 "github.com/hashicorp/consul/agent/structs" 17 "github.com/hashicorp/consul/agent/xds" 18 "github.com/hashicorp/consul/api" 19 proxyCmd "github.com/hashicorp/consul/command/connect/proxy" 20 "github.com/hashicorp/consul/command/flags" 21 "github.com/hashicorp/consul/ipaddr" 22) 23 24func New(ui cli.Ui) *cmd { 25 ui = &cli.PrefixedUi{ 26 OutputPrefix: "==> ", 27 InfoPrefix: " ", 28 ErrorPrefix: "==> ", 29 Ui: ui, 30 } 31 32 c := &cmd{UI: ui} 33 c.init() 34 return c 35} 36 37const DefaultAdminAccessLogPath = "/dev/null" 38 39type cmd struct { 40 UI cli.Ui 41 flags *flag.FlagSet 42 http *flags.HTTPFlags 43 help string 44 client *api.Client 45 46 // flags 47 meshGateway bool 48 gateway string 49 proxyID string 50 sidecarFor string 51 adminAccessLogPath string 52 adminBind string 53 envoyBin string 54 bootstrap bool 55 disableCentralConfig bool 56 grpcAddr string 57 envoyVersion string 58 59 // mesh gateway registration information 60 register bool 61 lanAddress ServiceAddressValue 62 wanAddress ServiceAddressValue 63 deregAfterCritical string 64 bindAddresses ServiceAddressMapValue 65 exposeServers bool 66 67 gatewaySvcName string 68 gatewayKind api.ServiceKind 69} 70 71const ( 72 defaultEnvoyVersion = "1.14.2" 73 meshGatewayVal = "mesh" 74) 75 76var supportedGateways = map[string]api.ServiceKind{ 77 "mesh": api.ServiceKindMeshGateway, 78 "terminating": api.ServiceKindTerminatingGateway, 79 "ingress": api.ServiceKindIngressGateway, 80} 81 82func (c *cmd) init() { 83 c.flags = flag.NewFlagSet("", flag.ContinueOnError) 84 85 c.flags.StringVar(&c.proxyID, "proxy-id", os.Getenv("CONNECT_PROXY_ID"), 86 "The proxy's ID on the local agent.") 87 88 // Deprecated in favor of `gateway` 89 c.flags.BoolVar(&c.meshGateway, "mesh-gateway", false, 90 "Configure Envoy as a Mesh Gateway.") 91 92 c.flags.StringVar(&c.gateway, "gateway", "", 93 "The type of gateway to register. One of: terminating, ingress, or mesh") 94 95 c.flags.StringVar(&c.sidecarFor, "sidecar-for", os.Getenv("CONNECT_SIDECAR_FOR"), 96 "The ID of a service instance on the local agent that this proxy should "+ 97 "become a sidecar for. It requires that the proxy service is registered "+ 98 "with the agent as a connect-proxy with Proxy.DestinationServiceID set "+ 99 "to this value. If more than one such proxy is registered it will fail.") 100 101 c.flags.StringVar(&c.envoyBin, "envoy-binary", "", 102 "The full path to the envoy binary to run. By default will just search "+ 103 "$PATH. Ignored if -bootstrap is used.") 104 105 c.flags.StringVar(&c.adminAccessLogPath, "admin-access-log-path", DefaultAdminAccessLogPath, 106 fmt.Sprintf("The path to write the access log for the administration server. If no access "+ 107 "log is desired specify %q. By default it will use %q.", 108 DefaultAdminAccessLogPath, DefaultAdminAccessLogPath)) 109 110 c.flags.StringVar(&c.adminBind, "admin-bind", "localhost:19000", 111 "The address:port to start envoy's admin server on. Envoy requires this "+ 112 "but care must be taken to ensure it's not exposed to an untrusted network "+ 113 "as it has full control over the secrets and config of the proxy.") 114 115 c.flags.BoolVar(&c.bootstrap, "bootstrap", false, 116 "Generate the bootstrap.json but don't exec envoy") 117 118 c.flags.BoolVar(&c.disableCentralConfig, "no-central-config", false, 119 "By default the proxy's bootstrap configuration can be customized "+ 120 "centrally. This requires that the command run on the same agent as the "+ 121 "proxy will and that the agent is reachable when the command is run. In "+ 122 "cases where either assumption is violated this flag will prevent the "+ 123 "command attempting to resolve config from the local agent.") 124 125 c.flags.StringVar(&c.grpcAddr, "grpc-addr", os.Getenv(api.GRPCAddrEnvName), 126 "Set the agent's gRPC address and port (in http(s)://host:port format). "+ 127 "Alternatively, you can specify CONSUL_GRPC_ADDR in ENV.") 128 129 c.flags.StringVar(&c.envoyVersion, "envoy-version", defaultEnvoyVersion, 130 "Sets the envoy-version that the envoy binary has.") 131 132 c.flags.BoolVar(&c.register, "register", false, 133 "Register a new gateway service before configuring and starting Envoy") 134 135 c.flags.Var(&c.lanAddress, "address", 136 "LAN address to advertise in the gateway service registration") 137 138 c.flags.Var(&c.wanAddress, "wan-address", 139 "WAN address to advertise in the gateway service registration. For ingress gateways, "+ 140 "only an IP address (without a port) is required.") 141 142 c.flags.Var(&c.bindAddresses, "bind-address", "Bind "+ 143 "address to use instead of the default binding rules given as `<name>=<ip>:<port>` "+ 144 "pairs. This flag may be specified multiple times to add multiple bind addresses.") 145 146 c.flags.StringVar(&c.gatewaySvcName, "service", "", 147 "Service name to use for the registration") 148 149 c.flags.BoolVar(&c.exposeServers, "expose-servers", false, 150 "Expose the servers for WAN federation via this mesh gateway") 151 152 c.flags.StringVar(&c.deregAfterCritical, "deregister-after-critical", "6h", 153 "The amount of time the gateway services health check can be failing before being deregistered") 154 155 c.http = &flags.HTTPFlags{} 156 flags.Merge(c.flags, c.http.ClientFlags()) 157 flags.Merge(c.flags, c.http.NamespaceFlags()) 158 c.help = flags.Usage(help, c.flags) 159} 160 161// canBindInternal is here mainly so we can unit test this with a constant net.Addr list 162func canBindInternal(addr string, ifAddrs []net.Addr) bool { 163 if addr == "" { 164 return false 165 } 166 167 ip := net.ParseIP(addr) 168 if ip == nil { 169 return false 170 } 171 172 ipStr := ip.String() 173 174 for _, addr := range ifAddrs { 175 switch v := addr.(type) { 176 case *net.IPNet: 177 if v.IP.String() == ipStr { 178 return true 179 } 180 default: 181 if addr.String() == ipStr { 182 return true 183 } 184 } 185 } 186 187 return false 188} 189 190func canBind(addr api.ServiceAddress) bool { 191 ifAddrs, err := net.InterfaceAddrs() 192 if err != nil { 193 return false 194 } 195 196 return canBindInternal(addr.Address, ifAddrs) 197} 198 199func (c *cmd) Run(args []string) int { 200 if err := c.flags.Parse(args); err != nil { 201 return 1 202 } 203 204 // Setup Consul client 205 var err error 206 c.client, err = c.http.APIClient() 207 if err != nil { 208 c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) 209 return 1 210 } 211 // TODO: refactor 212 return c.run(c.flags.Args()) 213} 214 215func (c *cmd) run(args []string) int { 216 // Fixup for deprecated mesh-gateway flag 217 if c.meshGateway && c.gateway != "" { 218 c.UI.Error("The mesh-gateway flag is deprecated and cannot be used alongside the gateway flag") 219 return 1 220 } 221 222 if c.meshGateway { 223 c.gateway = meshGatewayVal 224 } 225 226 if c.exposeServers { 227 if c.gateway != meshGatewayVal { 228 c.UI.Error("'-expose-servers' can only be used for mesh gateways") 229 return 1 230 } 231 if !c.register { 232 c.UI.Error("'-expose-servers' requires '-register'") 233 return 1 234 } 235 } 236 237 // Gateway kind is set so that it is available even if not auto-registering the gateway 238 if c.gateway != "" { 239 kind, ok := supportedGateways[c.gateway] 240 if !ok { 241 c.UI.Error("Gateway must be one of: terminating, mesh, or ingress") 242 return 1 243 } 244 c.gatewayKind = kind 245 246 if c.gatewaySvcName == "" { 247 c.gatewaySvcName = string(c.gatewayKind) 248 } 249 } 250 251 if c.proxyID == "" { 252 switch { 253 case c.sidecarFor != "": 254 proxyID, err := proxyCmd.LookupProxyIDForSidecar(c.client, c.sidecarFor) 255 if err != nil { 256 c.UI.Error(err.Error()) 257 return 1 258 } 259 c.proxyID = proxyID 260 261 case c.gateway != "" && !c.register: 262 gatewaySvc, err := proxyCmd.LookupGatewayProxy(c.client, c.gatewayKind) 263 if err != nil { 264 c.UI.Error(err.Error()) 265 return 1 266 } 267 c.proxyID = gatewaySvc.ID 268 c.gatewaySvcName = gatewaySvc.Service 269 270 case c.gateway != "" && c.register: 271 c.proxyID = c.gatewaySvcName 272 273 } 274 } 275 if c.proxyID == "" { 276 c.UI.Error("No proxy ID specified. One of -proxy-id, -sidecar-for, or -gateway is " + 277 "required") 278 return 1 279 } 280 281 if c.register { 282 if c.gateway == "" { 283 c.UI.Error("Auto-Registration can only be used for gateways") 284 return 1 285 } 286 287 taggedAddrs := make(map[string]api.ServiceAddress) 288 lanAddr := c.lanAddress.Value() 289 if lanAddr.Address != "" { 290 taggedAddrs[structs.TaggedAddressLAN] = lanAddr 291 } 292 293 wanAddr := c.wanAddress.Value() 294 if wanAddr.Address != "" { 295 taggedAddrs[structs.TaggedAddressWAN] = wanAddr 296 } 297 298 tcpCheckAddr := lanAddr.Address 299 if tcpCheckAddr == "" { 300 // fallback to localhost as the gateway has to reside in the same network namespace 301 // as the agent 302 tcpCheckAddr = "127.0.0.1" 303 } 304 305 var proxyConf *api.AgentServiceConnectProxyConfig 306 if len(c.bindAddresses.value) > 0 { 307 // override all default binding rules and just bind to the user-supplied addresses 308 proxyConf = &api.AgentServiceConnectProxyConfig{ 309 Config: map[string]interface{}{ 310 "envoy_gateway_no_default_bind": true, 311 "envoy_gateway_bind_addresses": c.bindAddresses.value, 312 }, 313 } 314 } else if canBind(lanAddr) && canBind(wanAddr) { 315 // when both addresses are bindable then we bind to the tagged addresses 316 // for creating the envoy listeners 317 proxyConf = &api.AgentServiceConnectProxyConfig{ 318 Config: map[string]interface{}{ 319 "envoy_gateway_no_default_bind": true, 320 "envoy_gateway_bind_tagged_addresses": true, 321 }, 322 } 323 } else if !canBind(lanAddr) && lanAddr.Address != "" { 324 c.UI.Error(fmt.Sprintf("The LAN address %q will not be bindable. Either set a bindable address or override the bind addresses with -bind-address", lanAddr.Address)) 325 return 1 326 } 327 328 var meta map[string]string 329 if c.exposeServers { 330 meta = map[string]string{structs.MetaWANFederationKey: "1"} 331 } 332 333 svc := api.AgentServiceRegistration{ 334 Kind: c.gatewayKind, 335 Name: c.gatewaySvcName, 336 ID: c.proxyID, 337 Address: lanAddr.Address, 338 Port: lanAddr.Port, 339 Meta: meta, 340 TaggedAddresses: taggedAddrs, 341 Proxy: proxyConf, 342 Check: &api.AgentServiceCheck{ 343 Name: fmt.Sprintf("%s listening", c.gatewayKind), 344 TCP: ipaddr.FormatAddressPort(tcpCheckAddr, lanAddr.Port), 345 Interval: "10s", 346 DeregisterCriticalServiceAfter: c.deregAfterCritical, 347 }, 348 } 349 350 if err := c.client.Agent().ServiceRegister(&svc); err != nil { 351 c.UI.Error(fmt.Sprintf("Error registering service %q: %s", svc.Name, err)) 352 return 1 353 } 354 355 c.UI.Output(fmt.Sprintf("Registered service: %s", svc.Name)) 356 } 357 358 // Generate config 359 bootstrapJson, err := c.generateConfig() 360 if err != nil { 361 c.UI.Error(err.Error()) 362 return 1 363 } 364 365 if c.bootstrap { 366 // Just output it and we are done 367 os.Stdout.Write(bootstrapJson) 368 return 0 369 } 370 371 // Find Envoy binary 372 binary, err := c.findBinary() 373 if err != nil { 374 c.UI.Error("Couldn't find envoy binary: " + err.Error()) 375 return 1 376 } 377 378 err = execEnvoy(binary, nil, args, bootstrapJson) 379 if err == errUnsupportedOS { 380 c.UI.Error("Directly running Envoy is only supported on linux and macOS " + 381 "since envoy itself doesn't build on other platforms currently.") 382 c.UI.Error("Use the -bootstrap option to generate the JSON to use when running envoy " + 383 "on a supported OS or via a container or VM.") 384 return 1 385 } else if err != nil { 386 c.UI.Error(err.Error()) 387 return 1 388 } 389 390 return 0 391} 392 393var errUnsupportedOS = errors.New("envoy: not implemented on this operating system") 394 395func (c *cmd) findBinary() (string, error) { 396 if c.envoyBin != "" { 397 return c.envoyBin, nil 398 } 399 return exec.LookPath("envoy") 400} 401 402func (c *cmd) templateArgs() (*BootstrapTplArgs, error) { 403 httpCfg := api.DefaultConfig() 404 c.http.MergeOntoConfig(httpCfg) 405 406 // api.NewClient normalizes some values (Token, Scheme) on the Config. 407 if _, err := api.NewClient(httpCfg); err != nil { 408 return nil, err 409 } 410 411 grpcAddr, err := c.grpcAddress(httpCfg) 412 if err != nil { 413 return nil, err 414 } 415 416 adminAddr, adminPort, err := net.SplitHostPort(c.adminBind) 417 if err != nil { 418 return nil, fmt.Errorf("Invalid Consul HTTP address: %s", err) 419 } 420 421 // Envoy requires IP addresses to bind too when using static so resolve DNS or 422 // localhost here. 423 adminBindIP, err := net.ResolveIPAddr("ip", adminAddr) 424 if err != nil { 425 return nil, fmt.Errorf("Failed to resolve admin bind address: %s", err) 426 } 427 428 // Ideally the cluster should be the service name. We may or may not have that 429 // yet depending on the arguments used so make a best effort here. In the 430 // common case, even if the command was invoked with proxy-id and we don't 431 // know service name yet, we will after we resolve the proxy's config in a bit 432 // and will update this then. 433 cluster := c.proxyID 434 if c.sidecarFor != "" { 435 cluster = c.sidecarFor 436 } else if c.gateway != "" && c.gatewaySvcName != "" { 437 cluster = c.gatewaySvcName 438 } 439 440 adminAccessLogPath := c.adminAccessLogPath 441 if adminAccessLogPath == "" { 442 adminAccessLogPath = DefaultAdminAccessLogPath 443 } 444 445 var caPEM string 446 if httpCfg.TLSConfig.CAFile != "" { 447 content, err := ioutil.ReadFile(httpCfg.TLSConfig.CAFile) 448 if err != nil { 449 return nil, fmt.Errorf("Failed to read CA file: %s", err) 450 } 451 caPEM = strings.Replace(string(content), "\n", "\\n", -1) 452 } 453 454 return &BootstrapTplArgs{ 455 GRPC: grpcAddr, 456 ProxyCluster: cluster, 457 ProxyID: c.proxyID, 458 AgentCAPEM: caPEM, 459 AdminAccessLogPath: adminAccessLogPath, 460 AdminBindAddress: adminBindIP.String(), 461 AdminBindPort: adminPort, 462 Token: httpCfg.Token, 463 LocalAgentClusterName: xds.LocalAgentClusterName, 464 Namespace: httpCfg.Namespace, 465 EnvoyVersion: c.envoyVersion, 466 }, nil 467} 468 469func (c *cmd) generateConfig() ([]byte, error) { 470 args, err := c.templateArgs() 471 if err != nil { 472 return nil, err 473 } 474 475 var bsCfg BootstrapConfig 476 477 // Setup ready listener for ingress gateway to pass healthcheck 478 if c.gatewayKind == api.ServiceKindIngressGateway { 479 lanAddr := c.lanAddress.String() 480 // Deal with possibility of address not being specified and defaulting to 481 // ":443" 482 if strings.HasPrefix(lanAddr, ":") { 483 lanAddr = "127.0.0.1" + lanAddr 484 } 485 bsCfg.ReadyBindAddr = lanAddr 486 } 487 488 if !c.disableCentralConfig { 489 // Fetch any customization from the registration 490 svc, _, err := c.client.Agent().Service(c.proxyID, nil) 491 if err != nil { 492 return nil, fmt.Errorf("failed fetch proxy config from local agent: %s", err) 493 } 494 495 if svc.Proxy == nil { 496 return nil, errors.New("service is not a Connect proxy or gateway") 497 } 498 499 // Parse the bootstrap config 500 if err := mapstructure.WeakDecode(svc.Proxy.Config, &bsCfg); err != nil { 501 return nil, fmt.Errorf("failed parsing Proxy.Config: %s", err) 502 } 503 504 if svc.Proxy.DestinationServiceName != "" { 505 // Override cluster now we know the actual service name 506 args.ProxyCluster = svc.Proxy.DestinationServiceName 507 } 508 } 509 510 return bsCfg.GenerateJSON(args) 511} 512 513// TODO: make method a function 514func (c *cmd) grpcAddress(httpCfg *api.Config) (GRPC, error) { 515 g := GRPC{} 516 517 addr := c.grpcAddr 518 // See if we need to lookup grpcAddr 519 if addr == "" { 520 port, err := c.lookupGRPCPort() 521 if err != nil { 522 c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) 523 } 524 if port <= 0 { 525 // This is the dev mode default and recommended production setting if 526 // enabled. 527 port = 8502 528 c.UI.Info(fmt.Sprintf("Defaulting to grpc port = %d", port)) 529 } 530 addr = fmt.Sprintf("localhost:%v", port) 531 } 532 533 // TODO: parse addr as a url instead of strings.HasPrefix/TrimPrefix 534 535 // Decide on TLS if the scheme is provided and indicates it, if the HTTP env 536 // suggests TLS is supported explicitly (CONSUL_HTTP_SSL) or implicitly 537 // (CONSUL_HTTP_ADDR) is https:// 538 switch { 539 case strings.HasPrefix(strings.ToLower(addr), "https://"): 540 g.AgentTLS = true 541 case httpCfg.Scheme == "https": 542 g.AgentTLS = true 543 } 544 545 // We want to allow grpcAddr set as host:port with no scheme but if the host 546 // is an IP this will fail to parse as a URL with "parse 127.0.0.1:8500: first 547 // path segment in URL cannot contain colon". On the other hand we also 548 // support both http(s)://host:port and unix:///path/to/file. 549 if grpcAddr := strings.TrimPrefix(addr, "unix://"); grpcAddr != addr { 550 // Path to unix socket 551 g.AgentSocket = grpcAddr 552 } else { 553 // Parse as host:port with option http prefix 554 grpcAddr = strings.TrimPrefix(addr, "http://") 555 grpcAddr = strings.TrimPrefix(grpcAddr, "https://") 556 557 var err error 558 var host string 559 host, g.AgentPort, err = net.SplitHostPort(grpcAddr) 560 if err != nil { 561 return g, fmt.Errorf("Invalid Consul HTTP address: %s", err) 562 } 563 564 // We use STATIC for agent which means we need to resolve DNS names like 565 // `localhost` ourselves. We could use STRICT_DNS or LOGICAL_DNS with envoy 566 // but Envoy resolves `localhost` differently to go on macOS at least which 567 // causes paper cuts like default dev agent (which binds specifically to 568 // 127.0.0.1) isn't reachable since Envoy resolves localhost to `[::]` and 569 // can't connect. 570 agentIP, err := net.ResolveIPAddr("ip", host) 571 if err != nil { 572 return g, fmt.Errorf("Failed to resolve agent address: %s", err) 573 } 574 g.AgentAddress = agentIP.String() 575 } 576 return g, nil 577} 578 579func (c *cmd) lookupGRPCPort() (int, error) { 580 self, err := c.client.Agent().Self() 581 if err != nil { 582 return 0, err 583 } 584 cfg, ok := self["DebugConfig"] 585 if !ok { 586 return 0, fmt.Errorf("unexpected agent response: no debug config") 587 } 588 port, ok := cfg["GRPCPort"] 589 if !ok { 590 return 0, fmt.Errorf("agent does not have grpc port enabled") 591 } 592 portN, ok := port.(float64) 593 if !ok { 594 return 0, fmt.Errorf("invalid grpc port in agent response") 595 } 596 597 return int(portN), nil 598} 599 600func (c *cmd) Synopsis() string { 601 return synopsis 602} 603 604func (c *cmd) Help() string { 605 return c.help 606} 607 608const synopsis = "Runs or Configures Envoy as a Connect proxy" 609const help = ` 610Usage: consul connect envoy [options] 611 612 Generates the bootstrap configuration needed to start an Envoy proxy instance 613 for use as a Connect sidecar for a particular service instance. By default it 614 will generate the config and then exec Envoy directly until it exits normally. 615 616 It will search $PATH for the envoy binary but this can be overridden with 617 -envoy-binary. 618 619 It can instead only generate the bootstrap.json based on the current ENV and 620 arguments using -bootstrap. 621 622 The proxy requires service:write permissions for the service it represents. 623 The token may be passed via the CLI or the CONSUL_HTTP_TOKEN environment 624 variable. 625 626 The example below shows how to start a local proxy as a sidecar to a "web" 627 service instance. It assumes that the proxy was already registered with it's 628 Config for example via a sidecar_service block. 629 630 $ consul connect envoy -sidecar-for web 631 632` 633