1package envoy 2 3import ( 4 "errors" 5 "flag" 6 "fmt" 7 "io/ioutil" 8 "net" 9 "os" 10 "os/exec" 11 "strconv" 12 "strings" 13 14 "github.com/mitchellh/mapstructure" 15 16 "github.com/hashicorp/consul/agent/structs" 17 "github.com/hashicorp/consul/agent/xds" 18 "github.com/hashicorp/consul/api" 19 proxyCmd "github.com/hashicorp/consul/command/connect/proxy" 20 "github.com/hashicorp/consul/command/flags" 21 "github.com/hashicorp/consul/ipaddr" 22 "github.com/hashicorp/go-sockaddr/template" 23 24 "github.com/mitchellh/cli" 25) 26 27func New(ui cli.Ui) *cmd { 28 ui = &cli.PrefixedUi{ 29 OutputPrefix: "==> ", 30 InfoPrefix: " ", 31 ErrorPrefix: "==> ", 32 Ui: ui, 33 } 34 35 c := &cmd{UI: ui} 36 c.init() 37 return c 38} 39 40const DefaultAdminAccessLogPath = "/dev/null" 41 42type cmd struct { 43 UI cli.Ui 44 flags *flag.FlagSet 45 http *flags.HTTPFlags 46 help string 47 client *api.Client 48 49 // flags 50 meshGateway bool 51 proxyID string 52 sidecarFor string 53 adminAccessLogPath string 54 adminBind string 55 envoyBin string 56 bootstrap bool 57 disableCentralConfig bool 58 grpcAddr string 59 envoyVersion string 60 61 // mesh gateway registration information 62 register bool 63 address string 64 wanAddress string 65 deregAfterCritical string 66 bindAddresses map[string]string 67 68 meshGatewaySvcName string 69} 70 71func (c *cmd) init() { 72 c.flags = flag.NewFlagSet("", flag.ContinueOnError) 73 74 c.flags.StringVar(&c.proxyID, "proxy-id", "", 75 "The proxy's ID on the local agent.") 76 77 c.flags.BoolVar(&c.meshGateway, "mesh-gateway", false, 78 "Generate the bootstrap.json but don't exec envoy") 79 80 c.flags.StringVar(&c.sidecarFor, "sidecar-for", "", 81 "The ID of a service instance on the local agent that this proxy should "+ 82 "become a sidecar for. It requires that the proxy service is registered "+ 83 "with the agent as a connect-proxy with Proxy.DestinationServiceID set "+ 84 "to this value. If more than one such proxy is registered it will fail.") 85 86 c.flags.StringVar(&c.envoyBin, "envoy-binary", "", 87 "The full path to the envoy binary to run. By default will just search "+ 88 "$PATH. Ignored if -bootstrap is used.") 89 90 c.flags.StringVar(&c.adminAccessLogPath, "admin-access-log-path", DefaultAdminAccessLogPath, 91 fmt.Sprintf("The path to write the access log for the administration server. If no access "+ 92 "log is desired specify %q. By default it will use %q.", 93 DefaultAdminAccessLogPath, DefaultAdminAccessLogPath)) 94 95 c.flags.StringVar(&c.adminBind, "admin-bind", "localhost:19000", 96 "The address:port to start envoy's admin server on. Envoy requires this "+ 97 "but care must be taken to ensure it's not exposed to an untrusted network "+ 98 "as it has full control over the secrets and config of the proxy.") 99 100 c.flags.BoolVar(&c.bootstrap, "bootstrap", false, 101 "Generate the bootstrap.json but don't exec envoy") 102 103 c.flags.BoolVar(&c.disableCentralConfig, "no-central-config", false, 104 "By default the proxy's bootstrap configuration can be customized "+ 105 "centrally. This requires that the command run on the same agent as the "+ 106 "proxy will and that the agent is reachable when the command is run. In "+ 107 "cases where either assumption is violated this flag will prevent the "+ 108 "command attempting to resolve config from the local agent.") 109 110 c.flags.StringVar(&c.grpcAddr, "grpc-addr", "", 111 "Set the agent's gRPC address and port (in http(s)://host:port format). "+ 112 "Alternatively, you can specify CONSUL_GRPC_ADDR in ENV.") 113 114 c.flags.StringVar(&c.envoyVersion, "envoy-version", "1.13.0", 115 "Sets the envoy-version that the envoy binary has.") 116 117 c.flags.BoolVar(&c.register, "register", false, 118 "Register a new Mesh Gateway service before configuring and starting Envoy") 119 120 c.flags.StringVar(&c.address, "address", "", 121 "LAN address to advertise in the Mesh Gateway service registration") 122 123 c.flags.StringVar(&c.wanAddress, "wan-address", "", 124 "WAN address to advertise in the Mesh Gateway service registration") 125 126 c.flags.Var((*flags.FlagMapValue)(&c.bindAddresses), "bind-address", "Bind "+ 127 "address to use instead of the default binding rules given as `<name>=<ip>:<port>` "+ 128 "pairs. This flag may be specified multiple times to add multiple bind addresses.") 129 130 c.flags.StringVar(&c.meshGatewaySvcName, "service", "mesh-gateway", 131 "Service name to use for the registration") 132 133 c.flags.StringVar(&c.deregAfterCritical, "deregister-after-critical", "6h", 134 "The amount of time the gateway services health check can be failing before being deregistered") 135 136 c.http = &flags.HTTPFlags{} 137 flags.Merge(c.flags, c.http.ClientFlags()) 138 flags.Merge(c.flags, c.http.NamespaceFlags()) 139 c.help = flags.Usage(help, c.flags) 140} 141 142const ( 143 DefaultMeshGatewayPort int = 443 144) 145 146func parseAddress(addrStr string) (string, int, error) { 147 if addrStr == "" { 148 // defaulting the port to 443 149 return "", DefaultMeshGatewayPort, nil 150 } 151 152 x, err := template.Parse(addrStr) 153 if err != nil { 154 return "", DefaultMeshGatewayPort, fmt.Errorf("Error parsing address %q: %v", addrStr, err) 155 } 156 157 addr, portStr, err := net.SplitHostPort(x) 158 if err != nil { 159 return "", DefaultMeshGatewayPort, fmt.Errorf("Error parsing address %q: %v", x, err) 160 } 161 162 port := DefaultMeshGatewayPort 163 164 if portStr != "" { 165 port, err = strconv.Atoi(portStr) 166 if err != nil { 167 return "", DefaultMeshGatewayPort, fmt.Errorf("Error parsing port %q: %v", portStr, err) 168 } 169 } 170 171 return addr, port, nil 172} 173 174// canBindInternal is here mainly so we can unit test this with a constant net.Addr list 175func canBindInternal(addr string, ifAddrs []net.Addr) bool { 176 if addr == "" { 177 return false 178 } 179 180 ip := net.ParseIP(addr) 181 if ip == nil { 182 return false 183 } 184 185 ipStr := ip.String() 186 187 for _, addr := range ifAddrs { 188 switch v := addr.(type) { 189 case *net.IPNet: 190 if v.IP.String() == ipStr { 191 return true 192 } 193 default: 194 if addr.String() == ipStr { 195 return true 196 } 197 } 198 } 199 200 return false 201} 202 203func canBind(addr string) bool { 204 ifAddrs, err := net.InterfaceAddrs() 205 206 if err != nil { 207 return false 208 } 209 210 return canBindInternal(addr, ifAddrs) 211} 212 213func (c *cmd) Run(args []string) int { 214 if err := c.flags.Parse(args); err != nil { 215 return 1 216 } 217 passThroughArgs := c.flags.Args() 218 219 // Load the proxy ID and token from env vars if they're set 220 if c.proxyID == "" { 221 c.proxyID = os.Getenv("CONNECT_PROXY_ID") 222 } 223 if c.sidecarFor == "" { 224 c.sidecarFor = os.Getenv("CONNECT_SIDECAR_FOR") 225 } 226 if c.grpcAddr == "" { 227 c.grpcAddr = os.Getenv(api.GRPCAddrEnvName) 228 } 229 230 // Setup Consul client 231 client, err := c.http.APIClient() 232 if err != nil { 233 c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) 234 return 1 235 } 236 c.client = client 237 238 if c.register { 239 if !c.meshGateway { 240 c.UI.Error("Auto-Registration can only be used for mesh gateways") 241 return 1 242 } 243 244 lanAddr, lanPort, err := parseAddress(c.address) 245 if err != nil { 246 c.UI.Error(fmt.Sprintf("Failed to parse the -address parameter: %v", err)) 247 return 1 248 } 249 250 taggedAddrs := make(map[string]api.ServiceAddress) 251 252 if lanAddr != "" { 253 taggedAddrs[structs.TaggedAddressLAN] = api.ServiceAddress{Address: lanAddr, Port: lanPort} 254 } 255 256 wanAddr := "" 257 wanPort := lanPort 258 if c.wanAddress != "" { 259 wanAddr, wanPort, err = parseAddress(c.wanAddress) 260 if err != nil { 261 c.UI.Error(fmt.Sprintf("Failed to parse the -wan-address parameter: %v", err)) 262 return 1 263 } 264 taggedAddrs[structs.TaggedAddressWAN] = api.ServiceAddress{Address: wanAddr, Port: wanPort} 265 } 266 267 tcpCheckAddr := lanAddr 268 if tcpCheckAddr == "" { 269 // fallback to localhost as the gateway has to reside in the same network namespace 270 // as the agent 271 tcpCheckAddr = "127.0.0.1" 272 } 273 274 var proxyConf *api.AgentServiceConnectProxyConfig 275 276 if len(c.bindAddresses) > 0 { 277 // override all default binding rules and just bind to the user-supplied addresses 278 bindAddresses := make(map[string]api.ServiceAddress) 279 280 for addrName, addrStr := range c.bindAddresses { 281 addr, port, err := parseAddress(addrStr) 282 if err != nil { 283 c.UI.Error(fmt.Sprintf("Failed to parse the bind address: %s=%s: %v", addrName, addrStr, err)) 284 return 1 285 } 286 287 bindAddresses[addrName] = api.ServiceAddress{Address: addr, Port: port} 288 } 289 290 proxyConf = &api.AgentServiceConnectProxyConfig{ 291 Config: map[string]interface{}{ 292 "envoy_mesh_gateway_no_default_bind": true, 293 "envoy_mesh_gateway_bind_addresses": bindAddresses, 294 }, 295 } 296 } else if canBind(lanAddr) && canBind(wanAddr) { 297 // when both addresses are bindable then we bind to the tagged addresses 298 // for creating the envoy listeners 299 proxyConf = &api.AgentServiceConnectProxyConfig{ 300 Config: map[string]interface{}{ 301 "envoy_mesh_gateway_no_default_bind": true, 302 "envoy_mesh_gateway_bind_tagged_addresses": true, 303 }, 304 } 305 } else if !canBind(lanAddr) && lanAddr != "" { 306 c.UI.Error(fmt.Sprintf("The LAN address %q will not be bindable. Either set a bindable address or override the bind addresses with -bind-address", lanAddr)) 307 return 1 308 } 309 310 svc := api.AgentServiceRegistration{ 311 Kind: api.ServiceKindMeshGateway, 312 Name: c.meshGatewaySvcName, 313 Address: lanAddr, 314 Port: lanPort, 315 TaggedAddresses: taggedAddrs, 316 Proxy: proxyConf, 317 Check: &api.AgentServiceCheck{ 318 Name: "Mesh Gateway Listening", 319 TCP: ipaddr.FormatAddressPort(tcpCheckAddr, lanPort), 320 Interval: "10s", 321 DeregisterCriticalServiceAfter: c.deregAfterCritical, 322 }, 323 } 324 325 if err := client.Agent().ServiceRegister(&svc); err != nil { 326 c.UI.Error(fmt.Sprintf("Error registering service %q: %s", svc.Name, err)) 327 return 1 328 } 329 330 c.UI.Output(fmt.Sprintf("Registered service: %s", svc.Name)) 331 } 332 333 // See if we need to lookup proxyID 334 if c.proxyID == "" && c.sidecarFor != "" { 335 proxyID, err := c.lookupProxyIDForSidecar() 336 if err != nil { 337 c.UI.Error(err.Error()) 338 return 1 339 } 340 c.proxyID = proxyID 341 } else if c.proxyID == "" && c.meshGateway { 342 gatewaySvc, err := c.lookupGatewayProxy() 343 if err != nil { 344 c.UI.Error(err.Error()) 345 return 1 346 } 347 c.proxyID = gatewaySvc.ID 348 c.meshGatewaySvcName = gatewaySvc.Service 349 } 350 351 if c.proxyID == "" { 352 c.UI.Error("No proxy ID specified. One of -proxy-id or -sidecar-for/-mesh-gateway is " + 353 "required") 354 return 1 355 } 356 357 // See if we need to lookup grpcAddr 358 if c.grpcAddr == "" { 359 port, err := c.lookupGRPCPort() 360 if err != nil { 361 c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) 362 } 363 if port <= 0 { 364 // This is the dev mode default and recommended production setting if 365 // enabled. 366 port = 8502 367 c.UI.Info(fmt.Sprintf("Defaulting to grpc port = %d", port)) 368 } 369 c.grpcAddr = fmt.Sprintf("localhost:%v", port) 370 } 371 372 // Generate config 373 bootstrapJson, err := c.generateConfig() 374 if err != nil { 375 c.UI.Error(err.Error()) 376 return 1 377 } 378 379 if c.bootstrap { 380 // Just output it and we are done 381 os.Stdout.Write(bootstrapJson) 382 return 0 383 } 384 385 // Find Envoy binary 386 binary, err := c.findBinary() 387 if err != nil { 388 c.UI.Error("Couldn't find envoy binary: " + err.Error()) 389 return 1 390 } 391 392 err = execEnvoy(binary, nil, passThroughArgs, bootstrapJson) 393 if err == errUnsupportedOS { 394 c.UI.Error("Directly running Envoy is only supported on linux and macOS " + 395 "since envoy itself doesn't build on other platforms currently.") 396 c.UI.Error("Use the -bootstrap option to generate the JSON to use when running envoy " + 397 "on a supported OS or via a container or VM.") 398 return 1 399 } else if err != nil { 400 c.UI.Error(err.Error()) 401 return 1 402 } 403 404 return 0 405} 406 407var errUnsupportedOS = errors.New("envoy: not implemented on this operating system") 408 409func (c *cmd) findBinary() (string, error) { 410 if c.envoyBin != "" { 411 return c.envoyBin, nil 412 } 413 return exec.LookPath("envoy") 414} 415 416func (c *cmd) templateArgs() (*BootstrapTplArgs, error) { 417 httpCfg := api.DefaultConfig() 418 c.http.MergeOntoConfig(httpCfg) 419 420 // Trigger the Client init to do any last-minute updates to the Config. 421 if _, err := api.NewClient(httpCfg); err != nil { 422 return nil, err 423 } 424 425 // Decide on TLS if the scheme is provided and indicates it, if the HTTP env 426 // suggests TLS is supported explicitly (CONSUL_HTTP_SSL) or implicitly 427 // (CONSUL_HTTP_ADDR) is https:// 428 useTLS := false 429 if strings.HasPrefix(strings.ToLower(c.grpcAddr), "https://") { 430 useTLS = true 431 } else if useSSLEnv := os.Getenv(api.HTTPSSLEnvName); useSSLEnv != "" { 432 if enabled, err := strconv.ParseBool(useSSLEnv); err == nil { 433 useTLS = enabled 434 } 435 } else if strings.HasPrefix(strings.ToLower(httpCfg.Address), "https://") { 436 useTLS = true 437 } 438 439 // We want to allow grpcAddr set as host:port with no scheme but if the host 440 // is an IP this will fail to parse as a URL with "parse 127.0.0.1:8500: first 441 // path segment in URL cannot contain colon". On the other hand we also 442 // support both http(s)://host:port and unix:///path/to/file. 443 var agentAddr, agentPort, agentSock string 444 if grpcAddr := strings.TrimPrefix(c.grpcAddr, "unix://"); grpcAddr != c.grpcAddr { 445 // Path to unix socket 446 agentSock = grpcAddr 447 } else { 448 // Parse as host:port with option http prefix 449 grpcAddr = strings.TrimPrefix(c.grpcAddr, "http://") 450 grpcAddr = strings.TrimPrefix(c.grpcAddr, "https://") 451 452 var err error 453 agentAddr, agentPort, err = net.SplitHostPort(grpcAddr) 454 if err != nil { 455 return nil, fmt.Errorf("Invalid Consul HTTP address: %s", err) 456 } 457 if agentAddr == "" { 458 agentAddr = "127.0.0.1" 459 } 460 461 // We use STATIC for agent which means we need to resolve DNS names like 462 // `localhost` ourselves. We could use STRICT_DNS or LOGICAL_DNS with envoy 463 // but Envoy resolves `localhost` differently to go on macOS at least which 464 // causes paper cuts like default dev agent (which binds specifically to 465 // 127.0.0.1) isn't reachable since Envoy resolves localhost to `[::]` and 466 // can't connect. 467 agentIP, err := net.ResolveIPAddr("ip", agentAddr) 468 if err != nil { 469 return nil, fmt.Errorf("Failed to resolve agent address: %s", err) 470 } 471 agentAddr = agentIP.String() 472 } 473 474 adminAddr, adminPort, err := net.SplitHostPort(c.adminBind) 475 if err != nil { 476 return nil, fmt.Errorf("Invalid Consul HTTP address: %s", err) 477 } 478 479 // Envoy requires IP addresses to bind too when using static so resolve DNS or 480 // localhost here. 481 adminBindIP, err := net.ResolveIPAddr("ip", adminAddr) 482 if err != nil { 483 return nil, fmt.Errorf("Failed to resolve admin bind address: %s", err) 484 } 485 486 // Ideally the cluster should be the service name. We may or may not have that 487 // yet depending on the arguments used so make a best effort here. In the 488 // common case, even if the command was invoked with proxy-id and we don't 489 // know service name yet, we will after we resolve the proxy's config in a bit 490 // and will update this then. 491 cluster := c.proxyID 492 if c.sidecarFor != "" { 493 cluster = c.sidecarFor 494 } else if c.meshGateway && c.meshGatewaySvcName != "" { 495 cluster = c.meshGatewaySvcName 496 } 497 498 adminAccessLogPath := c.adminAccessLogPath 499 if adminAccessLogPath == "" { 500 adminAccessLogPath = DefaultAdminAccessLogPath 501 } 502 503 var caPEM string 504 if httpCfg.TLSConfig.CAFile != "" { 505 content, err := ioutil.ReadFile(httpCfg.TLSConfig.CAFile) 506 if err != nil { 507 return nil, fmt.Errorf("Failed to read CA file: %s", err) 508 } 509 caPEM = strings.Replace(string(content), "\n", "\\n", -1) 510 } 511 512 return &BootstrapTplArgs{ 513 ProxyCluster: cluster, 514 ProxyID: c.proxyID, 515 AgentAddress: agentAddr, 516 AgentPort: agentPort, 517 AgentSocket: agentSock, 518 AgentTLS: useTLS, 519 AgentCAPEM: caPEM, 520 AdminAccessLogPath: adminAccessLogPath, 521 AdminBindAddress: adminBindIP.String(), 522 AdminBindPort: adminPort, 523 Token: httpCfg.Token, 524 LocalAgentClusterName: xds.LocalAgentClusterName, 525 Namespace: httpCfg.Namespace, 526 EnvoyVersion: c.envoyVersion, 527 }, nil 528} 529 530func (c *cmd) generateConfig() ([]byte, error) { 531 args, err := c.templateArgs() 532 if err != nil { 533 return nil, err 534 } 535 536 var bsCfg BootstrapConfig 537 538 if !c.disableCentralConfig { 539 // Fetch any customization from the registration 540 svc, _, err := c.client.Agent().Service(c.proxyID, nil) 541 if err != nil { 542 return nil, fmt.Errorf("failed fetch proxy config from local agent: %s", err) 543 } 544 545 if svc.Proxy == nil { 546 return nil, errors.New("service is not a Connect proxy or mesh gateway") 547 } 548 549 // Parse the bootstrap config 550 if err := mapstructure.WeakDecode(svc.Proxy.Config, &bsCfg); err != nil { 551 return nil, fmt.Errorf("failed parsing Proxy.Config: %s", err) 552 } 553 554 if svc.Proxy.DestinationServiceName != "" { 555 // Override cluster now we know the actual service name 556 args.ProxyCluster = svc.Proxy.DestinationServiceName 557 } 558 } 559 560 return bsCfg.GenerateJSON(args) 561} 562 563func (c *cmd) lookupProxyIDForSidecar() (string, error) { 564 return proxyCmd.LookupProxyIDForSidecar(c.client, c.sidecarFor) 565} 566 567func (c *cmd) lookupGatewayProxy() (*api.AgentService, error) { 568 return proxyCmd.LookupGatewayProxy(c.client) 569} 570 571func (c *cmd) lookupGRPCPort() (int, error) { 572 self, err := c.client.Agent().Self() 573 if err != nil { 574 return 0, err 575 } 576 cfg, ok := self["DebugConfig"] 577 if !ok { 578 return 0, fmt.Errorf("unexpected agent response: no debug config") 579 } 580 port, ok := cfg["GRPCPort"] 581 if !ok { 582 return 0, fmt.Errorf("agent does not have grpc port enabled") 583 } 584 portN, ok := port.(float64) 585 if !ok { 586 return 0, fmt.Errorf("invalid grpc port in agent response") 587 } 588 589 return int(portN), nil 590} 591 592func (c *cmd) Synopsis() string { 593 return synopsis 594} 595 596func (c *cmd) Help() string { 597 return c.help 598} 599 600const synopsis = "Runs or Configures Envoy as a Connect proxy" 601const help = ` 602Usage: consul connect envoy [options] 603 604 Generates the bootstrap configuration needed to start an Envoy proxy instance 605 for use as a Connect sidecar for a particular service instance. By default it 606 will generate the config and then exec Envoy directly until it exits normally. 607 608 It will search $PATH for the envoy binary but this can be overridden with 609 -envoy-binary. 610 611 It can instead only generate the bootstrap.json based on the current ENV and 612 arguments using -bootstrap. 613 614 The proxy requires service:write permissions for the service it represents. 615 The token may be passed via the CLI or the CONSUL_HTTP_TOKEN environment 616 variable. 617 618 The example below shows how to start a local proxy as a sidecar to a "web" 619 service instance. It assumes that the proxy was already registered with it's 620 Config for example via a sidecar_service block. 621 622 $ consul connect envoy -sidecar-for web 623 624` 625