1// Copyright 2017 Istio Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package main 16 17import ( 18 "context" 19 "fmt" 20 "net" 21 "os" 22 "time" 23 24 "github.com/gogo/protobuf/jsonpb" 25 "github.com/spf13/cobra" 26 "github.com/spf13/cobra/doc" 27 28 "istio.io/istio/pkg/dns" 29 30 meshconfig "istio.io/api/mesh/v1alpha1" 31 "istio.io/pkg/collateral" 32 "istio.io/pkg/env" 33 "istio.io/pkg/log" 34 "istio.io/pkg/version" 35 36 "istio.io/istio/pilot/cmd/pilot-agent/status" 37 "istio.io/istio/pilot/pkg/features" 38 "istio.io/istio/pilot/pkg/model" 39 "istio.io/istio/pilot/pkg/proxy" 40 envoyDiscovery "istio.io/istio/pilot/pkg/proxy/envoy" 41 securityModel "istio.io/istio/pilot/pkg/security/model" 42 "istio.io/istio/pilot/pkg/serviceregistry" 43 "istio.io/istio/pilot/pkg/util/sets" 44 "istio.io/istio/pkg/cmd" 45 "istio.io/istio/pkg/config/constants" 46 "istio.io/istio/pkg/envoy" 47 istio_agent "istio.io/istio/pkg/istio-agent" 48 "istio.io/istio/pkg/jwt" 49 "istio.io/istio/pkg/spiffe" 50 "istio.io/istio/pkg/util/gogoprotomarshal" 51 citadel "istio.io/istio/security/pkg/nodeagent/caclient/providers/citadel" 52 stsserver "istio.io/istio/security/pkg/stsservice/server" 53 "istio.io/istio/security/pkg/stsservice/tokenmanager" 54 cleaniptables "istio.io/istio/tools/istio-clean-iptables/pkg/cmd" 55 iptables "istio.io/istio/tools/istio-iptables/pkg/cmd" 56) 57 58const ( 59 trustworthyJWTPath = "./var/run/secrets/tokens/istio-token" 60 localHostIPv4 = "127.0.0.1" 61 localHostIPv6 = "[::1]" 62) 63 64// TODO: Move most of this to pkg. 65 66var ( 67 role = &model.Proxy{} 68 proxyIP string 69 registryID serviceregistry.ProviderID 70 trustDomain string 71 pilotIdentity string 72 mixerIdentity string 73 stsPort int 74 tokenManagerPlugin string 75 76 meshConfigFile string 77 78 // proxy config flags (named identically) 79 serviceCluster string 80 proxyLogLevel string 81 proxyComponentLogLevel string 82 concurrency int 83 templateFile string 84 disableInternalTelemetry bool 85 loggingOptions = log.DefaultOptions() 86 outlierLogPath string 87 88 instanceIPVar = env.RegisterStringVar("INSTANCE_IP", "", "") 89 podNameVar = env.RegisterStringVar("POD_NAME", "", "") 90 podNamespaceVar = env.RegisterStringVar("POD_NAMESPACE", "", "") 91 istioNamespaceVar = env.RegisterStringVar("ISTIO_NAMESPACE", "", "") 92 kubeAppProberNameVar = env.RegisterStringVar(status.KubeAppProberEnvName, "", "") 93 clusterIDVar = env.RegisterStringVar("ISTIO_META_CLUSTER_ID", "", "") 94 95 pilotCertProvider = env.RegisterStringVar("PILOT_CERT_PROVIDER", "istiod", 96 "the provider of Pilot DNS certificate.").Get() 97 jwtPolicy = env.RegisterStringVar("JWT_POLICY", jwt.JWTPolicyThirdPartyJWT, 98 "The JWT validation policy.") 99 outputKeyCertToDir = env.RegisterStringVar("OUTPUT_CERTS", "", 100 "The output directory for the key and certificate. If empty, key and certificate will not be saved. "+ 101 "Must be set for VMs using provisioning certificates.").Get() 102 proxyConfigEnv = env.RegisterStringVar( 103 "PROXY_CONFIG", 104 "", 105 "The proxy configuration. This will be set by the injection - gateways will use file mounts.", 106 ).Get() 107 108 rootCmd = &cobra.Command{ 109 Use: "pilot-agent", 110 Short: "Istio Pilot agent.", 111 Long: "Istio Pilot agent runs in the sidecar or gateway container and bootstraps Envoy.", 112 SilenceUsage: true, 113 FParseErrWhitelist: cobra.FParseErrWhitelist{ 114 // Allow unknown flags for backward-compatibility. 115 UnknownFlags: true, 116 }, 117 } 118 119 proxyCmd = &cobra.Command{ 120 Use: "proxy", 121 Short: "Envoy proxy agent", 122 FParseErrWhitelist: cobra.FParseErrWhitelist{ 123 // Allow unknown flags for backward-compatibility. 124 UnknownFlags: true, 125 }, 126 RunE: func(c *cobra.Command, args []string) error { 127 cmd.PrintFlags(c.Flags()) 128 if err := log.Configure(loggingOptions); err != nil { 129 return err 130 } 131 132 // Extract pod variables. 133 podName := podNameVar.Get() 134 podNamespace := podNamespaceVar.Get() 135 podIP := net.ParseIP(instanceIPVar.Get()) // protobuf encoding of IP_ADDRESS type 136 137 log.Infof("Version %s", version.Info.String()) 138 role.Type = model.SidecarProxy 139 if len(args) > 0 { 140 role.Type = model.NodeType(args[0]) 141 if !model.IsApplicationNodeType(role.Type) { 142 log.Errorf("Invalid role Type: %#v", role.Type) 143 return fmt.Errorf("Invalid role Type: " + string(role.Type)) 144 } 145 } 146 147 if len(proxyIP) != 0 { 148 role.IPAddresses = []string{proxyIP} 149 } else if podIP != nil { 150 role.IPAddresses = []string{podIP.String()} 151 } 152 153 // Obtain all the IPs from the node 154 if ipAddrs, ok := proxy.GetPrivateIPs(context.Background()); ok { 155 log.Infof("Obtained private IP %v", ipAddrs) 156 if len(role.IPAddresses) == 1 { 157 for _, ip := range ipAddrs { 158 // prevent duplicate ips, the first one must be the pod ip 159 // as we pick the first ip as pod ip in istiod 160 if role.IPAddresses[0] != ip { 161 role.IPAddresses = append(role.IPAddresses, ip) 162 } 163 } 164 } else { 165 role.IPAddresses = append(role.IPAddresses, ipAddrs...) 166 } 167 } 168 169 // No IP addresses provided, append 127.0.0.1 for ipv4 and ::1 for ipv6 170 if len(role.IPAddresses) == 0 { 171 role.IPAddresses = append(role.IPAddresses, "127.0.0.1") 172 role.IPAddresses = append(role.IPAddresses, "::1") 173 } 174 175 // Check if proxy runs in ipv4 or ipv6 environment to set Envoy's 176 // operational parameters correctly. 177 proxyIPv6 := isIPv6Proxy(role.IPAddresses) 178 if len(role.ID) == 0 { 179 if registryID == serviceregistry.Kubernetes { 180 role.ID = podName + "." + podNamespace 181 } else if registryID == serviceregistry.Consul { 182 role.ID = role.IPAddresses[0] + ".service.consul" 183 } else { 184 role.ID = role.IPAddresses[0] 185 } 186 } 187 188 proxyConfig, err := constructProxyConfig() 189 if err != nil { 190 return fmt.Errorf("failed to get proxy config: %v", err) 191 } 192 if out, err := gogoprotomarshal.ToYAML(&proxyConfig); err != nil { 193 log.Infof("Failed to serialize to YAML: %v", err) 194 } else { 195 log.Infof("Effective config: %s", out) 196 } 197 198 // If not set, set a default based on platform - podNamespace.svc.cluster.local for 199 // K8S 200 role.DNSDomain = getDNSDomain(podNamespace, role.DNSDomain) 201 log.Infof("Proxy role: %#v", role) 202 203 var jwtPath string 204 if jwtPolicy.Get() == jwt.JWTPolicyThirdPartyJWT { 205 log.Info("JWT policy is third-party-jwt") 206 jwtPath = trustworthyJWTPath 207 } else if jwtPolicy.Get() == jwt.JWTPolicyFirstPartyJWT { 208 log.Info("JWT policy is first-party-jwt") 209 jwtPath = securityModel.K8sSAJwtFileName 210 } else { 211 log.Info("Using existing certs") 212 } 213 sa := istio_agent.NewSDSAgent(proxyConfig.DiscoveryAddress, proxyConfig.ControlPlaneAuthPolicy == meshconfig.AuthenticationPolicy_MUTUAL_TLS, 214 pilotCertProvider, jwtPath, outputKeyCertToDir, clusterIDVar.Get()) 215 216 // Connection to Istiod secure port 217 if sa.RequireCerts { 218 proxyConfig.ControlPlaneAuthPolicy = meshconfig.AuthenticationPolicy_MUTUAL_TLS 219 } 220 221 var pilotSAN, mixerSAN []string 222 if proxyConfig.ControlPlaneAuthPolicy == meshconfig.AuthenticationPolicy_MUTUAL_TLS { 223 setSpiffeTrustDomain(podNamespace, role.DNSDomain) 224 // Obtain the Mixer SAN, which uses SPIFFE certs. Used below to create a Envoy proxy. 225 mixerSAN = getSAN(getControlPlaneNamespace(podNamespace, proxyConfig.DiscoveryAddress), envoyDiscovery.MixerSvcAccName, mixerIdentity) 226 // Obtain Pilot SAN, using DNS. 227 pilotSAN = []string{getPilotSan(proxyConfig.DiscoveryAddress)} 228 } 229 log.Infof("PilotSAN %#v", pilotSAN) 230 log.Infof("MixerSAN %#v", mixerSAN) 231 232 // Start in process SDS. 233 _, err = sa.Start(role.Type == model.SidecarProxy, podNamespaceVar.Get()) 234 if err != nil { 235 log.Fatala("Failed to start in-process SDS", err) 236 } 237 238 // dedupe cert paths so we don't set up 2 watchers for the same file 239 tlsCerts := dedupeStrings(getTLSCerts(proxyConfig)) 240 241 // Since Envoy needs the file-mounted certs for mTLS, we wait for them to become available 242 // before starting it. 243 if len(tlsCerts) > 0 { 244 log.Infof("Monitored certs: %#v", tlsCerts) 245 for _, cert := range tlsCerts { 246 waitForFile(cert, 2*time.Minute) 247 } 248 } 249 250 // If we are using a custom template file (for control plane proxy, for example), configure this. 251 if templateFile != "" && proxyConfig.CustomConfigFile == "" { 252 proxyConfig.ProxyBootstrapTemplatePath = templateFile 253 } 254 255 ctx, cancel := context.WithCancel(context.Background()) 256 257 // If a status port was provided, start handling status probes. 258 if proxyConfig.StatusPort > 0 { 259 localHostAddr := localHostIPv4 260 if proxyIPv6 { 261 localHostAddr = localHostIPv6 262 } 263 prober := kubeAppProberNameVar.Get() 264 statusServer, err := status.NewServer(status.Config{ 265 LocalHostAddr: localHostAddr, 266 AdminPort: uint16(proxyConfig.ProxyAdminPort), 267 StatusPort: uint16(proxyConfig.StatusPort), 268 KubeAppProbers: prober, 269 NodeType: role.Type, 270 }) 271 if err != nil { 272 cancel() 273 return err 274 } 275 go statusServer.Run(ctx) 276 } 277 278 // If security token service (STS) port is not zero, start STS server and 279 // listen on STS port for STS requests. For STS, see 280 // https://tools.ietf.org/html/draft-ietf-oauth-token-exchange-16. 281 if stsPort > 0 { 282 localHostAddr := localHostIPv4 283 if proxyIPv6 { 284 localHostAddr = localHostIPv6 285 } 286 tokenManager := tokenmanager.CreateTokenManager(tokenManagerPlugin, 287 tokenmanager.Config{TrustDomain: trustDomain}) 288 stsServer, err := stsserver.NewServer(stsserver.Config{ 289 LocalHostAddr: localHostAddr, 290 LocalPort: stsPort, 291 }, tokenManager) 292 if err != nil { 293 cancel() 294 return err 295 } 296 defer stsServer.Stop() 297 } 298 299 // Start a local DNS server on 15053, forwarding to DNS-over-TLS server 300 // This will not have any impact on app unless interception is enabled. 301 // We can't start on 53 - istio-agent runs as user istio-proxy. 302 // This is available to apps even if interception is not enabled. 303 304 // TODO: replace hardcoded .global. Right now the ingress templates are 305 // hardcoding it as well, so there is little benefit to do it only here. 306 if dns.DNSTLSEnableAgent.Get() != "" { 307 // In the injection template the only place where global.proxy.clusterDomain 308 // is made available is in the --domain param. 309 // Instead of introducing a new config, use that. 310 311 dnsSrv := dns.InitDNSAgent(proxyConfig.DiscoveryAddress, 312 role.DNSDomain, sa.RootCert, 313 []string{".global."}) 314 dnsSrv.StartDNS(dns.DNSAgentAddr, nil) 315 } 316 317 envoyProxy := envoy.NewProxy(envoy.ProxyConfig{ 318 Config: proxyConfig, 319 Node: role.ServiceNode(), 320 LogLevel: proxyLogLevel, 321 ComponentLogLevel: proxyComponentLogLevel, 322 PilotSubjectAltName: pilotSAN, 323 MixerSubjectAltName: mixerSAN, 324 NodeIPs: role.IPAddresses, 325 PodName: podName, 326 PodNamespace: podNamespace, 327 PodIP: podIP, 328 STSPort: stsPort, 329 ControlPlaneAuth: proxyConfig.ControlPlaneAuthPolicy == meshconfig.AuthenticationPolicy_MUTUAL_TLS, 330 DisableReportCalls: disableInternalTelemetry, 331 OutlierLogPath: outlierLogPath, 332 PilotCertProvider: pilotCertProvider, 333 ProvCert: citadel.ProvCert, 334 }) 335 336 agent := envoy.NewAgent(envoyProxy, features.TerminationDrainDuration()) 337 338 // Watcher is also kicking envoy start. 339 watcher := envoy.NewWatcher(tlsCerts, agent.Restart) 340 go watcher.Run(ctx) 341 342 // On SIGINT or SIGTERM, cancel the context, triggering a graceful shutdown 343 go cmd.WaitSignalFunc(cancel) 344 345 return agent.Run(ctx) 346 }, 347 } 348) 349 350// dedupes the string array and also ignores the empty string. 351func dedupeStrings(in []string) []string { 352 set := sets.NewSet(in...) 353 return set.UnsortedList() 354} 355 356// explicitly set the trustdomain so the pilot and mixer SAN will have same trustdomain 357// and the initialization of the spiffe pkg isn't linked to generating pilot's SAN first 358func setSpiffeTrustDomain(podNamespace string, domain string) { 359 pilotTrustDomain := trustDomain 360 if len(pilotTrustDomain) == 0 { 361 if registryID == serviceregistry.Kubernetes && 362 (domain == podNamespace+".svc.cluster.local" || domain == "") { 363 pilotTrustDomain = "cluster.local" 364 } else if registryID == serviceregistry.Consul && 365 (domain == "service.consul" || domain == "") { 366 pilotTrustDomain = "" 367 } else { 368 pilotTrustDomain = domain 369 } 370 } 371 spiffe.SetTrustDomain(pilotTrustDomain) 372} 373 374func getSAN(ns string, defaultSA string, overrideIdentity string) []string { 375 var san []string 376 if overrideIdentity == "" { 377 san = append(san, envoyDiscovery.GetSAN(ns, defaultSA)) 378 } else { 379 san = append(san, envoyDiscovery.GetSAN("", overrideIdentity)) 380 381 } 382 return san 383} 384 385func getDNSDomain(podNamespace, domain string) string { 386 if len(domain) == 0 { 387 if registryID == serviceregistry.Kubernetes { 388 domain = podNamespace + ".svc.cluster.local" 389 } else if registryID == serviceregistry.Consul { 390 domain = "service.consul" 391 } else { 392 domain = "" 393 } 394 } 395 return domain 396} 397 398func fromJSON(j string) *meshconfig.RemoteService { 399 var m meshconfig.RemoteService 400 err := jsonpb.UnmarshalString(j, &m) 401 if err != nil { 402 log.Warnf("Unable to unmarshal %s: %v", j, err) 403 return nil 404 } 405 406 return &m 407} 408 409func init() { 410 proxyCmd.PersistentFlags().StringVar((*string)(®istryID), "serviceregistry", 411 string(serviceregistry.Kubernetes), 412 fmt.Sprintf("Select the platform for service registry, options are {%s, %s, %s}", 413 serviceregistry.Kubernetes, serviceregistry.Consul, serviceregistry.Mock)) 414 proxyCmd.PersistentFlags().StringVar(&proxyIP, "ip", "", 415 "Proxy IP address. If not provided uses ${INSTANCE_IP} environment variable.") 416 proxyCmd.PersistentFlags().StringVar(&role.ID, "id", "", 417 "Proxy unique ID. If not provided uses ${POD_NAME}.${POD_NAMESPACE} from environment variables") 418 proxyCmd.PersistentFlags().StringVar(&role.DNSDomain, "domain", "", 419 "DNS domain suffix. If not provided uses ${POD_NAMESPACE}.svc.cluster.local") 420 proxyCmd.PersistentFlags().StringVar(&trustDomain, "trust-domain", "", 421 "The domain to use for identities") 422 proxyCmd.PersistentFlags().StringVar(&pilotIdentity, "pilotIdentity", "", 423 "The identity used as the suffix for pilot's spiffe SAN ") 424 proxyCmd.PersistentFlags().StringVar(&mixerIdentity, "mixerIdentity", "", 425 "The identity used as the suffix for mixer's spiffe SAN. This would only be used by pilot all other proxy would get this value from pilot") 426 427 proxyCmd.PersistentFlags().StringVar(&meshConfigFile, "meshConfig", "./etc/istio/config/mesh", 428 "File name for Istio mesh configuration. If not specified, a default mesh will be used. This may be overridden by "+ 429 "PROXY_CONFIG environment variable or proxy.istio.io/config annotation.") 430 proxyCmd.PersistentFlags().IntVar(&stsPort, "stsPort", 0, 431 "HTTP Port on which to serve Security Token Service (STS). If zero, STS service will not be provided.") 432 proxyCmd.PersistentFlags().StringVar(&tokenManagerPlugin, "tokenManagerPlugin", tokenmanager.GoogleTokenExchange, 433 "Token provider specific plugin name.") 434 // Flags for proxy configuration 435 proxyCmd.PersistentFlags().StringVar(&serviceCluster, "serviceCluster", constants.ServiceClusterName, "Service cluster") 436 // Log levels are provided by the library https://github.com/gabime/spdlog, used by Envoy. 437 proxyCmd.PersistentFlags().StringVar(&proxyLogLevel, "proxyLogLevel", "warning", 438 fmt.Sprintf("The log level used to start the Envoy proxy (choose from {%s, %s, %s, %s, %s, %s, %s})", 439 "trace", "debug", "info", "warning", "error", "critical", "off")) 440 proxyCmd.PersistentFlags().IntVar(&concurrency, "concurrency", 0, "number of worker threads to run") 441 // See https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-component-log-level 442 proxyCmd.PersistentFlags().StringVar(&proxyComponentLogLevel, "proxyComponentLogLevel", "misc:error", 443 "The component log level used to start the Envoy proxy") 444 proxyCmd.PersistentFlags().StringVar(&templateFile, "templateFile", "", 445 "Go template bootstrap config") 446 proxyCmd.PersistentFlags().BoolVar(&disableInternalTelemetry, "disableInternalTelemetry", false, 447 "Disable internal telemetry") 448 proxyCmd.PersistentFlags().StringVar(&outlierLogPath, "outlierLogPath", "", 449 "The log path for outlier detection") 450 451 // Attach the Istio logging options to the command. 452 loggingOptions.AttachCobraFlags(rootCmd) 453 454 cmd.AddFlags(rootCmd) 455 456 rootCmd.AddCommand(proxyCmd) 457 rootCmd.AddCommand(version.CobraCommand()) 458 rootCmd.AddCommand(iptables.GetCommand()) 459 rootCmd.AddCommand(cleaniptables.GetCommand()) 460 461 rootCmd.AddCommand(collateral.CobraCommand(rootCmd, &doc.GenManHeader{ 462 Title: "Istio Pilot Agent", 463 Section: "pilot-agent CLI", 464 Manual: "Istio Pilot Agent", 465 })) 466} 467 468func waitForFile(fname string, maxWait time.Duration) bool { 469 log.Infof("waiting %v for %s", maxWait, fname) 470 471 logDelay := 1 * time.Second 472 nextLog := time.Now().Add(logDelay) 473 endWait := time.Now().Add(maxWait) 474 475 for { 476 _, err := os.Stat(fname) 477 if err == nil { 478 return true 479 } 480 if !os.IsNotExist(err) { // another error (e.g., permission) - likely no point in waiting longer 481 log.Errora("error while waiting for file", err.Error()) 482 return false 483 } 484 485 now := time.Now() 486 if now.After(endWait) { 487 log.Warna("file still not available after", maxWait) 488 return false 489 } 490 if now.After(nextLog) { 491 log.Infof("waiting for file") 492 logDelay *= 2 493 nextLog.Add(logDelay) 494 } 495 time.Sleep(100 * time.Millisecond) 496 } 497} 498 499// TODO: get the config and bootstrap from istiod, by passing the env 500 501// Use env variables - from injection, k8s and local namespace config map. 502// No CLI parameters. 503func main() { 504 if err := rootCmd.Execute(); err != nil { 505 log.Errora(err) 506 os.Exit(-1) 507 } 508} 509 510// isIPv6Proxy check the addresses slice and returns true for a valid IPv6 address 511// for all other cases it returns false 512func isIPv6Proxy(ipAddrs []string) bool { 513 for i := 0; i < len(ipAddrs); i++ { 514 addr := net.ParseIP(ipAddrs[i]) 515 if addr == nil { 516 // Should not happen, invalid IP in proxy's IPAddresses slice should have been caught earlier, 517 // skip it to prevent a panic. 518 continue 519 } 520 if addr.To4() != nil { 521 return false 522 } 523 } 524 return true 525} 526