1// Copyright 2017 Istio Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18	"context"
19	"fmt"
20	"net"
21	"os"
22	"time"
23
24	"github.com/gogo/protobuf/jsonpb"
25	"github.com/spf13/cobra"
26	"github.com/spf13/cobra/doc"
27
28	"istio.io/istio/pkg/dns"
29
30	meshconfig "istio.io/api/mesh/v1alpha1"
31	"istio.io/pkg/collateral"
32	"istio.io/pkg/env"
33	"istio.io/pkg/log"
34	"istio.io/pkg/version"
35
36	"istio.io/istio/pilot/cmd/pilot-agent/status"
37	"istio.io/istio/pilot/pkg/features"
38	"istio.io/istio/pilot/pkg/model"
39	"istio.io/istio/pilot/pkg/proxy"
40	envoyDiscovery "istio.io/istio/pilot/pkg/proxy/envoy"
41	securityModel "istio.io/istio/pilot/pkg/security/model"
42	"istio.io/istio/pilot/pkg/serviceregistry"
43	"istio.io/istio/pilot/pkg/util/sets"
44	"istio.io/istio/pkg/cmd"
45	"istio.io/istio/pkg/config/constants"
46	"istio.io/istio/pkg/envoy"
47	istio_agent "istio.io/istio/pkg/istio-agent"
48	"istio.io/istio/pkg/jwt"
49	"istio.io/istio/pkg/spiffe"
50	"istio.io/istio/pkg/util/gogoprotomarshal"
51	citadel "istio.io/istio/security/pkg/nodeagent/caclient/providers/citadel"
52	stsserver "istio.io/istio/security/pkg/stsservice/server"
53	"istio.io/istio/security/pkg/stsservice/tokenmanager"
54	cleaniptables "istio.io/istio/tools/istio-clean-iptables/pkg/cmd"
55	iptables "istio.io/istio/tools/istio-iptables/pkg/cmd"
56)
57
58const (
59	trustworthyJWTPath = "./var/run/secrets/tokens/istio-token"
60	localHostIPv4      = "127.0.0.1"
61	localHostIPv6      = "[::1]"
62)
63
64// TODO: Move most of this to pkg.
65
66var (
67	role               = &model.Proxy{}
68	proxyIP            string
69	registryID         serviceregistry.ProviderID
70	trustDomain        string
71	pilotIdentity      string
72	mixerIdentity      string
73	stsPort            int
74	tokenManagerPlugin string
75
76	meshConfigFile string
77
78	// proxy config flags (named identically)
79	serviceCluster           string
80	proxyLogLevel            string
81	proxyComponentLogLevel   string
82	concurrency              int
83	templateFile             string
84	disableInternalTelemetry bool
85	loggingOptions           = log.DefaultOptions()
86	outlierLogPath           string
87
88	instanceIPVar        = env.RegisterStringVar("INSTANCE_IP", "", "")
89	podNameVar           = env.RegisterStringVar("POD_NAME", "", "")
90	podNamespaceVar      = env.RegisterStringVar("POD_NAMESPACE", "", "")
91	istioNamespaceVar    = env.RegisterStringVar("ISTIO_NAMESPACE", "", "")
92	kubeAppProberNameVar = env.RegisterStringVar(status.KubeAppProberEnvName, "", "")
93	clusterIDVar         = env.RegisterStringVar("ISTIO_META_CLUSTER_ID", "", "")
94
95	pilotCertProvider = env.RegisterStringVar("PILOT_CERT_PROVIDER", "istiod",
96		"the provider of Pilot DNS certificate.").Get()
97	jwtPolicy = env.RegisterStringVar("JWT_POLICY", jwt.JWTPolicyThirdPartyJWT,
98		"The JWT validation policy.")
99	outputKeyCertToDir = env.RegisterStringVar("OUTPUT_CERTS", "",
100		"The output directory for the key and certificate. If empty, key and certificate will not be saved. "+
101			"Must be set for VMs using provisioning certificates.").Get()
102	proxyConfigEnv = env.RegisterStringVar(
103		"PROXY_CONFIG",
104		"",
105		"The proxy configuration. This will be set by the injection - gateways will use file mounts.",
106	).Get()
107
108	rootCmd = &cobra.Command{
109		Use:          "pilot-agent",
110		Short:        "Istio Pilot agent.",
111		Long:         "Istio Pilot agent runs in the sidecar or gateway container and bootstraps Envoy.",
112		SilenceUsage: true,
113		FParseErrWhitelist: cobra.FParseErrWhitelist{
114			// Allow unknown flags for backward-compatibility.
115			UnknownFlags: true,
116		},
117	}
118
119	proxyCmd = &cobra.Command{
120		Use:   "proxy",
121		Short: "Envoy proxy agent",
122		FParseErrWhitelist: cobra.FParseErrWhitelist{
123			// Allow unknown flags for backward-compatibility.
124			UnknownFlags: true,
125		},
126		RunE: func(c *cobra.Command, args []string) error {
127			cmd.PrintFlags(c.Flags())
128			if err := log.Configure(loggingOptions); err != nil {
129				return err
130			}
131
132			// Extract pod variables.
133			podName := podNameVar.Get()
134			podNamespace := podNamespaceVar.Get()
135			podIP := net.ParseIP(instanceIPVar.Get()) // protobuf encoding of IP_ADDRESS type
136
137			log.Infof("Version %s", version.Info.String())
138			role.Type = model.SidecarProxy
139			if len(args) > 0 {
140				role.Type = model.NodeType(args[0])
141				if !model.IsApplicationNodeType(role.Type) {
142					log.Errorf("Invalid role Type: %#v", role.Type)
143					return fmt.Errorf("Invalid role Type: " + string(role.Type))
144				}
145			}
146
147			if len(proxyIP) != 0 {
148				role.IPAddresses = []string{proxyIP}
149			} else if podIP != nil {
150				role.IPAddresses = []string{podIP.String()}
151			}
152
153			// Obtain all the IPs from the node
154			if ipAddrs, ok := proxy.GetPrivateIPs(context.Background()); ok {
155				log.Infof("Obtained private IP %v", ipAddrs)
156				if len(role.IPAddresses) == 1 {
157					for _, ip := range ipAddrs {
158						// prevent duplicate ips, the first one must be the pod ip
159						// as we pick the first ip as pod ip in istiod
160						if role.IPAddresses[0] != ip {
161							role.IPAddresses = append(role.IPAddresses, ip)
162						}
163					}
164				} else {
165					role.IPAddresses = append(role.IPAddresses, ipAddrs...)
166				}
167			}
168
169			// No IP addresses provided, append 127.0.0.1 for ipv4 and ::1 for ipv6
170			if len(role.IPAddresses) == 0 {
171				role.IPAddresses = append(role.IPAddresses, "127.0.0.1")
172				role.IPAddresses = append(role.IPAddresses, "::1")
173			}
174
175			// Check if proxy runs in ipv4 or ipv6 environment to set Envoy's
176			// operational parameters correctly.
177			proxyIPv6 := isIPv6Proxy(role.IPAddresses)
178			if len(role.ID) == 0 {
179				if registryID == serviceregistry.Kubernetes {
180					role.ID = podName + "." + podNamespace
181				} else if registryID == serviceregistry.Consul {
182					role.ID = role.IPAddresses[0] + ".service.consul"
183				} else {
184					role.ID = role.IPAddresses[0]
185				}
186			}
187
188			proxyConfig, err := constructProxyConfig()
189			if err != nil {
190				return fmt.Errorf("failed to get proxy config: %v", err)
191			}
192			if out, err := gogoprotomarshal.ToYAML(&proxyConfig); err != nil {
193				log.Infof("Failed to serialize to YAML: %v", err)
194			} else {
195				log.Infof("Effective config: %s", out)
196			}
197
198			// If not set, set a default based on platform - podNamespace.svc.cluster.local for
199			// K8S
200			role.DNSDomain = getDNSDomain(podNamespace, role.DNSDomain)
201			log.Infof("Proxy role: %#v", role)
202
203			var jwtPath string
204			if jwtPolicy.Get() == jwt.JWTPolicyThirdPartyJWT {
205				log.Info("JWT policy is third-party-jwt")
206				jwtPath = trustworthyJWTPath
207			} else if jwtPolicy.Get() == jwt.JWTPolicyFirstPartyJWT {
208				log.Info("JWT policy is first-party-jwt")
209				jwtPath = securityModel.K8sSAJwtFileName
210			} else {
211				log.Info("Using existing certs")
212			}
213			sa := istio_agent.NewSDSAgent(proxyConfig.DiscoveryAddress, proxyConfig.ControlPlaneAuthPolicy == meshconfig.AuthenticationPolicy_MUTUAL_TLS,
214				pilotCertProvider, jwtPath, outputKeyCertToDir, clusterIDVar.Get())
215
216			// Connection to Istiod secure port
217			if sa.RequireCerts {
218				proxyConfig.ControlPlaneAuthPolicy = meshconfig.AuthenticationPolicy_MUTUAL_TLS
219			}
220
221			var pilotSAN, mixerSAN []string
222			if proxyConfig.ControlPlaneAuthPolicy == meshconfig.AuthenticationPolicy_MUTUAL_TLS {
223				setSpiffeTrustDomain(podNamespace, role.DNSDomain)
224				// Obtain the Mixer SAN, which uses SPIFFE certs. Used below to create a Envoy proxy.
225				mixerSAN = getSAN(getControlPlaneNamespace(podNamespace, proxyConfig.DiscoveryAddress), envoyDiscovery.MixerSvcAccName, mixerIdentity)
226				// Obtain Pilot SAN, using DNS.
227				pilotSAN = []string{getPilotSan(proxyConfig.DiscoveryAddress)}
228			}
229			log.Infof("PilotSAN %#v", pilotSAN)
230			log.Infof("MixerSAN %#v", mixerSAN)
231
232			// Start in process SDS.
233			_, err = sa.Start(role.Type == model.SidecarProxy, podNamespaceVar.Get())
234			if err != nil {
235				log.Fatala("Failed to start in-process SDS", err)
236			}
237
238			// dedupe cert paths so we don't set up 2 watchers for the same file
239			tlsCerts := dedupeStrings(getTLSCerts(proxyConfig))
240
241			// Since Envoy needs the file-mounted certs for mTLS, we wait for them to become available
242			// before starting it.
243			if len(tlsCerts) > 0 {
244				log.Infof("Monitored certs: %#v", tlsCerts)
245				for _, cert := range tlsCerts {
246					waitForFile(cert, 2*time.Minute)
247				}
248			}
249
250			// If we are using a custom template file (for control plane proxy, for example), configure this.
251			if templateFile != "" && proxyConfig.CustomConfigFile == "" {
252				proxyConfig.ProxyBootstrapTemplatePath = templateFile
253			}
254
255			ctx, cancel := context.WithCancel(context.Background())
256
257			// If a status port was provided, start handling status probes.
258			if proxyConfig.StatusPort > 0 {
259				localHostAddr := localHostIPv4
260				if proxyIPv6 {
261					localHostAddr = localHostIPv6
262				}
263				prober := kubeAppProberNameVar.Get()
264				statusServer, err := status.NewServer(status.Config{
265					LocalHostAddr:  localHostAddr,
266					AdminPort:      uint16(proxyConfig.ProxyAdminPort),
267					StatusPort:     uint16(proxyConfig.StatusPort),
268					KubeAppProbers: prober,
269					NodeType:       role.Type,
270				})
271				if err != nil {
272					cancel()
273					return err
274				}
275				go statusServer.Run(ctx)
276			}
277
278			// If security token service (STS) port is not zero, start STS server and
279			// listen on STS port for STS requests. For STS, see
280			// https://tools.ietf.org/html/draft-ietf-oauth-token-exchange-16.
281			if stsPort > 0 {
282				localHostAddr := localHostIPv4
283				if proxyIPv6 {
284					localHostAddr = localHostIPv6
285				}
286				tokenManager := tokenmanager.CreateTokenManager(tokenManagerPlugin,
287					tokenmanager.Config{TrustDomain: trustDomain})
288				stsServer, err := stsserver.NewServer(stsserver.Config{
289					LocalHostAddr: localHostAddr,
290					LocalPort:     stsPort,
291				}, tokenManager)
292				if err != nil {
293					cancel()
294					return err
295				}
296				defer stsServer.Stop()
297			}
298
299			// Start a local DNS server on 15053, forwarding to DNS-over-TLS server
300			// This will not have any impact on app unless interception is enabled.
301			// We can't start on 53 - istio-agent runs as user istio-proxy.
302			// This is available to apps even if interception is not enabled.
303
304			// TODO: replace hardcoded .global. Right now the ingress templates are
305			// hardcoding it as well, so there is little benefit to do it only here.
306			if dns.DNSTLSEnableAgent.Get() != "" {
307				// In the injection template the only place where global.proxy.clusterDomain
308				// is made available is in the --domain param.
309				// Instead of introducing a new config, use that.
310
311				dnsSrv := dns.InitDNSAgent(proxyConfig.DiscoveryAddress,
312					role.DNSDomain, sa.RootCert,
313					[]string{".global."})
314				dnsSrv.StartDNS(dns.DNSAgentAddr, nil)
315			}
316
317			envoyProxy := envoy.NewProxy(envoy.ProxyConfig{
318				Config:              proxyConfig,
319				Node:                role.ServiceNode(),
320				LogLevel:            proxyLogLevel,
321				ComponentLogLevel:   proxyComponentLogLevel,
322				PilotSubjectAltName: pilotSAN,
323				MixerSubjectAltName: mixerSAN,
324				NodeIPs:             role.IPAddresses,
325				PodName:             podName,
326				PodNamespace:        podNamespace,
327				PodIP:               podIP,
328				STSPort:             stsPort,
329				ControlPlaneAuth:    proxyConfig.ControlPlaneAuthPolicy == meshconfig.AuthenticationPolicy_MUTUAL_TLS,
330				DisableReportCalls:  disableInternalTelemetry,
331				OutlierLogPath:      outlierLogPath,
332				PilotCertProvider:   pilotCertProvider,
333				ProvCert:            citadel.ProvCert,
334			})
335
336			agent := envoy.NewAgent(envoyProxy, features.TerminationDrainDuration())
337
338			// Watcher is also kicking envoy start.
339			watcher := envoy.NewWatcher(tlsCerts, agent.Restart)
340			go watcher.Run(ctx)
341
342			// On SIGINT or SIGTERM, cancel the context, triggering a graceful shutdown
343			go cmd.WaitSignalFunc(cancel)
344
345			return agent.Run(ctx)
346		},
347	}
348)
349
350// dedupes the string array and also ignores the empty string.
351func dedupeStrings(in []string) []string {
352	set := sets.NewSet(in...)
353	return set.UnsortedList()
354}
355
356// explicitly set the trustdomain so the pilot and mixer SAN will have same trustdomain
357// and the initialization of the spiffe pkg isn't linked to generating pilot's SAN first
358func setSpiffeTrustDomain(podNamespace string, domain string) {
359	pilotTrustDomain := trustDomain
360	if len(pilotTrustDomain) == 0 {
361		if registryID == serviceregistry.Kubernetes &&
362			(domain == podNamespace+".svc.cluster.local" || domain == "") {
363			pilotTrustDomain = "cluster.local"
364		} else if registryID == serviceregistry.Consul &&
365			(domain == "service.consul" || domain == "") {
366			pilotTrustDomain = ""
367		} else {
368			pilotTrustDomain = domain
369		}
370	}
371	spiffe.SetTrustDomain(pilotTrustDomain)
372}
373
374func getSAN(ns string, defaultSA string, overrideIdentity string) []string {
375	var san []string
376	if overrideIdentity == "" {
377		san = append(san, envoyDiscovery.GetSAN(ns, defaultSA))
378	} else {
379		san = append(san, envoyDiscovery.GetSAN("", overrideIdentity))
380
381	}
382	return san
383}
384
385func getDNSDomain(podNamespace, domain string) string {
386	if len(domain) == 0 {
387		if registryID == serviceregistry.Kubernetes {
388			domain = podNamespace + ".svc.cluster.local"
389		} else if registryID == serviceregistry.Consul {
390			domain = "service.consul"
391		} else {
392			domain = ""
393		}
394	}
395	return domain
396}
397
398func fromJSON(j string) *meshconfig.RemoteService {
399	var m meshconfig.RemoteService
400	err := jsonpb.UnmarshalString(j, &m)
401	if err != nil {
402		log.Warnf("Unable to unmarshal %s: %v", j, err)
403		return nil
404	}
405
406	return &m
407}
408
409func init() {
410	proxyCmd.PersistentFlags().StringVar((*string)(&registryID), "serviceregistry",
411		string(serviceregistry.Kubernetes),
412		fmt.Sprintf("Select the platform for service registry, options are {%s, %s, %s}",
413			serviceregistry.Kubernetes, serviceregistry.Consul, serviceregistry.Mock))
414	proxyCmd.PersistentFlags().StringVar(&proxyIP, "ip", "",
415		"Proxy IP address. If not provided uses ${INSTANCE_IP} environment variable.")
416	proxyCmd.PersistentFlags().StringVar(&role.ID, "id", "",
417		"Proxy unique ID. If not provided uses ${POD_NAME}.${POD_NAMESPACE} from environment variables")
418	proxyCmd.PersistentFlags().StringVar(&role.DNSDomain, "domain", "",
419		"DNS domain suffix. If not provided uses ${POD_NAMESPACE}.svc.cluster.local")
420	proxyCmd.PersistentFlags().StringVar(&trustDomain, "trust-domain", "",
421		"The domain to use for identities")
422	proxyCmd.PersistentFlags().StringVar(&pilotIdentity, "pilotIdentity", "",
423		"The identity used as the suffix for pilot's spiffe SAN ")
424	proxyCmd.PersistentFlags().StringVar(&mixerIdentity, "mixerIdentity", "",
425		"The identity used as the suffix for mixer's spiffe SAN. This would only be used by pilot all other proxy would get this value from pilot")
426
427	proxyCmd.PersistentFlags().StringVar(&meshConfigFile, "meshConfig", "./etc/istio/config/mesh",
428		"File name for Istio mesh configuration. If not specified, a default mesh will be used. This may be overridden by "+
429			"PROXY_CONFIG environment variable or proxy.istio.io/config annotation.")
430	proxyCmd.PersistentFlags().IntVar(&stsPort, "stsPort", 0,
431		"HTTP Port on which to serve Security Token Service (STS). If zero, STS service will not be provided.")
432	proxyCmd.PersistentFlags().StringVar(&tokenManagerPlugin, "tokenManagerPlugin", tokenmanager.GoogleTokenExchange,
433		"Token provider specific plugin name.")
434	// Flags for proxy configuration
435	proxyCmd.PersistentFlags().StringVar(&serviceCluster, "serviceCluster", constants.ServiceClusterName, "Service cluster")
436	// Log levels are provided by the library https://github.com/gabime/spdlog, used by Envoy.
437	proxyCmd.PersistentFlags().StringVar(&proxyLogLevel, "proxyLogLevel", "warning",
438		fmt.Sprintf("The log level used to start the Envoy proxy (choose from {%s, %s, %s, %s, %s, %s, %s})",
439			"trace", "debug", "info", "warning", "error", "critical", "off"))
440	proxyCmd.PersistentFlags().IntVar(&concurrency, "concurrency", 0, "number of worker threads to run")
441	// See https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-component-log-level
442	proxyCmd.PersistentFlags().StringVar(&proxyComponentLogLevel, "proxyComponentLogLevel", "misc:error",
443		"The component log level used to start the Envoy proxy")
444	proxyCmd.PersistentFlags().StringVar(&templateFile, "templateFile", "",
445		"Go template bootstrap config")
446	proxyCmd.PersistentFlags().BoolVar(&disableInternalTelemetry, "disableInternalTelemetry", false,
447		"Disable internal telemetry")
448	proxyCmd.PersistentFlags().StringVar(&outlierLogPath, "outlierLogPath", "",
449		"The log path for outlier detection")
450
451	// Attach the Istio logging options to the command.
452	loggingOptions.AttachCobraFlags(rootCmd)
453
454	cmd.AddFlags(rootCmd)
455
456	rootCmd.AddCommand(proxyCmd)
457	rootCmd.AddCommand(version.CobraCommand())
458	rootCmd.AddCommand(iptables.GetCommand())
459	rootCmd.AddCommand(cleaniptables.GetCommand())
460
461	rootCmd.AddCommand(collateral.CobraCommand(rootCmd, &doc.GenManHeader{
462		Title:   "Istio Pilot Agent",
463		Section: "pilot-agent CLI",
464		Manual:  "Istio Pilot Agent",
465	}))
466}
467
468func waitForFile(fname string, maxWait time.Duration) bool {
469	log.Infof("waiting %v for %s", maxWait, fname)
470
471	logDelay := 1 * time.Second
472	nextLog := time.Now().Add(logDelay)
473	endWait := time.Now().Add(maxWait)
474
475	for {
476		_, err := os.Stat(fname)
477		if err == nil {
478			return true
479		}
480		if !os.IsNotExist(err) { // another error (e.g., permission) - likely no point in waiting longer
481			log.Errora("error while waiting for file", err.Error())
482			return false
483		}
484
485		now := time.Now()
486		if now.After(endWait) {
487			log.Warna("file still not available after", maxWait)
488			return false
489		}
490		if now.After(nextLog) {
491			log.Infof("waiting for file")
492			logDelay *= 2
493			nextLog.Add(logDelay)
494		}
495		time.Sleep(100 * time.Millisecond)
496	}
497}
498
499// TODO: get the config and bootstrap from istiod, by passing the env
500
501// Use env variables - from injection, k8s and local namespace config map.
502// No CLI parameters.
503func main() {
504	if err := rootCmd.Execute(); err != nil {
505		log.Errora(err)
506		os.Exit(-1)
507	}
508}
509
510// isIPv6Proxy check the addresses slice and returns true for a valid IPv6 address
511// for all other cases it returns false
512func isIPv6Proxy(ipAddrs []string) bool {
513	for i := 0; i < len(ipAddrs); i++ {
514		addr := net.ParseIP(ipAddrs[i])
515		if addr == nil {
516			// Should not happen, invalid IP in proxy's IPAddresses slice should have been caught earlier,
517			// skip it to prevent a panic.
518			continue
519		}
520		if addr.To4() != nil {
521			return false
522		}
523	}
524	return true
525}
526