1// Copyright 2019 Istio Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package bootstrap 16 17import ( 18 "encoding/json" 19 "fmt" 20 "io/ioutil" 21 "net" 22 "os" 23 "path" 24 "strconv" 25 "strings" 26 27 md "cloud.google.com/go/compute/metadata" 28 envoy_api_v2_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" 29 30 "istio.io/istio/pkg/config/constants" 31 32 "github.com/gogo/protobuf/types" 33 34 meshAPI "istio.io/api/mesh/v1alpha1" 35 "istio.io/pkg/log" 36 37 "istio.io/istio/pilot/pkg/model" 38 "istio.io/istio/pilot/pkg/networking/util" 39 "istio.io/istio/pkg/bootstrap/option" 40 "istio.io/istio/pkg/bootstrap/platform" 41 "istio.io/istio/pkg/spiffe" 42) 43 44const ( 45 // IstioMetaPrefix is used to pass env vars as node metadata. 46 IstioMetaPrefix = "ISTIO_META_" 47 48 // IstioMetaJSONPrefix is used to pass annotations and similar environment info. 49 IstioMetaJSONPrefix = "ISTIO_METAJSON_" 50 51 lightstepAccessTokenBase = "lightstep_access_token.txt" 52 53 // required stats are used by readiness checks. 54 requiredEnvoyStatsMatcherInclusionPrefixes = "cluster_manager,listener_manager,http_mixer_filter,tcp_mixer_filter,server,cluster.xds-grpc,wasm" 55 56 // Prefixes of V2 metrics. 57 // "reporter" prefix is for istio standard metrics. 58 // "component" suffix is for istio_build metric. 59 v2Prefixes = "reporter=," 60 v2Suffix = ",component" 61) 62 63var ( 64 // These must match the json field names in model.nodeMetadata 65 metadataExchangeKeys = []string{ 66 "NAME", 67 "NAMESPACE", 68 "INSTANCE_IPS", 69 "LABELS", 70 "OWNER", 71 "PLATFORM_METADATA", 72 "WORKLOAD_NAME", 73 "MESH_ID", 74 "SERVICE_ACCOUNT", 75 "CLUSTER_ID", 76 } 77) 78 79// Config for creating a bootstrap file. 80type Config struct { 81 Node string 82 Proxy *meshAPI.ProxyConfig 83 PlatEnv platform.Environment 84 PilotSubjectAltName []string 85 MixerSubjectAltName []string 86 LocalEnv []string 87 NodeIPs []string 88 PodName string 89 PodNamespace string 90 PodIP net.IP 91 STSPort int 92 ControlPlaneAuth bool 93 DisableReportCalls bool 94 OutlierLogPath string 95 PilotCertProvider string 96 ProvCert string 97} 98 99// newTemplateParams creates a new template configuration for the given configuration. 100func (cfg Config) toTemplateParams() (map[string]interface{}, error) { 101 opts := make([]option.Instance, 0) 102 103 // Fill in default config values. 104 if cfg.PilotSubjectAltName == nil { 105 cfg.PilotSubjectAltName = defaultPilotSAN() 106 } 107 if cfg.PlatEnv == nil { 108 cfg.PlatEnv = platform.Discover() 109 } 110 111 // Remove duplicates from the node IPs. 112 cfg.NodeIPs = removeDuplicates(cfg.NodeIPs) 113 114 opts = append(opts, 115 option.NodeID(cfg.Node), 116 option.PodName(cfg.PodName), 117 option.PodNamespace(cfg.PodNamespace), 118 option.PodIP(cfg.PodIP), 119 option.PilotSubjectAltName(cfg.PilotSubjectAltName), 120 option.MixerSubjectAltName(cfg.MixerSubjectAltName), 121 option.ControlPlaneAuth(cfg.ControlPlaneAuth), 122 option.DisableReportCalls(cfg.DisableReportCalls), 123 option.PilotCertProvider(cfg.PilotCertProvider), 124 option.OutlierLogPath(cfg.OutlierLogPath), 125 option.ProvCert(cfg.ProvCert)) 126 127 if cfg.STSPort > 0 { 128 opts = append(opts, 129 option.STSEnabled(true), 130 option.STSPort(cfg.STSPort)) 131 md := cfg.PlatEnv.Metadata() 132 if projectID, found := md[platform.GCPProject]; found { 133 opts = append(opts, option.GCPProjectID(projectID)) 134 } 135 } 136 137 // Support passing extra info from node environment as metadata 138 meta, rawMeta, err := getNodeMetaData(cfg.LocalEnv, cfg.PlatEnv, cfg.NodeIPs, cfg.STSPort, cfg.Proxy) 139 if err != nil { 140 return nil, err 141 } 142 opts = append(opts, getNodeMetadataOptions(meta, rawMeta, cfg.PlatEnv, cfg.Proxy)...) 143 144 // Check if nodeIP carries IPv4 or IPv6 and set up proxy accordingly 145 if isIPv6Proxy(cfg.NodeIPs) { 146 opts = append(opts, 147 option.Localhost(option.LocalhostIPv6), 148 option.Wildcard(option.WildcardIPv6), 149 option.DNSLookupFamily(option.DNSLookupFamilyIPv6)) 150 } else { 151 opts = append(opts, 152 option.Localhost(option.LocalhostIPv4), 153 option.Wildcard(option.WildcardIPv4), 154 option.DNSLookupFamily(option.DNSLookupFamilyIPv4)) 155 } 156 157 proxyOpts, err := getProxyConfigOptions(cfg.Proxy, meta) 158 if err != nil { 159 return nil, err 160 } 161 opts = append(opts, proxyOpts...) 162 163 // TODO: allow reading a file with additional metadata (for example if created with 164 // 'envref'. This will allow Istio to generate the right config even if the pod info 165 // is not available (in particular in some multi-cluster cases) 166 return option.NewTemplateParams(opts...) 167} 168 169// substituteValues substitutes variables known to the bootstrap like pod_ip. 170// "http.{pod_ip}_" with pod_id = [10.3.3.3,10.4.4.4] --> [http.10.3.3.3_,http.10.4.4.4_] 171func substituteValues(patterns []string, varName string, values []string) []string { 172 ret := make([]string, 0, len(patterns)) 173 for _, pattern := range patterns { 174 if !strings.Contains(pattern, varName) { 175 ret = append(ret, pattern) 176 continue 177 } 178 179 for _, val := range values { 180 ret = append(ret, strings.Replace(pattern, varName, val, -1)) 181 } 182 } 183 return ret 184} 185 186var ( 187 // DefaultStatTags for telemetry v2 tag extraction. 188 DefaultStatTags = []string{ 189 "reporter", 190 "source_namespace", 191 "source_workload", 192 "source_workload_namespace", 193 "source_principal", 194 "source_app", 195 "source_version", 196 "source_cluster", 197 "destination_namespace", 198 "destination_workload", 199 "destination_workload_namespace", 200 "destination_principal", 201 "destination_app", 202 "destination_version", 203 "destination_service", 204 "destination_service_name", 205 "destination_service_namespace", 206 "destination_port", 207 "destination_cluster", 208 "request_protocol", 209 "request_operation", 210 "request_host", 211 "response_flags", 212 "grpc_response_status", 213 "connection_security_policy", 214 "permissive_response_code", 215 "permissive_response_policyid", 216 "source_canonical_service", 217 "destination_canonical_service", 218 "source_canonical_revision", 219 "destination_canonical_revision", 220 } 221) 222 223func getStatsOptions(meta *model.NodeMetadata, nodeIPs []string, config *meshAPI.ProxyConfig) []option.Instance { 224 parseOption := func(metaOption string, required string) []string { 225 var inclusionOption []string 226 if len(metaOption) > 0 { 227 inclusionOption = strings.Split(metaOption, ",") 228 } 229 230 if len(required) > 0 { 231 inclusionOption = append(inclusionOption, strings.Split(required, ",")...) 232 } 233 234 // At the sidecar we can limit downstream metrics collection to the inbound listener. 235 // Inbound downstream metrics are named as: http.{pod_ip}_{port}.downstream_rq_* 236 // Other outbound downstream metrics are numerous and not very interesting for a sidecar. 237 // specifying http.{pod_ip}_ as a prefix will capture these downstream metrics. 238 return substituteValues(inclusionOption, "{pod_ip}", nodeIPs) 239 } 240 241 extraStatTags := make([]string, 0, len(DefaultStatTags)) 242 extraStatTags = append(extraStatTags, 243 DefaultStatTags...) 244 for _, tag := range config.ExtraStatTags { 245 if tag != "" { 246 extraStatTags = append(extraStatTags, tag) 247 } 248 } 249 for _, tag := range strings.Split(meta.ExtraStatTags, ",") { 250 if tag != "" { 251 extraStatTags = append(extraStatTags, tag) 252 } 253 } 254 255 return []option.Instance{ 256 option.EnvoyStatsMatcherInclusionPrefix(parseOption(meta.StatsInclusionPrefixes, requiredEnvoyStatsMatcherInclusionPrefixes)), 257 option.EnvoyStatsMatcherInclusionSuffix(parseOption(meta.StatsInclusionSuffixes, "")), 258 option.EnvoyStatsMatcherInclusionRegexp(parseOption(meta.StatsInclusionRegexps, "")), 259 option.EnvoyExtraStatTags(extraStatTags), 260 } 261} 262 263func defaultPilotSAN() []string { 264 return []string{ 265 spiffe.MustGenSpiffeURI("istio-system", "istio-pilot-service-account")} 266} 267 268func lightstepAccessTokenFile(config string) string { 269 return path.Join(config, lightstepAccessTokenBase) 270} 271 272func getNodeMetadataOptions(meta *model.NodeMetadata, rawMeta map[string]interface{}, 273 platEnv platform.Environment, config *meshAPI.ProxyConfig) []option.Instance { 274 // Add locality options. 275 opts := getLocalityOptions(meta, platEnv) 276 277 opts = append(opts, getStatsOptions(meta, meta.InstanceIPs, config)...) 278 279 opts = append(opts, option.NodeMetadata(meta, rawMeta)) 280 return opts 281} 282 283func getLocalityOptions(meta *model.NodeMetadata, platEnv platform.Environment) []option.Instance { 284 var l *envoy_api_v2_core.Locality 285 if meta.Labels[model.LocalityLabel] == "" { 286 l = platEnv.Locality() 287 // The locality string was not set, try to get locality from platform 288 } else { 289 localityString := model.GetLocalityLabelOrDefault(meta.Labels[model.LocalityLabel], "") 290 l = util.ConvertLocality(localityString) 291 } 292 293 return []option.Instance{option.Region(l.Region), option.Zone(l.Zone), option.SubZone(l.SubZone)} 294} 295 296func getProxyConfigOptions(config *meshAPI.ProxyConfig, metadata *model.NodeMetadata) ([]option.Instance, error) { 297 // Add a few misc options. 298 opts := make([]option.Instance, 0) 299 300 opts = append(opts, option.ProxyConfig(config), 301 option.Cluster(config.ServiceCluster), 302 option.PilotGRPCAddress(config.DiscoveryAddress), 303 option.DiscoveryAddress(config.DiscoveryAddress), 304 option.StatsdAddress(config.StatsdUdpAddress)) 305 306 // Add tracing options. 307 if config.Tracing != nil { 308 var isH2 bool = false 309 switch tracer := config.Tracing.Tracer.(type) { 310 case *meshAPI.Tracing_Zipkin_: 311 opts = append(opts, option.ZipkinAddress(tracer.Zipkin.Address)) 312 case *meshAPI.Tracing_Lightstep_: 313 isH2 = true 314 // Create the token file. 315 lightstepAccessTokenPath := lightstepAccessTokenFile(config.ConfigPath) 316 lsConfigOut, err := os.Create(lightstepAccessTokenPath) 317 if err != nil { 318 return nil, err 319 } 320 _, err = lsConfigOut.WriteString(tracer.Lightstep.AccessToken) 321 if err != nil { 322 return nil, err 323 } 324 325 opts = append(opts, option.LightstepAddress(tracer.Lightstep.Address), 326 option.LightstepToken(lightstepAccessTokenPath)) 327 case *meshAPI.Tracing_Datadog_: 328 opts = append(opts, option.DataDogAddress(tracer.Datadog.Address)) 329 case *meshAPI.Tracing_Stackdriver_: 330 var projectID string 331 var err error 332 if projectID, err = md.ProjectID(); err != nil { 333 return nil, fmt.Errorf("unable to process Stackdriver tracer: %v", err) 334 } 335 336 opts = append(opts, option.StackDriverEnabled(true), 337 option.StackDriverProjectID(projectID), 338 option.StackDriverDebug(tracer.Stackdriver.Debug), 339 option.StackDriverMaxAnnotations(getInt64ValueOrDefault(tracer.Stackdriver.MaxNumberOfAnnotations, 200)), 340 option.StackDriverMaxAttributes(getInt64ValueOrDefault(tracer.Stackdriver.MaxNumberOfAttributes, 200)), 341 option.StackDriverMaxEvents(getInt64ValueOrDefault(tracer.Stackdriver.MaxNumberOfMessageEvents, 200))) 342 } 343 opts = append(opts, option.TracingTLS(config.Tracing.TlsSettings, metadata, isH2)) 344 } 345 346 // Add options for Envoy metrics. 347 if config.EnvoyMetricsService != nil && config.EnvoyMetricsService.Address != "" { 348 opts = append(opts, option.EnvoyMetricsServiceAddress(config.EnvoyMetricsService.Address), 349 option.EnvoyMetricsServiceTLS(config.EnvoyMetricsService.TlsSettings, metadata), 350 option.EnvoyMetricsServiceTCPKeepalive(config.EnvoyMetricsService.TcpKeepalive)) 351 } else if config.EnvoyMetricsServiceAddress != "" { 352 opts = append(opts, option.EnvoyMetricsServiceAddress(config.EnvoyMetricsService.Address)) 353 } 354 355 // Add options for Envoy access log. 356 if config.EnvoyAccessLogService != nil && config.EnvoyAccessLogService.Address != "" { 357 opts = append(opts, option.EnvoyAccessLogServiceAddress(config.EnvoyAccessLogService.Address), 358 option.EnvoyAccessLogServiceTLS(config.EnvoyAccessLogService.TlsSettings, metadata), 359 option.EnvoyAccessLogServiceTCPKeepalive(config.EnvoyAccessLogService.TcpKeepalive)) 360 } 361 362 return opts, nil 363} 364 365func getInt64ValueOrDefault(src *types.Int64Value, defaultVal int64) int64 { 366 val := defaultVal 367 if src != nil { 368 val = src.Value 369 } 370 return val 371} 372 373// isIPv6Proxy check the addresses slice and returns true for a valid IPv6 address 374// for all other cases it returns false 375func isIPv6Proxy(ipAddrs []string) bool { 376 for i := 0; i < len(ipAddrs); i++ { 377 addr := net.ParseIP(ipAddrs[i]) 378 if addr == nil { 379 // Should not happen, invalid IP in proxy's IPAddresses slice should have been caught earlier, 380 // skip it to prevent a panic. 381 continue 382 } 383 if addr.To4() != nil { 384 return false 385 } 386 } 387 return true 388} 389 390type setMetaFunc func(m map[string]interface{}, key string, val string) 391 392func extractMetadata(envs []string, prefix string, set setMetaFunc, meta map[string]interface{}) { 393 metaPrefixLen := len(prefix) 394 for _, e := range envs { 395 if !shouldExtract(e, prefix) { 396 continue 397 } 398 v := e[metaPrefixLen:] 399 if !isEnvVar(v) { 400 continue 401 } 402 metaKey, metaVal := parseEnvVar(v) 403 set(meta, metaKey, metaVal) 404 } 405} 406 407func shouldExtract(envVar, prefix string) bool { 408 if strings.HasPrefix(envVar, "ISTIO_META_WORKLOAD") { 409 return false 410 } 411 return strings.HasPrefix(envVar, prefix) 412} 413 414func isEnvVar(str string) bool { 415 return strings.Contains(str, "=") 416} 417 418func parseEnvVar(varStr string) (string, string) { 419 parts := strings.SplitN(varStr, "=", 2) 420 if len(parts) != 2 { 421 return varStr, "" 422 } 423 return parts[0], parts[1] 424} 425 426func jsonStringToMap(jsonStr string) (m map[string]string) { 427 err := json.Unmarshal([]byte(jsonStr), &m) 428 if err != nil { 429 log.Warnf("Env variable with value %q failed json unmarshal: %v", jsonStr, err) 430 } 431 return 432} 433 434func extractAttributesMetadata(envVars []string, plat platform.Environment, meta *model.NodeMetadata) { 435 var additionalMetaExchangeKeys []string 436 for _, varStr := range envVars { 437 name, val := parseEnvVar(varStr) 438 switch name { 439 case "ISTIO_METAJSON_LABELS": 440 m := jsonStringToMap(val) 441 if len(m) > 0 { 442 meta.Labels = m 443 } 444 case "POD_NAME": 445 meta.InstanceName = val 446 case "POD_NAMESPACE": 447 meta.Namespace = val 448 meta.ConfigNamespace = val 449 case "ISTIO_META_OWNER": 450 meta.Owner = val 451 case "ISTIO_META_WORKLOAD_NAME": 452 meta.WorkloadName = val 453 case "SERVICE_ACCOUNT": 454 meta.ServiceAccount = val 455 case "ISTIO_ADDITIONAL_METADATA_EXCHANGE_KEYS": 456 // comma separated list of keys 457 additionalMetaExchangeKeys = strings.Split(val, ",") 458 } 459 } 460 if plat != nil && len(plat.Metadata()) > 0 { 461 meta.PlatformMetadata = plat.Metadata() 462 } 463 meta.ExchangeKeys = []string{} 464 meta.ExchangeKeys = append(meta.ExchangeKeys, metadataExchangeKeys...) 465 meta.ExchangeKeys = append(meta.ExchangeKeys, additionalMetaExchangeKeys...) 466 467} 468 469// getNodeMetaData function uses an environment variable contract 470// ISTIO_METAJSON_* env variables contain json_string in the value. 471// The name of variable is ignored. 472// ISTIO_META_* env variables are passed thru 473func getNodeMetaData(envs []string, plat platform.Environment, nodeIPs []string, 474 stsPort int, pc *meshAPI.ProxyConfig) (*model.NodeMetadata, map[string]interface{}, error) { 475 meta := &model.NodeMetadata{} 476 untypedMeta := map[string]interface{}{} 477 478 extractMetadata(envs, IstioMetaPrefix, func(m map[string]interface{}, key string, val string) { 479 m[key] = val 480 }, untypedMeta) 481 482 extractMetadata(envs, IstioMetaJSONPrefix, func(m map[string]interface{}, key string, val string) { 483 err := json.Unmarshal([]byte(val), &m) 484 if err != nil { 485 log.Warnf("Env variable %s [%s] failed json unmarshal: %v", key, val, err) 486 } 487 }, untypedMeta) 488 489 j, err := json.Marshal(untypedMeta) 490 if err != nil { 491 return nil, nil, err 492 } 493 if err := meta.UnmarshalJSON(j); err != nil { 494 return nil, nil, err 495 } 496 extractAttributesMetadata(envs, plat, meta) 497 498 // Support multiple network interfaces, removing duplicates. 499 meta.InstanceIPs = nodeIPs 500 501 // sds is enabled by default 502 meta.SdsEnabled = true 503 meta.SdsTrustJwt = true 504 505 // Add STS port into node metadata if it is not 0. 506 if stsPort != 0 { 507 meta.StsPort = strconv.Itoa(stsPort) 508 } 509 510 meta.ProxyConfig = (*model.NodeMetaProxyConfig)(pc) 511 512 // Add all pod labels found from filesystem 513 // These are typically volume mounted by the downward API 514 lbls, err := readPodLabels() 515 if err == nil { 516 if meta.Labels == nil { 517 meta.Labels = map[string]string{} 518 } 519 for k, v := range lbls { 520 meta.Labels[k] = v 521 } 522 } else { 523 log.Warnf("failed to read pod labels: %v", err) 524 } 525 526 return meta, untypedMeta, nil 527} 528 529func readPodLabels() (map[string]string, error) { 530 b, err := ioutil.ReadFile(constants.PodInfoLabelsPath) 531 if err != nil { 532 return nil, err 533 } 534 return ParseDownwardAPI(string(b)) 535} 536 537// Fields are stored as format `%s=%q`, we will parse this back to a map 538func ParseDownwardAPI(i string) (map[string]string, error) { 539 res := map[string]string{} 540 for _, line := range strings.Split(i, "\n") { 541 sl := strings.SplitN(line, "=", 2) 542 if len(sl) != 2 { 543 continue 544 } 545 key := sl[0] 546 // Strip the leading/trailing quotes 547 548 val, err := strconv.Unquote(sl[1]) 549 if err != nil { 550 return nil, fmt.Errorf("failed to unquote %v: %v", sl[1], err) 551 } 552 res[key] = val 553 } 554 return res, nil 555} 556 557func removeDuplicates(values []string) []string { 558 set := make(map[string]struct{}) 559 newValues := make([]string, 0, len(values)) 560 for _, v := range values { 561 if _, ok := set[v]; !ok { 562 set[v] = struct{}{} 563 newValues = append(newValues, v) 564 } 565 } 566 return newValues 567} 568