1// Copyright 2019 Istio Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package mesh 16 17import ( 18 "fmt" 19 "os" 20 "path/filepath" 21 "strings" 22 23 "gopkg.in/yaml.v2" 24 "k8s.io/client-go/rest" 25 26 "istio.io/api/operator/v1alpha1" 27 iopv1alpha1 "istio.io/istio/operator/pkg/apis/istio/v1alpha1" 28 "istio.io/istio/operator/pkg/apis/istio/v1alpha1/validation" 29 "istio.io/istio/operator/pkg/helm" 30 "istio.io/istio/operator/pkg/name" 31 "istio.io/istio/operator/pkg/tpath" 32 "istio.io/istio/operator/pkg/translate" 33 "istio.io/istio/operator/pkg/util" 34 "istio.io/istio/operator/pkg/util/clog" 35 "istio.io/istio/operator/pkg/validate" 36 "istio.io/istio/operator/version" 37 pkgversion "istio.io/pkg/version" 38) 39 40// GenerateConfig creates an IstioOperatorSpec from the following sources, overlaid sequentially: 41// 1. Compiled in base, or optionally base from paths pointing to one or multiple ICP/IOP files at inFilenames. 42// 2. Profile overlay, if non-default overlay is selected. This also comes either from compiled in or path specified in IOP contained in inFilenames. 43// 3. User overlays stored in inFilenames. 44// 4. setOverlayYAML, which comes from --set flag passed to manifest command. 45// 46// Note that the user overlay at inFilenames can optionally contain a file path to a set of profiles different from the 47// ones that are compiled in. If it does, the starting point will be the base and profile YAMLs at that file path. 48// Otherwise it will be the compiled in profile YAMLs. 49// In step 3, the remaining fields in the same user overlay are applied on the resulting profile base. 50// The force flag causes validation errors not to abort but only emit log/console warnings. 51func GenerateConfig(inFilenames []string, setFlags []string, force bool, kubeConfig *rest.Config, 52 l clog.Logger) (string, *v1alpha1.IstioOperatorSpec, error) { 53 if err := validateSetFlags(setFlags); err != nil { 54 return "", nil, err 55 } 56 57 fy, profile, err := readYamlProfile(inFilenames, setFlags, force, l) 58 if err != nil { 59 return "", nil, err 60 } 61 62 iopsString, iops, err := genIOPSFromProfile(profile, fy, setFlags, force, kubeConfig, l) 63 if err != nil { 64 return "", nil, err 65 } 66 67 errs, warning := validation.ValidateConfig(false, iops.Values, iops) 68 if warning != "" { 69 l.LogAndError(warning) 70 } 71 72 if errs.ToError() != nil { 73 return "", nil, fmt.Errorf("generated config failed semantic validation: %v", errs) 74 } 75 return iopsString, iops, nil 76} 77 78func readYamlProfile(inFilenames []string, setFlags []string, force bool, l clog.Logger) (string, string, error) { 79 profile := name.DefaultProfileName 80 // Get the overlay YAML from the list of files passed in. Also get the profile from the overlay files. 81 fy, fp, err := parseYAMLFiles(inFilenames, force, l) 82 if err != nil { 83 return "", "", err 84 } 85 if fp != "" { 86 profile = fp 87 } 88 // The profile coming from --set flag has the highest precedence. 89 psf := getValueForSetFlag(setFlags, "profile") 90 if psf != "" { 91 profile = psf 92 } 93 return fy, profile, nil 94} 95 96// parseYAMLFiles parses the given slice of filenames containing YAML and merges them into a single IstioOperator 97// format YAML strings. It returns the overlay YAML, the profile name and error result. 98func parseYAMLFiles(inFilenames []string, force bool, l clog.Logger) (overlayYAML string, profile string, err error) { 99 if inFilenames == nil { 100 return "", "", nil 101 } 102 y, err := ReadLayeredYAMLs(inFilenames) 103 if err != nil { 104 return "", "", err 105 } 106 var fileOverlayIOP *iopv1alpha1.IstioOperator 107 fileOverlayIOP, err = validate.UnmarshalIOP(y) 108 if err != nil { 109 return "", "", err 110 } 111 if err := validate.ValidIOP(fileOverlayIOP); err != nil { 112 if !force { 113 return "", "", fmt.Errorf("validation errors (use --force to override): \n%s", err) 114 } 115 l.LogAndErrorf("Validation errors (continuing because of --force):\n%s", err) 116 } 117 if fileOverlayIOP.Spec != nil && fileOverlayIOP.Spec.Profile != "" { 118 if profile != "" && profile != fileOverlayIOP.Spec.Profile { 119 return "", "", fmt.Errorf("different profiles cannot be overlaid") 120 } 121 profile = fileOverlayIOP.Spec.Profile 122 } 123 return y, profile, nil 124} 125 126// genIOPSFromProfile generates an IstioOperatorSpec from the given profile name or path, and overlay YAMLs from user 127// files and the --set flag. If successful, it returns an IstioOperatorSpec string and struct. 128func genIOPSFromProfile(profileOrPath, fileOverlayYAML string, setFlags []string, skipValidation bool, 129 kubeConfig *rest.Config, l clog.Logger) (string, *v1alpha1.IstioOperatorSpec, error) { 130 131 installPackagePath, err := getInstallPackagePath(fileOverlayYAML) 132 if err != nil { 133 return "", nil, err 134 } 135 if sfp := getValueForSetFlag(setFlags, "installPackagePath"); sfp != "" { 136 // set flag installPackagePath has the highest precedence, if set. 137 installPackagePath = sfp 138 } 139 140 // If installPackagePath is a URL, fetch and extract it and continue with the local filesystem path instead. 141 installPackagePath, profileOrPath, err = rewriteURLToLocalInstallPath(installPackagePath, profileOrPath, skipValidation) 142 if err != nil { 143 return "", nil, err 144 } 145 146 // To generate the base profileOrPath for overlaying with user values, we need the installPackagePath where the profiles 147 // can be found, and the selected profileOrPath. Both of these can come from either the user overlay file or --set flag. 148 outYAML, err := helm.GetProfileYAML(installPackagePath, profileOrPath) 149 if err != nil { 150 return "", nil, err 151 } 152 153 // Hub and tag are only known at build time and must be passed in here during runtime from build stamps. 154 outYAML, err = overlayHubAndTag(outYAML) 155 if err != nil { 156 return "", nil, err 157 } 158 159 // Merge k8s specific values. 160 if kubeConfig != nil { 161 kubeOverrides, err := getClusterSpecificValues(kubeConfig, skipValidation, l) 162 if err != nil { 163 return "", nil, err 164 } 165 installerScope.Infof("Applying Cluster specific settings: %v", kubeOverrides) 166 outYAML, err = util.OverlayYAML(outYAML, kubeOverrides) 167 if err != nil { 168 return "", nil, err 169 } 170 } 171 172 // Combine file and --set overlays and translate any K8s settings in values to IOP format. Users should not set 173 // these but we have to support this path until it's deprecated. 174 overlayYAML, err := overlaySetFlagValues(fileOverlayYAML, setFlags) 175 if err != nil { 176 return "", nil, err 177 } 178 mvs := version.OperatorBinaryVersion.MinorVersion 179 t, err := translate.NewReverseTranslator(mvs) 180 if err != nil { 181 return "", nil, fmt.Errorf("error creating values.yaml translator: %s", err) 182 } 183 overlayYAML, err = t.TranslateK8SfromValueToIOP(overlayYAML) 184 if err != nil { 185 return "", nil, fmt.Errorf("could not overlay k8s settings from values to IOP: %s", err) 186 } 187 188 // Merge user file and --set flags. 189 outYAML, err = util.OverlayYAML(outYAML, overlayYAML) 190 if err != nil { 191 return "", nil, fmt.Errorf("could not overlay user config over base: %s", err) 192 } 193 194 if err := name.ScanBundledAddonComponents(installPackagePath); err != nil { 195 return "", nil, err 196 } 197 // If enablement came from user values overlay (file or --set), translate into addonComponents paths and overlay that. 198 outYAML, err = translate.OverlayValuesEnablement(outYAML, overlayYAML, overlayYAML) 199 if err != nil { 200 return "", nil, err 201 } 202 203 // Grab just the IstioOperatorSpec subtree. 204 outYAML, err = tpath.GetSpecSubtree(outYAML) 205 if err != nil { 206 return "", nil, err 207 } 208 209 finalIOPS, err := unmarshalAndValidateIOPS(outYAML, skipValidation, l) 210 if err != nil { 211 return "", nil, err 212 } 213 // InstallPackagePath may have been a URL, change to extracted to local file path. 214 finalIOPS.InstallPackagePath = installPackagePath 215 return util.ToYAMLWithJSONPB(finalIOPS), finalIOPS, nil 216} 217 218// rewriteURLToLocalInstallPath checks installPackagePath and if it is a URL, it tries to download and extract the 219// Istio release tar at the URL to a local file path. If successful, it returns the resulting local paths to the 220// installation charts and profile file. 221// If installPackagePath is not a URL, it returns installPackagePath and profileOrPath unmodified. 222func rewriteURLToLocalInstallPath(installPackagePath, profileOrPath string, skipValidation bool) (string, string, error) { 223 isURL, err := util.IsHTTPURL(installPackagePath) 224 if err != nil && !skipValidation { 225 return "", "", err 226 } 227 if isURL { 228 installPackagePath, err = fetchExtractInstallPackageHTTP(installPackagePath) 229 if err != nil { 230 return "", "", err 231 } 232 // Transform a profileOrPath like "default" or "demo" into a filesystem path like 233 // /tmp/istio-install-packages/istio-1.5.1/manifests/profiles/default.yaml OR 234 // /tmp/istio-install-packages/istio-1.5.1/install/kubernetes/operator/profiles/default.yaml (before 1.6). 235 baseDir := filepath.Join(installPackagePath, helm.OperatorSubdirFilePath15) 236 if _, err := os.Stat(baseDir); os.IsNotExist(err) { 237 baseDir = filepath.Join(installPackagePath, helm.OperatorSubdirFilePath) 238 } 239 profileOrPath = filepath.Join(baseDir, "profiles", profileOrPath+".yaml") 240 // Rewrite installPackagePath to the local file path for further processing. 241 installPackagePath = baseDir 242 } 243 244 return installPackagePath, profileOrPath, nil 245} 246 247// Due to the fact that base profile is compiled in before a tag can be created, we must allow an additional 248// override from variables that are set during release build time. 249func overlayHubAndTag(yml string) (string, error) { 250 hub := pkgversion.DockerInfo.Hub 251 tag := pkgversion.DockerInfo.Tag 252 out := yml 253 if hub != "unknown" && tag != "unknown" { 254 buildHubTagOverlayYAML, err := helm.GenerateHubTagOverlay(hub, tag) 255 if err != nil { 256 return "", err 257 } 258 out, err = util.OverlayYAML(yml, buildHubTagOverlayYAML) 259 if err != nil { 260 return "", err 261 } 262 } 263 return out, nil 264} 265 266func getClusterSpecificValues(config *rest.Config, force bool, l clog.Logger) (string, error) { 267 overlays := []string{} 268 269 jwt, err := getJwtTypeOverlay(config, l) 270 if err != nil { 271 if force { 272 l.LogAndPrint(err) 273 } else { 274 return "", err 275 } 276 } else { 277 overlays = append(overlays, jwt) 278 } 279 280 return makeTreeFromSetList(overlays) 281 282} 283 284func getJwtTypeOverlay(config *rest.Config, l clog.Logger) (string, error) { 285 jwtPolicy, err := util.DetectSupportedJWTPolicy(config) 286 if err != nil { 287 return "", fmt.Errorf("failed to determine JWT policy support. Use the --force flag to ignore this: %v", err) 288 } 289 if jwtPolicy == util.FirstPartyJWT { 290 // nolint: lll 291 l.LogAndPrint("Detected that your cluster does not support third party JWT authentication. " + 292 "Falling back to less secure first party JWT. See https://istio.io/docs/ops/best-practices/security/#configure-third-party-service-account-tokens for details.") 293 } 294 return "values.global.jwtPolicy=" + string(jwtPolicy), nil 295} 296 297// unmarshalAndValidateIOPS unmarshals a string containing IstioOperator YAML, validates it, and returns a struct 298// representation if successful. If force is set, validation errors are written to logger rather than causing an 299// error. 300func unmarshalAndValidateIOPS(iopsYAML string, force bool, l clog.Logger) (*v1alpha1.IstioOperatorSpec, error) { 301 iops := &v1alpha1.IstioOperatorSpec{} 302 if err := util.UnmarshalWithJSONPB(iopsYAML, iops, false); err != nil { 303 return nil, fmt.Errorf("could not unmarshal merged YAML: %s\n\nYAML:\n%s", err, iopsYAML) 304 } 305 if errs := validate.CheckIstioOperatorSpec(iops, true); len(errs) != 0 && !force { 306 l.LogAndError("Run the command with the --force flag if you want to ignore the validation error and proceed.") 307 return iops, fmt.Errorf(errs.Error()) 308 } 309 return iops, nil 310} 311 312// getInstallPackagePath returns the installPackagePath in the given IstioOperator YAML string. 313func getInstallPackagePath(iopYAML string) (string, error) { 314 iop, err := validate.UnmarshalIOP(iopYAML) 315 if err != nil { 316 return "", err 317 } 318 if iop.Spec == nil { 319 return "", nil 320 } 321 return iop.Spec.InstallPackagePath, nil 322} 323 324// validateSetFlags validates that setFlags all have path=value format. 325func validateSetFlags(setFlags []string) error { 326 for _, sf := range setFlags { 327 pv := strings.Split(sf, "=") 328 if len(pv) != 2 { 329 return fmt.Errorf("set flag %s has incorrect format, must be path=value", sf) 330 } 331 } 332 return nil 333} 334 335// overlaySetFlagValues overlays each of the setFlags on top of the passed in IOP YAML string. 336func overlaySetFlagValues(iopYAML string, setFlags []string) (string, error) { 337 iop := make(map[string]interface{}) 338 if err := yaml.Unmarshal([]byte(iopYAML), &iop); err != nil { 339 return "", err 340 } 341 // Unmarshal returns nil for empty manifests but we need something to insert into. 342 if iop == nil { 343 iop = make(map[string]interface{}) 344 } 345 346 for _, sf := range setFlags { 347 p, v := getPV(sf) 348 p = strings.TrimPrefix(p, "spec.") 349 inc, _, err := tpath.GetPathContext(iop, util.PathFromString("spec."+p), true) 350 if err != nil { 351 return "", err 352 } 353 // input value type is always string, transform it to correct type before setting. 354 if err := tpath.WritePathContext(inc, util.ParseValue(v), false); err != nil { 355 return "", err 356 } 357 } 358 359 out, err := yaml.Marshal(iop) 360 if err != nil { 361 return "", err 362 } 363 364 return string(out), nil 365} 366 367// getValueForSetFlag parses the passed set flags which have format key=value and if any set the given path, 368// returns the corresponding value, otherwise returns the empty string. setFlags must have valid format. 369func getValueForSetFlag(setFlags []string, path string) string { 370 ret := "" 371 for _, sf := range setFlags { 372 p, v := getPV(sf) 373 if p == path { 374 ret = v 375 } 376 // if set multiple times, return last set value 377 } 378 return ret 379} 380 381// getPV returns the path and value components for the given set flag string, which must be in path=value format. 382func getPV(setFlag string) (path string, value string) { 383 pv := strings.Split(setFlag, "=") 384 if len(pv) != 2 { 385 return setFlag, "" 386 } 387 path, value = strings.TrimSpace(pv[0]), strings.TrimSpace(pv[1]) 388 return 389} 390