1package providercache 2 3import ( 4 "context" 5 "fmt" 6 "sort" 7 "strings" 8 9 "github.com/apparentlymart/go-versions/versions" 10 11 "github.com/hashicorp/terraform/internal/addrs" 12 copydir "github.com/hashicorp/terraform/internal/copy" 13 "github.com/hashicorp/terraform/internal/depsfile" 14 "github.com/hashicorp/terraform/internal/getproviders" 15) 16 17// Installer is the main type in this package, representing a provider installer 18// with a particular configuration-specific cache directory and an optional 19// global cache directory. 20type Installer struct { 21 // targetDir is the cache directory we're ultimately aiming to get the 22 // requested providers installed into. 23 targetDir *Dir 24 25 // source is the provider source that the installer will use to discover 26 // what provider versions are available for installation and to 27 // find the source locations for any versions that are not already 28 // available via one of the cache directories. 29 source getproviders.Source 30 31 // globalCacheDir is an optional additional directory that will, if 32 // provided, be treated as a read-through cache when retrieving new 33 // provider versions. That is, new packages are fetched into this 34 // directory first and then linked into targetDir, which allows sharing 35 // both the disk space and the download time for a particular provider 36 // version between different configurations on the same system. 37 globalCacheDir *Dir 38 39 // builtInProviderTypes is an optional set of types that should be 40 // considered valid to appear in the special terraform.io/builtin/... 41 // namespace, which we use for providers that are built in to Terraform 42 // and thus do not need any separate installation step. 43 builtInProviderTypes []string 44 45 // unmanagedProviderTypes is a set of provider addresses that should be 46 // considered implemented, but that Terraform does not manage the 47 // lifecycle for, and therefore does not need to worry about the 48 // installation of. 49 unmanagedProviderTypes map[addrs.Provider]struct{} 50} 51 52// NewInstaller constructs and returns a new installer with the given target 53// directory and provider source. 54// 55// A newly-created installer does not have a global cache directory configured, 56// but a caller can make a follow-up call to SetGlobalCacheDir to provide 57// one prior to taking any installation actions. 58// 59// The target directory MUST NOT also be an input consulted by the given source, 60// or the result is undefined. 61func NewInstaller(targetDir *Dir, source getproviders.Source) *Installer { 62 return &Installer{ 63 targetDir: targetDir, 64 source: source, 65 } 66} 67 68// Clone returns a new Installer which has the a new target directory but 69// the same optional global cache directory, the same installation sources, 70// and the same built-in/unmanaged providers. The result can be mutated further 71// using the various setter methods without affecting the original. 72func (i *Installer) Clone(targetDir *Dir) *Installer { 73 // For now all of our setter methods just overwrite field values in 74 // their entirety, rather than mutating things on the other side of 75 // the shared pointers, and so we can safely just shallow-copy the 76 // root. We might need to be more careful here if in future we add 77 // methods that allow deeper mutations through the stored pointers. 78 ret := *i 79 ret.targetDir = targetDir 80 return &ret 81} 82 83// ProviderSource returns the getproviders.Source that the installer would 84// use for installing any new providers. 85func (i *Installer) ProviderSource() getproviders.Source { 86 return i.source 87} 88 89// SetGlobalCacheDir activates a second tier of caching for the receiving 90// installer, with the given directory used as a read-through cache for 91// installation operations that need to retrieve new packages. 92// 93// The global cache directory for an installer must never be the same as its 94// target directory, and must not be used as one of its provider sources. 95// If these overlap then undefined behavior will result. 96func (i *Installer) SetGlobalCacheDir(cacheDir *Dir) { 97 // A little safety check to catch straightforward mistakes where the 98 // directories overlap. Better to panic early than to do 99 // possibly-distructive actions on the cache directory downstream. 100 if same, err := copydir.SameFile(i.targetDir.baseDir, cacheDir.baseDir); err == nil && same { 101 panic(fmt.Sprintf("global cache directory %s must not match the installation target directory %s", cacheDir.baseDir, i.targetDir.baseDir)) 102 } 103 i.globalCacheDir = cacheDir 104} 105 106// HasGlobalCacheDir returns true if someone has previously called 107// SetGlobalCacheDir to configure a global cache directory for this installer. 108func (i *Installer) HasGlobalCacheDir() bool { 109 return i.globalCacheDir != nil 110} 111 112// SetBuiltInProviderTypes tells the receiver to consider the type names in the 113// given slice to be valid as providers in the special special 114// terraform.io/builtin/... namespace that we use for providers that are 115// built in to Terraform and thus do not need a separate installation step. 116// 117// If a caller requests installation of a provider in that namespace, the 118// installer will treat it as a no-op if its name exists in this list, but 119// will produce an error if it does not. 120// 121// The default, if this method isn't called, is for there to be no valid 122// builtin providers. 123// 124// Do not modify the buffer under the given slice after passing it to this 125// method. 126func (i *Installer) SetBuiltInProviderTypes(types []string) { 127 i.builtInProviderTypes = types 128} 129 130// SetUnmanagedProviderTypes tells the receiver to consider the providers 131// indicated by the passed addrs.Providers as unmanaged. Terraform does not 132// need to control the lifecycle of these providers, and they are assumed to be 133// running already when Terraform is started. Because these are essentially 134// processes, not binaries, Terraform will not do any work to ensure presence 135// or versioning of these binaries. 136func (i *Installer) SetUnmanagedProviderTypes(types map[addrs.Provider]struct{}) { 137 i.unmanagedProviderTypes = types 138} 139 140// EnsureProviderVersions compares the given provider requirements with what 141// is already available in the installer's target directory and then takes 142// appropriate installation actions to ensure that suitable packages 143// are available in the target cache directory. 144// 145// The given mode modifies how the operation will treat providers that already 146// have acceptable versions available in the target cache directory. See the 147// documentation for InstallMode and the InstallMode values for more 148// information. 149// 150// The given context can be used to cancel the overall installation operation 151// (causing any operations in progress to fail with an error), and can also 152// include an InstallerEvents value for optional intermediate progress 153// notifications. 154// 155// If a given InstallerEvents subscribes to notifications about installation 156// failures then those notifications will be redundant with the ones included 157// in the final returned error value so callers should show either one or the 158// other, and not both. 159func (i *Installer) EnsureProviderVersions(ctx context.Context, locks *depsfile.Locks, reqs getproviders.Requirements, mode InstallMode) (*depsfile.Locks, error) { 160 errs := map[addrs.Provider]error{} 161 evts := installerEventsForContext(ctx) 162 163 // We'll work with a copy of the given locks, so we can modify it and 164 // return the updated locks without affecting the caller's object. 165 // We'll add or replace locks in here during our work so that the final 166 // locks file reflects what the installer has selected. 167 locks = locks.DeepCopy() 168 169 if cb := evts.PendingProviders; cb != nil { 170 cb(reqs) 171 } 172 173 // Step 1: Which providers might we need to fetch a new version of? 174 // This produces the subset of requirements we need to ask the provider 175 // source about. If we're in the normal (non-upgrade) mode then we'll 176 // just ask the source to confirm the continued existence of what 177 // was locked, or otherwise we'll find the newest version matching the 178 // configured version constraint. 179 mightNeed := map[addrs.Provider]getproviders.VersionSet{} 180 locked := map[addrs.Provider]bool{} 181 for provider, versionConstraints := range reqs { 182 if provider.IsBuiltIn() { 183 // Built in providers do not require installation but we'll still 184 // verify that the requested provider name is valid. 185 valid := false 186 for _, name := range i.builtInProviderTypes { 187 if name == provider.Type { 188 valid = true 189 break 190 } 191 } 192 var err error 193 if valid { 194 if len(versionConstraints) == 0 { 195 // Other than reporting an event for the outcome of this 196 // provider, we'll do nothing else with it: it's just 197 // automatically available for use. 198 if cb := evts.BuiltInProviderAvailable; cb != nil { 199 cb(provider) 200 } 201 } else { 202 // A built-in provider is not permitted to have an explicit 203 // version constraint, because we can only use the version 204 // that is built in to the current Terraform release. 205 err = fmt.Errorf("built-in providers do not support explicit version constraints") 206 } 207 } else { 208 err = fmt.Errorf("this Terraform release has no built-in provider named %q", provider.Type) 209 } 210 if err != nil { 211 errs[provider] = err 212 if cb := evts.BuiltInProviderFailure; cb != nil { 213 cb(provider, err) 214 } 215 } 216 continue 217 } 218 if _, ok := i.unmanagedProviderTypes[provider]; ok { 219 // unmanaged providers do not require installation 220 continue 221 } 222 acceptableVersions := versions.MeetingConstraints(versionConstraints) 223 if !mode.forceQueryAllProviders() { 224 // If we're not forcing potential changes of version then an 225 // existing selection from the lock file takes priority over 226 // the currently-configured version constraints. 227 if lock := locks.Provider(provider); lock != nil { 228 if !acceptableVersions.Has(lock.Version()) { 229 err := fmt.Errorf( 230 "locked provider %s %s does not match configured version constraint %s; must use terraform init -upgrade to allow selection of new versions", 231 provider, lock.Version(), getproviders.VersionConstraintsString(versionConstraints), 232 ) 233 errs[provider] = err 234 // This is a funny case where we're returning an error 235 // before we do any querying at all. To keep the event 236 // stream consistent without introducing an extra event 237 // type, we'll emit an artificial QueryPackagesBegin for 238 // this provider before we indicate that it failed using 239 // QueryPackagesFailure. 240 if cb := evts.QueryPackagesBegin; cb != nil { 241 cb(provider, versionConstraints, true) 242 } 243 if cb := evts.QueryPackagesFailure; cb != nil { 244 cb(provider, err) 245 } 246 continue 247 } 248 acceptableVersions = versions.Only(lock.Version()) 249 locked[provider] = true 250 } 251 } 252 mightNeed[provider] = acceptableVersions 253 } 254 255 // Step 2: Query the provider source for each of the providers we selected 256 // in the first step and select the latest available version that is 257 // in the set of acceptable versions. 258 // 259 // This produces a set of packages to install to our cache in the next step. 260 need := map[addrs.Provider]getproviders.Version{} 261NeedProvider: 262 for provider, acceptableVersions := range mightNeed { 263 if err := ctx.Err(); err != nil { 264 // If our context has been cancelled or reached a timeout then 265 // we'll abort early, because subsequent operations against 266 // that context will fail immediately anyway. 267 return nil, err 268 } 269 270 if cb := evts.QueryPackagesBegin; cb != nil { 271 cb(provider, reqs[provider], locked[provider]) 272 } 273 available, warnings, err := i.source.AvailableVersions(ctx, provider) 274 if err != nil { 275 // TODO: Consider retrying a few times for certain types of 276 // source errors that seem likely to be transient. 277 errs[provider] = err 278 if cb := evts.QueryPackagesFailure; cb != nil { 279 cb(provider, err) 280 } 281 // We will take no further actions for this provider. 282 continue 283 } 284 if len(warnings) > 0 { 285 if cb := evts.QueryPackagesWarning; cb != nil { 286 cb(provider, warnings) 287 } 288 } 289 available.Sort() // put the versions in increasing order of precedence 290 for i := len(available) - 1; i >= 0; i-- { // walk backwards to consider newer versions first 291 if acceptableVersions.Has(available[i]) { 292 need[provider] = available[i] 293 if cb := evts.QueryPackagesSuccess; cb != nil { 294 cb(provider, available[i]) 295 } 296 continue NeedProvider 297 } 298 } 299 // If we get here then the source has no packages that meet the given 300 // version constraint, which we model as a query error. 301 if locked[provider] { 302 // This situation should be a rare one: it suggests that a 303 // version was previously available but was yanked for some 304 // reason. 305 lock := locks.Provider(provider) 306 err = fmt.Errorf("the previously-selected version %s is no longer available", lock.Version()) 307 } else { 308 err = fmt.Errorf("no available releases match the given constraints %s", getproviders.VersionConstraintsString(reqs[provider])) 309 } 310 errs[provider] = err 311 if cb := evts.QueryPackagesFailure; cb != nil { 312 cb(provider, err) 313 } 314 } 315 316 // Step 3: For each provider version we've decided we need to install, 317 // install its package into our target cache (possibly via the global cache). 318 authResults := map[addrs.Provider]*getproviders.PackageAuthenticationResult{} // record auth results for all successfully fetched providers 319 targetPlatform := i.targetDir.targetPlatform // we inherit this to behave correctly in unit tests 320 for provider, version := range need { 321 if err := ctx.Err(); err != nil { 322 // If our context has been cancelled or reached a timeout then 323 // we'll abort early, because subsequent operations against 324 // that context will fail immediately anyway. 325 return nil, err 326 } 327 328 lock := locks.Provider(provider) 329 var preferredHashes []getproviders.Hash 330 if lock != nil && lock.Version() == version { // hash changes are expected if the version is also changing 331 preferredHashes = lock.PreferredHashes() 332 } 333 334 // If our target directory already has the provider version that fulfills the lock file, carry on 335 if installed := i.targetDir.ProviderVersion(provider, version); installed != nil { 336 if len(preferredHashes) > 0 { 337 if matches, _ := installed.MatchesAnyHash(preferredHashes); matches { 338 if cb := evts.ProviderAlreadyInstalled; cb != nil { 339 cb(provider, version) 340 } 341 continue 342 } 343 } 344 } 345 346 if i.globalCacheDir != nil { 347 // Step 3a: If our global cache already has this version available then 348 // we'll just link it in. 349 if cached := i.globalCacheDir.ProviderVersion(provider, version); cached != nil { 350 if cb := evts.LinkFromCacheBegin; cb != nil { 351 cb(provider, version, i.globalCacheDir.baseDir) 352 } 353 if _, err := cached.ExecutableFile(); err != nil { 354 err := fmt.Errorf("provider binary not found: %s", err) 355 errs[provider] = err 356 if cb := evts.LinkFromCacheFailure; cb != nil { 357 cb(provider, version, err) 358 } 359 continue 360 } 361 362 err := i.targetDir.LinkFromOtherCache(cached, preferredHashes) 363 if err != nil { 364 errs[provider] = err 365 if cb := evts.LinkFromCacheFailure; cb != nil { 366 cb(provider, version, err) 367 } 368 continue 369 } 370 // We'll fetch what we just linked to make sure it actually 371 // did show up there. 372 new := i.targetDir.ProviderVersion(provider, version) 373 if new == nil { 374 err := fmt.Errorf("after linking %s from provider cache at %s it is still not detected in the target directory; this is a bug in Terraform", provider, i.globalCacheDir.baseDir) 375 errs[provider] = err 376 if cb := evts.LinkFromCacheFailure; cb != nil { 377 cb(provider, version, err) 378 } 379 continue 380 } 381 382 // The LinkFromOtherCache call above should've verified that 383 // the package matches one of the hashes previously recorded, 384 // if any. We'll now augment those hashes with one freshly 385 // calculated from the package we just linked, which allows 386 // the lock file to gradually transition to recording newer hash 387 // schemes when they become available. 388 var newHashes []getproviders.Hash 389 if lock != nil && lock.Version() == version { 390 // If the version we're installing is identical to the 391 // one we previously locked then we'll keep all of the 392 // hashes we saved previously and add to it. Otherwise 393 // we'll be starting fresh, because each version has its 394 // own set of packages and thus its own hashes. 395 newHashes = append(newHashes, preferredHashes...) 396 397 // NOTE: The behavior here is unfortunate when a particular 398 // provider version was already cached on the first time 399 // the current configuration requested it, because that 400 // means we don't currently get the opportunity to fetch 401 // and verify the checksums for the new package from 402 // upstream. That's currently unavoidable because upstream 403 // checksums are in the "ziphash" format and so we can't 404 // verify them against our cache directory's unpacked 405 // packages: we'd need to go fetch the package from the 406 // origin and compare against it, which would defeat the 407 // purpose of the global cache. 408 // 409 // If we fetch from upstream on the first encounter with 410 // a particular provider then we'll end up in the other 411 // codepath below where we're able to also include the 412 // checksums from the origin registry. 413 } 414 newHash, err := cached.Hash() 415 if err != nil { 416 err := fmt.Errorf("after linking %s from provider cache at %s, failed to compute a checksum for it: %s", provider, i.globalCacheDir.baseDir, err) 417 errs[provider] = err 418 if cb := evts.LinkFromCacheFailure; cb != nil { 419 cb(provider, version, err) 420 } 421 continue 422 } 423 // The hashes slice gets deduplicated in the lock file 424 // implementation, so we don't worry about potentially 425 // creating a duplicate here. 426 newHashes = append(newHashes, newHash) 427 locks.SetProvider(provider, version, reqs[provider], newHashes) 428 429 if cb := evts.LinkFromCacheSuccess; cb != nil { 430 cb(provider, version, new.PackageDir) 431 } 432 continue // Don't need to do full install, then. 433 } 434 } 435 436 // Step 3b: Get the package metadata for the selected version from our 437 // provider source. 438 // 439 // This is the step where we might detect and report that the provider 440 // isn't available for the current platform. 441 if cb := evts.FetchPackageMeta; cb != nil { 442 cb(provider, version) 443 } 444 meta, err := i.source.PackageMeta(ctx, provider, version, targetPlatform) 445 if err != nil { 446 errs[provider] = err 447 if cb := evts.FetchPackageFailure; cb != nil { 448 cb(provider, version, err) 449 } 450 continue 451 } 452 453 // Step 3c: Retrieve the package indicated by the metadata we received, 454 // either directly into our target directory or via the global cache 455 // directory. 456 if cb := evts.FetchPackageBegin; cb != nil { 457 cb(provider, version, meta.Location) 458 } 459 var installTo, linkTo *Dir 460 if i.globalCacheDir != nil { 461 installTo = i.globalCacheDir 462 linkTo = i.targetDir 463 } else { 464 installTo = i.targetDir 465 linkTo = nil // no linking needed 466 } 467 authResult, err := installTo.InstallPackage(ctx, meta, preferredHashes) 468 if err != nil { 469 // TODO: Consider retrying for certain kinds of error that seem 470 // likely to be transient. For now, we just treat all errors equally. 471 errs[provider] = err 472 if cb := evts.FetchPackageFailure; cb != nil { 473 cb(provider, version, err) 474 } 475 continue 476 } 477 new := installTo.ProviderVersion(provider, version) 478 if new == nil { 479 err := fmt.Errorf("after installing %s it is still not detected in the target directory; this is a bug in Terraform", provider) 480 errs[provider] = err 481 if cb := evts.FetchPackageFailure; cb != nil { 482 cb(provider, version, err) 483 } 484 continue 485 } 486 if _, err := new.ExecutableFile(); err != nil { 487 err := fmt.Errorf("provider binary not found: %s", err) 488 errs[provider] = err 489 if cb := evts.FetchPackageFailure; cb != nil { 490 cb(provider, version, err) 491 } 492 continue 493 } 494 if linkTo != nil { 495 // We skip emitting the "LinkFromCache..." events here because 496 // it's simpler for the caller to treat them as mutually exclusive. 497 // We can just subsume the linking step under the "FetchPackage..." 498 // series here (and that's why we use FetchPackageFailure below). 499 // We also don't do a hash check here because we already did that 500 // as part of the installTo.InstallPackage call above. 501 err := linkTo.LinkFromOtherCache(new, nil) 502 if err != nil { 503 errs[provider] = err 504 if cb := evts.FetchPackageFailure; cb != nil { 505 cb(provider, version, err) 506 } 507 continue 508 } 509 } 510 authResults[provider] = authResult 511 512 // The InstallPackage call above should've verified that 513 // the package matches one of the hashes previously recorded, 514 // if any. We'll now augment those hashes with a new set populated 515 // with the hashes returned by the upstream source and from the 516 // package we've just installed, which allows the lock file to 517 // gradually transition to newer hash schemes when they become 518 // available. 519 // 520 // This is assuming that if a package matches both a hash we saw before 521 // _and_ a new hash then the new hash is a valid substitute for 522 // the previous hash. 523 // 524 // The hashes slice gets deduplicated in the lock file 525 // implementation, so we don't worry about potentially 526 // creating duplicates here. 527 var newHashes []getproviders.Hash 528 if lock != nil && lock.Version() == version { 529 // If the version we're installing is identical to the 530 // one we previously locked then we'll keep all of the 531 // hashes we saved previously and add to it. Otherwise 532 // we'll be starting fresh, because each version has its 533 // own set of packages and thus its own hashes. 534 newHashes = append(newHashes, preferredHashes...) 535 } 536 newHash, err := new.Hash() 537 if err != nil { 538 err := fmt.Errorf("after installing %s, failed to compute a checksum for it: %s", provider, err) 539 errs[provider] = err 540 if cb := evts.FetchPackageFailure; cb != nil { 541 cb(provider, version, err) 542 } 543 continue 544 } 545 newHashes = append(newHashes, newHash) 546 if authResult.SignedByAnyParty() { 547 // We'll trust new hashes from upstream only if they were verified 548 // as signed by a suitable key. Otherwise, we'd record only 549 // a new hash we just calculated ourselves from the bytes on disk, 550 // and so the hashes would cover only the current platform. 551 newHashes = append(newHashes, meta.AcceptableHashes()...) 552 } 553 locks.SetProvider(provider, version, reqs[provider], newHashes) 554 555 if cb := evts.FetchPackageSuccess; cb != nil { 556 cb(provider, version, new.PackageDir, authResult) 557 } 558 } 559 560 // Emit final event for fetching if any were successfully fetched 561 if cb := evts.ProvidersFetched; cb != nil && len(authResults) > 0 { 562 cb(authResults) 563 } 564 565 if len(errs) > 0 { 566 return locks, InstallerError{ 567 ProviderErrors: errs, 568 } 569 } 570 return locks, nil 571} 572 573// InstallMode customizes the details of how an install operation treats 574// providers that have versions already cached in the target directory. 575type InstallMode rune 576 577const ( 578 // InstallNewProvidersOnly is an InstallMode that causes the installer 579 // to accept any existing version of a requested provider that is already 580 // cached as long as it's in the given version sets, without checking 581 // whether new versions are available that are also in the given version 582 // sets. 583 InstallNewProvidersOnly InstallMode = 'N' 584 585 // InstallUpgrades is an InstallMode that causes the installer to check 586 // all requested providers to see if new versions are available that 587 // are also in the given version sets, even if a suitable version of 588 // a given provider is already available. 589 InstallUpgrades InstallMode = 'U' 590) 591 592func (m InstallMode) forceQueryAllProviders() bool { 593 return m == InstallUpgrades 594} 595 596// InstallerError is an error type that may be returned (but is not guaranteed) 597// from Installer.EnsureProviderVersions to indicate potentially several 598// separate failed installation outcomes for different providers included in 599// the overall request. 600type InstallerError struct { 601 ProviderErrors map[addrs.Provider]error 602} 603 604func (err InstallerError) Error() string { 605 addrs := make([]addrs.Provider, 0, len(err.ProviderErrors)) 606 for addr := range err.ProviderErrors { 607 addrs = append(addrs, addr) 608 } 609 sort.Slice(addrs, func(i, j int) bool { 610 return addrs[i].LessThan(addrs[j]) 611 }) 612 var b strings.Builder 613 b.WriteString("some providers could not be installed:\n") 614 for _, addr := range addrs { 615 providerErr := err.ProviderErrors[addr] 616 fmt.Fprintf(&b, "- %s: %s\n", addr, providerErr) 617 } 618 return strings.TrimSpace(b.String()) 619} 620