1package moduletest 2 3import ( 4 "fmt" 5 "log" 6 "sync" 7 8 "github.com/zclconf/go-cty/cty" 9 "github.com/zclconf/go-cty/cty/gocty" 10 ctyjson "github.com/zclconf/go-cty/cty/json" 11 12 "github.com/hashicorp/hcl/v2/hclsyntax" 13 "github.com/hashicorp/terraform/internal/configs/configschema" 14 "github.com/hashicorp/terraform/internal/providers" 15 "github.com/hashicorp/terraform/internal/repl" 16 "github.com/hashicorp/terraform/internal/tfdiags" 17) 18 19// Provider is an implementation of providers.Interface which we're 20// using as a likely-only-temporary vehicle for research on an opinionated 21// module testing workflow in Terraform. 22// 23// We expose this to configuration as "terraform.io/builtin/test", but 24// any attempt to configure it will emit a warning that it is experimental 25// and likely to change or be removed entirely in future Terraform CLI 26// releases. 27// 28// The testing provider exists to gather up test results during a Terraform 29// apply operation. Its "test_results" managed resource type doesn't have any 30// user-visible effect on its own, but when used in conjunction with the 31// "terraform test" experimental command it is the intermediary that holds 32// the test results while the test runs, so that the test command can then 33// report them. 34// 35// For correct behavior of the assertion tracking, the "terraform test" 36// command must be sure to use the same instance of Provider for both the 37// plan and apply steps, so that the assertions that were planned can still 38// be tracked during apply. For other commands that don't explicitly support 39// test assertions, the provider will still succeed but the assertions data 40// may not be complete if the apply step fails. 41type Provider struct { 42 // components tracks all of the "component" names that have been 43 // used in test assertions resources so far. Each resource must have 44 // a unique component name. 45 components map[string]*Component 46 47 // Must lock mutex in order to interact with the components map, because 48 // test assertions can potentially run concurrently. 49 mutex sync.RWMutex 50} 51 52var _ providers.Interface = (*Provider)(nil) 53 54// NewProvider returns a new instance of the test provider. 55func NewProvider() *Provider { 56 return &Provider{ 57 components: make(map[string]*Component), 58 } 59} 60 61// TestResults returns the current record of test results tracked inside the 62// provider. 63// 64// The result is a direct reference to the internal state of the provider, 65// so the caller mustn't modify it nor store it across calls to provider 66// operations. 67func (p *Provider) TestResults() map[string]*Component { 68 return p.components 69} 70 71// Reset returns the recieving provider back to its original state, with no 72// recorded test results. 73// 74// It additionally detaches the instance from any data structure previously 75// returned by method TestResults, freeing the caller from the constraints 76// in its documentation about mutability and storage. 77// 78// For convenience in the presumed common case of resetting as part of 79// capturing the results for storage, this method also returns the result 80// that method TestResults would've returned if called prior to the call 81// to Reset. 82func (p *Provider) Reset() map[string]*Component { 83 p.mutex.Lock() 84 log.Print("[TRACE] moduletest.Provider: Reset") 85 ret := p.components 86 p.components = make(map[string]*Component) 87 p.mutex.Unlock() 88 return ret 89} 90 91// GetProviderSchema returns the complete schema for the provider. 92func (p *Provider) GetProviderSchema() providers.GetProviderSchemaResponse { 93 return providers.GetProviderSchemaResponse{ 94 ResourceTypes: map[string]providers.Schema{ 95 "test_assertions": testAssertionsSchema, 96 }, 97 } 98} 99 100// ValidateProviderConfig validates the provider configuration. 101func (p *Provider) ValidateProviderConfig(req providers.ValidateProviderConfigRequest) providers.ValidateProviderConfigResponse { 102 // This provider has no configurable settings, so nothing to validate. 103 var res providers.ValidateProviderConfigResponse 104 return res 105} 106 107// ConfigureProvider configures and initializes the provider. 108func (p *Provider) ConfigureProvider(providers.ConfigureProviderRequest) providers.ConfigureProviderResponse { 109 // This provider has no configurable settings, but we use the configure 110 // request as an opportunity to generate a warning about it being 111 // experimental. 112 var res providers.ConfigureProviderResponse 113 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 114 tfdiags.Warning, 115 "The test provider is experimental", 116 "The Terraform team is using the test provider (terraform.io/builtin/test) as part of ongoing research about declarative testing of Terraform modules.\n\nThe availability and behavior of this provider is expected to change significantly even in patch releases, so we recommend using this provider only in test configurations and constraining your test configurations to an exact Terraform version.", 117 nil, 118 )) 119 return res 120} 121 122// ValidateResourceConfig is used to validate configuration values for a resource. 123func (p *Provider) ValidateResourceConfig(req providers.ValidateResourceConfigRequest) providers.ValidateResourceConfigResponse { 124 log.Print("[TRACE] moduletest.Provider: ValidateResourceConfig") 125 126 var res providers.ValidateResourceConfigResponse 127 if req.TypeName != "test_assertions" { // we only have one resource type 128 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 129 return res 130 } 131 132 config := req.Config 133 if !config.GetAttr("component").IsKnown() { 134 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 135 tfdiags.Error, 136 "Invalid component expression", 137 "The component name must be a static value given in the configuration, and may not be derived from a resource type attribute that will only be known during the apply step.", 138 cty.GetAttrPath("component"), 139 )) 140 } 141 if !hclsyntax.ValidIdentifier(config.GetAttr("component").AsString()) { 142 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 143 tfdiags.Error, 144 "Invalid component name", 145 "The component name must be a valid identifier, starting with a letter followed by zero or more letters, digits, and underscores.", 146 cty.GetAttrPath("component"), 147 )) 148 } 149 for it := config.GetAttr("equal").ElementIterator(); it.Next(); { 150 k, obj := it.Element() 151 if !hclsyntax.ValidIdentifier(k.AsString()) { 152 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 153 tfdiags.Error, 154 "Invalid assertion name", 155 "An assertion name must be a valid identifier, starting with a letter followed by zero or more letters, digits, and underscores.", 156 cty.GetAttrPath("equal").Index(k), 157 )) 158 } 159 if !obj.GetAttr("description").IsKnown() { 160 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 161 tfdiags.Error, 162 "Invalid description expression", 163 "The description must be a static value given in the configuration, and may not be derived from a resource type attribute that will only be known during the apply step.", 164 cty.GetAttrPath("equal").Index(k).GetAttr("description"), 165 )) 166 } 167 } 168 for it := config.GetAttr("check").ElementIterator(); it.Next(); { 169 k, obj := it.Element() 170 if !hclsyntax.ValidIdentifier(k.AsString()) { 171 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 172 tfdiags.Error, 173 "Invalid assertion name", 174 "An assertion name must be a valid identifier, starting with a letter followed by zero or more letters, digits, and underscores.", 175 cty.GetAttrPath("check").Index(k), 176 )) 177 } 178 if !obj.GetAttr("description").IsKnown() { 179 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 180 tfdiags.Error, 181 "Invalid description expression", 182 "The description must be a static value given in the configuration, and may not be derived from a resource type attribute that will only be known during the apply step.", 183 cty.GetAttrPath("equal").Index(k).GetAttr("description"), 184 )) 185 } 186 } 187 188 return res 189} 190 191// ReadResource refreshes a resource and returns its current state. 192func (p *Provider) ReadResource(req providers.ReadResourceRequest) providers.ReadResourceResponse { 193 log.Print("[TRACE] moduletest.Provider: ReadResource") 194 195 var res providers.ReadResourceResponse 196 if req.TypeName != "test_assertions" { // we only have one resource type 197 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 198 return res 199 } 200 // Test assertions are not a real remote object, so there isn't actually 201 // anything to refresh here. 202 res.NewState = req.PriorState 203 return res 204} 205 206// UpgradeResourceState is called to allow the provider to adapt the raw value 207// stored in the state in case the schema has changed since it was originally 208// written. 209func (p *Provider) UpgradeResourceState(req providers.UpgradeResourceStateRequest) providers.UpgradeResourceStateResponse { 210 log.Print("[TRACE] moduletest.Provider: UpgradeResourceState") 211 212 var res providers.UpgradeResourceStateResponse 213 if req.TypeName != "test_assertions" { // we only have one resource type 214 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 215 return res 216 } 217 218 // We assume here that there can never be a flatmap version of this 219 // resource type's data, because this provider was never included in a 220 // version of Terraform that used flatmap and this provider's schema 221 // contains attributes that are not flatmap-compatible anyway. 222 if len(req.RawStateFlatmap) != 0 { 223 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("can't upgrade a flatmap state for %q", req.TypeName)) 224 return res 225 } 226 if req.Version != 0 { 227 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("the state for this %s was created by a newer version of the provider", req.TypeName)) 228 return res 229 } 230 231 v, err := ctyjson.Unmarshal(req.RawStateJSON, testAssertionsSchema.Block.ImpliedType()) 232 if err != nil { 233 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("failed to decode state for %s: %s", req.TypeName, err)) 234 return res 235 } 236 237 res.UpgradedState = v 238 return res 239} 240 241// PlanResourceChange takes the current state and proposed state of a 242// resource, and returns the planned final state. 243func (p *Provider) PlanResourceChange(req providers.PlanResourceChangeRequest) providers.PlanResourceChangeResponse { 244 log.Print("[TRACE] moduletest.Provider: PlanResourceChange") 245 246 var res providers.PlanResourceChangeResponse 247 if req.TypeName != "test_assertions" { // we only have one resource type 248 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 249 return res 250 } 251 252 // During planning, our job is to gather up all of the planned test 253 // assertions marked as pending, which will then allow us to include 254 // all of them in test results even if there's a failure during apply 255 // that prevents the full completion of the graph walk. 256 // 257 // In a sense our plan phase is similar to the compile step for a 258 // test program written in another language. Planning itself can fail, 259 // which means we won't be able to form a complete test plan at all, 260 // but if we succeed in planning then subsequent problems can be treated 261 // as test failures at "runtime", while still keeping a full manifest 262 // of all of the tests that ought to have run if the apply had run to 263 // completion. 264 265 proposed := req.ProposedNewState 266 res.PlannedState = proposed 267 componentName := proposed.GetAttr("component").AsString() // proven known during validate 268 p.mutex.Lock() 269 defer p.mutex.Unlock() 270 // NOTE: Ideally we'd do something here to verify if two assertions 271 // resources in the configuration attempt to declare the same component, 272 // but we can't actually do that because Terraform calls PlanResourceChange 273 // during both plan and apply, and so the second one would always fail. 274 // Since this is just providing a temporary pseudo-syntax for writing tests 275 // anyway, we'll live with this for now and aim to solve it with a future 276 // iteration of testing that's better integrated into the Terraform 277 // language. 278 /* 279 if _, exists := p.components[componentName]; exists { 280 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 281 tfdiags.Error, 282 "Duplicate test component", 283 fmt.Sprintf("Another test_assertions resource already declared assertions for the component name %q.", componentName), 284 cty.GetAttrPath("component"), 285 )) 286 return res 287 } 288 */ 289 290 component := Component{ 291 Assertions: make(map[string]*Assertion), 292 } 293 294 for it := proposed.GetAttr("equal").ElementIterator(); it.Next(); { 295 k, obj := it.Element() 296 name := k.AsString() 297 if _, exists := component.Assertions[name]; exists { 298 // We can't actually get here in practice because so far we've 299 // only been pulling keys from one map, and so any duplicates 300 // would've been caught during config decoding, but this is here 301 // just to make these two blocks symmetrical to avoid mishaps in 302 // future refactoring/reorganization. 303 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 304 tfdiags.Error, 305 "Duplicate test assertion", 306 fmt.Sprintf("Another assertion block in this resource already declared an assertion named %q.", name), 307 cty.GetAttrPath("equal").Index(k), 308 )) 309 continue 310 } 311 312 var desc string 313 descVal := obj.GetAttr("description") 314 if descVal.IsNull() { 315 descVal = cty.StringVal("") 316 } 317 err := gocty.FromCtyValue(descVal, &desc) 318 if err != nil { 319 // We shouldn't get here because we've already validated everything 320 // that would make FromCtyValue fail above and during validate. 321 res.Diagnostics = res.Diagnostics.Append(err) 322 } 323 324 component.Assertions[name] = &Assertion{ 325 Outcome: Pending, 326 Description: desc, 327 } 328 } 329 330 for it := proposed.GetAttr("check").ElementIterator(); it.Next(); { 331 k, obj := it.Element() 332 name := k.AsString() 333 if _, exists := component.Assertions[name]; exists { 334 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 335 tfdiags.Error, 336 "Duplicate test assertion", 337 fmt.Sprintf("Another assertion block in this resource already declared an assertion named %q.", name), 338 cty.GetAttrPath("check").Index(k), 339 )) 340 continue 341 } 342 343 var desc string 344 descVal := obj.GetAttr("description") 345 if descVal.IsNull() { 346 descVal = cty.StringVal("") 347 } 348 err := gocty.FromCtyValue(descVal, &desc) 349 if err != nil { 350 // We shouldn't get here because we've already validated everything 351 // that would make FromCtyValue fail above and during validate. 352 res.Diagnostics = res.Diagnostics.Append(err) 353 } 354 355 component.Assertions[name] = &Assertion{ 356 Outcome: Pending, 357 Description: desc, 358 } 359 } 360 361 p.components[componentName] = &component 362 return res 363} 364 365// ApplyResourceChange takes the planned state for a resource, which may 366// yet contain unknown computed values, and applies the changes returning 367// the final state. 368func (p *Provider) ApplyResourceChange(req providers.ApplyResourceChangeRequest) providers.ApplyResourceChangeResponse { 369 log.Print("[TRACE] moduletest.Provider: ApplyResourceChange") 370 371 var res providers.ApplyResourceChangeResponse 372 if req.TypeName != "test_assertions" { // we only have one resource type 373 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported resource type %s", req.TypeName)) 374 return res 375 } 376 377 // During apply we actually check the assertions and record the results. 378 // An assertion failure isn't reflected as an error from the apply call 379 // because if possible we'd like to continue exercising other objects 380 // downstream in case that allows us to gather more information to report. 381 // (If something downstream returns an error then that could prevent us 382 // from completing other assertions, though.) 383 384 planned := req.PlannedState 385 res.NewState = planned 386 if res.NewState.IsNull() { 387 // If we're destroying then we'll just quickly return success to 388 // allow the test process to clean up after itself. 389 return res 390 } 391 componentName := planned.GetAttr("component").AsString() // proven known during validate 392 393 p.mutex.Lock() 394 defer p.mutex.Unlock() 395 component := p.components[componentName] 396 if component == nil { 397 // We might get here when using this provider outside of the 398 // "terraform test" command, where there won't be any mechanism to 399 // preserve the test provider instance between the plan and apply 400 // phases. In that case, we assume that nobody will come looking to 401 // collect the results anyway, and so we can just silently skip 402 // checking. 403 return res 404 } 405 406 for it := planned.GetAttr("equal").ElementIterator(); it.Next(); { 407 k, obj := it.Element() 408 name := k.AsString() 409 var desc string 410 if plan, exists := component.Assertions[name]; exists { 411 desc = plan.Description 412 } 413 assert := &Assertion{ 414 Outcome: Pending, 415 Description: desc, 416 } 417 418 gotVal := obj.GetAttr("got") 419 wantVal := obj.GetAttr("want") 420 switch { 421 case wantVal.RawEquals(gotVal): 422 assert.Outcome = Passed 423 gotStr := repl.FormatValue(gotVal, 4) 424 assert.Message = fmt.Sprintf("correct value\n got: %s\n", gotStr) 425 default: 426 assert.Outcome = Failed 427 gotStr := repl.FormatValue(gotVal, 4) 428 wantStr := repl.FormatValue(wantVal, 4) 429 assert.Message = fmt.Sprintf("wrong value\n got: %s\n want: %s\n", gotStr, wantStr) 430 } 431 432 component.Assertions[name] = assert 433 } 434 435 for it := planned.GetAttr("check").ElementIterator(); it.Next(); { 436 k, obj := it.Element() 437 name := k.AsString() 438 var desc string 439 if plan, exists := component.Assertions[name]; exists { 440 desc = plan.Description 441 } 442 assert := &Assertion{ 443 Outcome: Pending, 444 Description: desc, 445 } 446 447 condVal := obj.GetAttr("condition") 448 switch { 449 case condVal.IsNull(): 450 res.Diagnostics = res.Diagnostics.Append(tfdiags.AttributeValue( 451 tfdiags.Error, 452 "Invalid check condition", 453 "The condition value must be a boolean expression, not null.", 454 cty.GetAttrPath("check").Index(k).GetAttr("condition"), 455 )) 456 continue 457 case condVal.True(): 458 assert.Outcome = Passed 459 assert.Message = "condition passed" 460 default: 461 assert.Outcome = Failed 462 // For "check" we can't really return a decent error message 463 // because we've lost all of the context by the time we get here. 464 // "equal" will be better for most tests for that reason, and also 465 // this is one reason why in the long run it would be better for 466 // test assertions to be a first-class language feature rather than 467 // just a provider-based concept. 468 assert.Message = "condition failed" 469 } 470 471 component.Assertions[name] = assert 472 } 473 474 return res 475} 476 477// ImportResourceState requests that the given resource be imported. 478func (p *Provider) ImportResourceState(req providers.ImportResourceStateRequest) providers.ImportResourceStateResponse { 479 var res providers.ImportResourceStateResponse 480 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("%s is not importable", req.TypeName)) 481 return res 482} 483 484// ValidateDataResourceConfig is used to to validate the resource configuration values. 485func (p *Provider) ValidateDataResourceConfig(req providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse { 486 // This provider has no data resouce types at all. 487 var res providers.ValidateDataResourceConfigResponse 488 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported data source %s", req.TypeName)) 489 return res 490} 491 492// ReadDataSource returns the data source's current state. 493func (p *Provider) ReadDataSource(req providers.ReadDataSourceRequest) providers.ReadDataSourceResponse { 494 // This provider has no data resouce types at all. 495 var res providers.ReadDataSourceResponse 496 res.Diagnostics = res.Diagnostics.Append(fmt.Errorf("unsupported data source %s", req.TypeName)) 497 return res 498} 499 500// Stop is called when the provider should halt any in-flight actions. 501func (p *Provider) Stop() error { 502 // This provider doesn't do anything that can be cancelled. 503 return nil 504} 505 506// Close is a noop for this provider, since it's run in-process. 507func (p *Provider) Close() error { 508 return nil 509} 510 511var testAssertionsSchema = providers.Schema{ 512 Block: &configschema.Block{ 513 Attributes: map[string]*configschema.Attribute{ 514 "component": { 515 Type: cty.String, 516 Description: "The name of the component being tested. This is just for namespacing assertions in a result report.", 517 DescriptionKind: configschema.StringPlain, 518 Required: true, 519 }, 520 }, 521 BlockTypes: map[string]*configschema.NestedBlock{ 522 "equal": { 523 Nesting: configschema.NestingMap, 524 Block: configschema.Block{ 525 Attributes: map[string]*configschema.Attribute{ 526 "description": { 527 Type: cty.String, 528 Description: "An optional human-readable description of what's being tested by this assertion.", 529 DescriptionKind: configschema.StringPlain, 530 Required: true, 531 }, 532 "got": { 533 Type: cty.DynamicPseudoType, 534 Description: "The actual result value generated by the relevant component.", 535 DescriptionKind: configschema.StringPlain, 536 Required: true, 537 }, 538 "want": { 539 Type: cty.DynamicPseudoType, 540 Description: "The value that the component is expected to have generated.", 541 DescriptionKind: configschema.StringPlain, 542 Required: true, 543 }, 544 }, 545 }, 546 }, 547 "check": { 548 Nesting: configschema.NestingMap, 549 Block: configschema.Block{ 550 Attributes: map[string]*configschema.Attribute{ 551 "description": { 552 Type: cty.String, 553 Description: "An optional (but strongly recommended) human-readable description of what's being tested by this assertion.", 554 DescriptionKind: configschema.StringPlain, 555 Required: true, 556 }, 557 "condition": { 558 Type: cty.Bool, 559 Description: "An expression that must be true in order for the test to pass.", 560 DescriptionKind: configschema.StringPlain, 561 Required: true, 562 }, 563 }, 564 }, 565 }, 566 }, 567 }, 568} 569