1package main 2 3import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io/ioutil" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "sort" 12 "strings" 13 14 "github.com/zclconf/go-cty-debug/ctydebug" 15 "github.com/zclconf/go-cty/cty" 16 "github.com/zclconf/go-cty/cty/convert" 17 ctyjson "github.com/zclconf/go-cty/cty/json" 18 19 "github.com/hashicorp/hcl/v2" 20 "github.com/hashicorp/hcl/v2/ext/typeexpr" 21 "github.com/hashicorp/hcl/v2/hclparse" 22) 23 24type Runner struct { 25 parser *hclparse.Parser 26 hcldecPath string 27 baseDir string 28 logBegin LogBeginCallback 29 logProblems LogProblemsCallback 30} 31 32func (r *Runner) Run() hcl.Diagnostics { 33 return r.runDir(r.baseDir) 34} 35 36func (r *Runner) runDir(dir string) hcl.Diagnostics { 37 var diags hcl.Diagnostics 38 39 infos, err := ioutil.ReadDir(dir) 40 if err != nil { 41 diags = append(diags, &hcl.Diagnostic{ 42 Severity: hcl.DiagError, 43 Summary: "Failed to read test directory", 44 Detail: fmt.Sprintf("The directory %q could not be opened: %s.", dir, err), 45 }) 46 return diags 47 } 48 49 var tests []string 50 var subDirs []string 51 for _, info := range infos { 52 name := info.Name() 53 if strings.HasPrefix(name, ".") { 54 continue 55 } 56 57 if info.IsDir() { 58 subDirs = append(subDirs, name) 59 } 60 if strings.HasSuffix(name, ".t") { 61 tests = append(tests, name) 62 } 63 } 64 sort.Strings(tests) 65 sort.Strings(subDirs) 66 67 for _, filename := range tests { 68 filename = filepath.Join(dir, filename) 69 testDiags := r.runTest(filename) 70 diags = append(diags, testDiags...) 71 } 72 73 for _, dirName := range subDirs { 74 dir := filepath.Join(dir, dirName) 75 dirDiags := r.runDir(dir) 76 diags = append(diags, dirDiags...) 77 } 78 79 return diags 80} 81 82func (r *Runner) runTest(filename string) hcl.Diagnostics { 83 prettyName := r.prettyTestName(filename) 84 tf, diags := r.LoadTestFile(filename) 85 if diags.HasErrors() { 86 // We'll still log, so it's clearer which test the diagnostics belong to. 87 if r.logBegin != nil { 88 r.logBegin(prettyName, nil) 89 } 90 if r.logProblems != nil { 91 r.logProblems(prettyName, nil, diags) 92 return nil // don't duplicate the diagnostics we already reported 93 } 94 return diags 95 } 96 97 if r.logBegin != nil { 98 r.logBegin(prettyName, tf) 99 } 100 101 basePath := filename[:len(filename)-2] 102 specFilename := basePath + ".hcldec" 103 nativeFilename := basePath + ".hcl" 104 jsonFilename := basePath + ".hcl.json" 105 106 // We'll add the source code of the spec file to our own parser, even 107 // though it'll actually be parsed by the hcldec child process, since that 108 // way we can produce nice diagnostic messages if hcldec fails to process 109 // the spec file. 110 src, err := ioutil.ReadFile(specFilename) 111 if err == nil { 112 r.parser.AddFile(specFilename, &hcl.File{ 113 Bytes: src, 114 }) 115 } 116 117 if _, err := os.Stat(specFilename); err != nil { 118 diags = append(diags, &hcl.Diagnostic{ 119 Severity: hcl.DiagError, 120 Summary: "Missing .hcldec file", 121 Detail: fmt.Sprintf("No specification file for test %s: %s.", prettyName, err), 122 }) 123 return diags 124 } 125 126 if _, err := os.Stat(nativeFilename); err == nil { 127 moreDiags := r.runTestInput(specFilename, nativeFilename, tf) 128 diags = append(diags, moreDiags...) 129 } 130 131 if _, err := os.Stat(jsonFilename); err == nil { 132 moreDiags := r.runTestInput(specFilename, jsonFilename, tf) 133 diags = append(diags, moreDiags...) 134 } 135 136 if r.logProblems != nil { 137 r.logProblems(prettyName, nil, diags) 138 return nil // don't duplicate the diagnostics we already reported 139 } 140 141 return diags 142} 143 144func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile) hcl.Diagnostics { 145 // We'll add the source code of the input file to our own parser, even 146 // though it'll actually be parsed by the hcldec child process, since that 147 // way we can produce nice diagnostic messages if hcldec fails to process 148 // the input file. 149 src, err := ioutil.ReadFile(inputFilename) 150 if err == nil { 151 r.parser.AddFile(inputFilename, &hcl.File{ 152 Bytes: src, 153 }) 154 } 155 156 var diags hcl.Diagnostics 157 158 if tf.ChecksTraversals { 159 gotTraversals, moreDiags := r.hcldecVariables(specFilename, inputFilename) 160 diags = append(diags, moreDiags...) 161 if !moreDiags.HasErrors() { 162 expected := tf.ExpectedTraversals 163 for _, got := range gotTraversals { 164 e := findTraversalSpec(got, expected) 165 rng := got.SourceRange() 166 if e == nil { 167 diags = append(diags, &hcl.Diagnostic{ 168 Severity: hcl.DiagError, 169 Summary: "Unexpected traversal", 170 Detail: "Detected traversal that is not indicated as expected in the test file.", 171 Subject: &rng, 172 }) 173 } else { 174 moreDiags := checkTraversalsMatch(got, inputFilename, e) 175 diags = append(diags, moreDiags...) 176 } 177 } 178 179 // Look for any traversals that didn't show up at all. 180 for _, e := range expected { 181 if t := findTraversalForSpec(e, gotTraversals); t == nil { 182 diags = append(diags, &hcl.Diagnostic{ 183 Severity: hcl.DiagError, 184 Summary: "Missing expected traversal", 185 Detail: "This expected traversal was not detected.", 186 Subject: e.Traversal.SourceRange().Ptr(), 187 }) 188 } 189 } 190 } 191 192 } 193 194 val, transformDiags := r.hcldecTransform(specFilename, inputFilename) 195 if len(tf.ExpectedDiags) == 0 { 196 diags = append(diags, transformDiags...) 197 if transformDiags.HasErrors() { 198 // If hcldec failed then there's no point in continuing. 199 return diags 200 } 201 202 if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 { 203 diags = append(diags, &hcl.Diagnostic{ 204 Severity: hcl.DiagError, 205 Summary: "Incorrect result type", 206 Detail: fmt.Sprintf( 207 "Input file %s produced %s, but was expecting %s.", 208 inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType), 209 ), 210 }) 211 } 212 213 if tf.Result != cty.NilVal { 214 cmpVal, err := convert.Convert(tf.Result, tf.ResultType) 215 if err != nil { 216 diags = append(diags, &hcl.Diagnostic{ 217 Severity: hcl.DiagError, 218 Summary: "Incorrect type for result value", 219 Detail: fmt.Sprintf( 220 "Result does not conform to the given result type: %s.", err, 221 ), 222 Subject: &tf.ResultRange, 223 }) 224 } else { 225 if !val.RawEquals(cmpVal) { 226 diags = append(diags, &hcl.Diagnostic{ 227 Severity: hcl.DiagError, 228 Summary: "Incorrect result value", 229 Detail: fmt.Sprintf( 230 "Input file %s produced %#v, but was expecting %#v.\n\n%s", 231 inputFilename, val, tf.Result, 232 ctydebug.DiffValues(tf.Result, val), 233 ), 234 }) 235 } 236 } 237 } 238 } else { 239 // We're expecting diagnostics, and so we'll need to correlate the 240 // severities and source ranges of our actual diagnostics against 241 // what we were expecting. 242 type DiagnosticEntry struct { 243 Severity hcl.DiagnosticSeverity 244 Range hcl.Range 245 } 246 got := make(map[DiagnosticEntry]*hcl.Diagnostic) 247 want := make(map[DiagnosticEntry]hcl.Range) 248 for _, diag := range transformDiags { 249 if diag.Subject == nil { 250 // Sourceless diagnostics can never be expected, so we'll just 251 // pass these through as-is and assume they are hcldec 252 // operational errors. 253 diags = append(diags, diag) 254 continue 255 } 256 if diag.Subject.Filename != inputFilename { 257 // If the problem is for something other than the input file 258 // then it can't be expected. 259 diags = append(diags, diag) 260 continue 261 } 262 entry := DiagnosticEntry{ 263 Severity: diag.Severity, 264 Range: *diag.Subject, 265 } 266 got[entry] = diag 267 } 268 for _, e := range tf.ExpectedDiags { 269 e.Range.Filename = inputFilename // assumed here, since we don't allow any other filename to be expected 270 entry := DiagnosticEntry{ 271 Severity: e.Severity, 272 Range: e.Range, 273 } 274 want[entry] = e.DeclRange 275 } 276 277 for gotEntry, diag := range got { 278 if _, wanted := want[gotEntry]; !wanted { 279 // Pass through the diagnostic itself so the user can see what happened 280 diags = append(diags, diag) 281 diags = append(diags, &hcl.Diagnostic{ 282 Severity: hcl.DiagError, 283 Summary: "Unexpected diagnostic", 284 Detail: fmt.Sprintf( 285 "No %s diagnostic was expected %s. The unexpected diagnostic was shown above.", 286 severityString(gotEntry.Severity), rangeString(gotEntry.Range), 287 ), 288 Subject: gotEntry.Range.Ptr(), 289 }) 290 } 291 } 292 293 for wantEntry, declRange := range want { 294 if _, gotted := got[wantEntry]; !gotted { 295 diags = append(diags, &hcl.Diagnostic{ 296 Severity: hcl.DiagError, 297 Summary: "Missing expected diagnostic", 298 Detail: fmt.Sprintf( 299 "No %s diagnostic was generated %s.", 300 severityString(wantEntry.Severity), rangeString(wantEntry.Range), 301 ), 302 Subject: declRange.Ptr(), 303 }) 304 } 305 } 306 } 307 308 return diags 309} 310 311func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Diagnostics) { 312 var diags hcl.Diagnostics 313 var outBuffer bytes.Buffer 314 var errBuffer bytes.Buffer 315 316 cmd := &exec.Cmd{ 317 Path: r.hcldecPath, 318 Args: []string{ 319 r.hcldecPath, 320 "--spec=" + specFile, 321 "--diags=json", 322 "--with-type", 323 "--keep-nulls", 324 inputFile, 325 }, 326 Stdout: &outBuffer, 327 Stderr: &errBuffer, 328 } 329 err := cmd.Run() 330 if err != nil { 331 if _, isExit := err.(*exec.ExitError); !isExit { 332 diags = append(diags, &hcl.Diagnostic{ 333 Severity: hcl.DiagError, 334 Summary: "Failed to run hcldec", 335 Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err), 336 }) 337 return cty.DynamicVal, diags 338 } 339 340 // If we exited unsuccessfully then we'll expect diagnostics on stderr 341 moreDiags := decodeJSONDiagnostics(errBuffer.Bytes()) 342 diags = append(diags, moreDiags...) 343 return cty.DynamicVal, diags 344 } else { 345 // Otherwise, we expect a JSON result value on stdout. Since we used 346 // --with-type above, we can decode as DynamicPseudoType to recover 347 // exactly the type that was saved, without the usual JSON lossiness. 348 val, err := ctyjson.Unmarshal(outBuffer.Bytes(), cty.DynamicPseudoType) 349 if err != nil { 350 diags = append(diags, &hcl.Diagnostic{ 351 Severity: hcl.DiagError, 352 Summary: "Failed to parse hcldec result", 353 Detail: fmt.Sprintf("Sub-program hcldec produced an invalid result: %s.", err), 354 }) 355 return cty.DynamicVal, diags 356 } 357 return val, diags 358 } 359} 360 361func (r *Runner) hcldecVariables(specFile, inputFile string) ([]hcl.Traversal, hcl.Diagnostics) { 362 var diags hcl.Diagnostics 363 var outBuffer bytes.Buffer 364 var errBuffer bytes.Buffer 365 366 cmd := &exec.Cmd{ 367 Path: r.hcldecPath, 368 Args: []string{ 369 r.hcldecPath, 370 "--spec=" + specFile, 371 "--diags=json", 372 "--var-refs", 373 inputFile, 374 }, 375 Stdout: &outBuffer, 376 Stderr: &errBuffer, 377 } 378 err := cmd.Run() 379 if err != nil { 380 if _, isExit := err.(*exec.ExitError); !isExit { 381 diags = append(diags, &hcl.Diagnostic{ 382 Severity: hcl.DiagError, 383 Summary: "Failed to run hcldec", 384 Detail: fmt.Sprintf("Sub-program hcldec (evaluating input) failed to start: %s.", err), 385 }) 386 return nil, diags 387 } 388 389 // If we exited unsuccessfully then we'll expect diagnostics on stderr 390 moreDiags := decodeJSONDiagnostics(errBuffer.Bytes()) 391 diags = append(diags, moreDiags...) 392 return nil, diags 393 } else { 394 // Otherwise, we expect a JSON description of the traversals on stdout. 395 type PosJSON struct { 396 Line int `json:"line"` 397 Column int `json:"column"` 398 Byte int `json:"byte"` 399 } 400 type RangeJSON struct { 401 Filename string `json:"filename"` 402 Start PosJSON `json:"start"` 403 End PosJSON `json:"end"` 404 } 405 type StepJSON struct { 406 Kind string `json:"kind"` 407 Name string `json:"name,omitempty"` 408 Key json.RawMessage `json:"key,omitempty"` 409 Range RangeJSON `json:"range"` 410 } 411 type TraversalJSON struct { 412 Steps []StepJSON `json:"steps"` 413 } 414 415 var raw []TraversalJSON 416 err := json.Unmarshal(outBuffer.Bytes(), &raw) 417 if err != nil { 418 diags = append(diags, &hcl.Diagnostic{ 419 Severity: hcl.DiagError, 420 Summary: "Failed to parse hcldec result", 421 Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: %s.", err), 422 }) 423 return nil, diags 424 } 425 426 var ret []hcl.Traversal 427 if len(raw) == 0 { 428 return ret, diags 429 } 430 431 ret = make([]hcl.Traversal, 0, len(raw)) 432 for _, rawT := range raw { 433 traversal := make(hcl.Traversal, 0, len(rawT.Steps)) 434 for _, rawS := range rawT.Steps { 435 rng := hcl.Range{ 436 Filename: rawS.Range.Filename, 437 Start: hcl.Pos{ 438 Line: rawS.Range.Start.Line, 439 Column: rawS.Range.Start.Column, 440 Byte: rawS.Range.Start.Byte, 441 }, 442 End: hcl.Pos{ 443 Line: rawS.Range.End.Line, 444 Column: rawS.Range.End.Column, 445 Byte: rawS.Range.End.Byte, 446 }, 447 } 448 449 switch rawS.Kind { 450 451 case "root": 452 traversal = append(traversal, hcl.TraverseRoot{ 453 Name: rawS.Name, 454 SrcRange: rng, 455 }) 456 457 case "attr": 458 traversal = append(traversal, hcl.TraverseAttr{ 459 Name: rawS.Name, 460 SrcRange: rng, 461 }) 462 463 case "index": 464 ty, err := ctyjson.ImpliedType([]byte(rawS.Key)) 465 if err != nil { 466 diags = append(diags, &hcl.Diagnostic{ 467 Severity: hcl.DiagError, 468 Summary: "Failed to parse hcldec result", 469 Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step has invalid index key %s.", rawS.Key), 470 }) 471 return nil, diags 472 } 473 keyVal, err := ctyjson.Unmarshal([]byte(rawS.Key), ty) 474 if err != nil { 475 diags = append(diags, &hcl.Diagnostic{ 476 Severity: hcl.DiagError, 477 Summary: "Failed to parse hcldec result", 478 Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced a result with an invalid index key %s: %s.", rawS.Key, err), 479 }) 480 return nil, diags 481 } 482 483 traversal = append(traversal, hcl.TraverseIndex{ 484 Key: keyVal, 485 SrcRange: rng, 486 }) 487 488 default: 489 // Should never happen since the above cases are exhaustive, 490 // but we'll catch it gracefully since this is coming from 491 // a possibly-buggy hcldec implementation that we're testing. 492 diags = append(diags, &hcl.Diagnostic{ 493 Severity: hcl.DiagError, 494 Summary: "Failed to parse hcldec result", 495 Detail: fmt.Sprintf("Sub-program hcldec (with --var-refs) produced an invalid result: traversal step of unsupported kind %q.", rawS.Kind), 496 }) 497 return nil, diags 498 } 499 } 500 501 ret = append(ret, traversal) 502 } 503 return ret, diags 504 } 505} 506 507func (r *Runner) prettyDirName(dir string) string { 508 rel, err := filepath.Rel(r.baseDir, dir) 509 if err != nil { 510 return filepath.ToSlash(dir) 511 } 512 return filepath.ToSlash(rel) 513} 514 515func (r *Runner) prettyTestName(filename string) string { 516 dir := filepath.Dir(filename) 517 dirName := r.prettyDirName(dir) 518 filename = filepath.Base(filename) 519 testName := filename[:len(filename)-2] 520 if dirName == "." { 521 return testName 522 } 523 return fmt.Sprintf("%s/%s", dirName, testName) 524} 525