1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Package analysistest provides utilities for testing analyzers. 6package analysistest 7 8import ( 9 "bytes" 10 "fmt" 11 "go/format" 12 "go/token" 13 "go/types" 14 "io/ioutil" 15 "log" 16 "os" 17 "path/filepath" 18 "regexp" 19 "sort" 20 "strconv" 21 "strings" 22 "text/scanner" 23 24 "golang.org/x/tools/go/analysis" 25 "golang.org/x/tools/go/analysis/internal/checker" 26 "golang.org/x/tools/go/packages" 27 "golang.org/x/tools/internal/lsp/diff" 28 "golang.org/x/tools/internal/lsp/diff/myers" 29 "golang.org/x/tools/internal/span" 30 "golang.org/x/tools/internal/testenv" 31 "golang.org/x/tools/txtar" 32) 33 34// WriteFiles is a helper function that creates a temporary directory 35// and populates it with a GOPATH-style project using filemap (which 36// maps file names to contents). On success it returns the name of the 37// directory and a cleanup function to delete it. 38func WriteFiles(filemap map[string]string) (dir string, cleanup func(), err error) { 39 gopath, err := ioutil.TempDir("", "analysistest") 40 if err != nil { 41 return "", nil, err 42 } 43 cleanup = func() { os.RemoveAll(gopath) } 44 45 for name, content := range filemap { 46 filename := filepath.Join(gopath, "src", name) 47 os.MkdirAll(filepath.Dir(filename), 0777) // ignore error 48 if err := ioutil.WriteFile(filename, []byte(content), 0666); err != nil { 49 cleanup() 50 return "", nil, err 51 } 52 } 53 return gopath, cleanup, nil 54} 55 56// TestData returns the effective filename of 57// the program's "testdata" directory. 58// This function may be overridden by projects using 59// an alternative build system (such as Blaze) that 60// does not run a test in its package directory. 61var TestData = func() string { 62 testdata, err := filepath.Abs("testdata") 63 if err != nil { 64 log.Fatal(err) 65 } 66 return testdata 67} 68 69// Testing is an abstraction of a *testing.T. 70type Testing interface { 71 Errorf(format string, args ...interface{}) 72} 73 74// RunWithSuggestedFixes behaves like Run, but additionally verifies suggested fixes. 75// It uses golden files placed alongside the source code under analysis: 76// suggested fixes for code in example.go will be compared against example.go.golden. 77// 78// Golden files can be formatted in one of two ways: as plain Go source code, or as txtar archives. 79// In the first case, all suggested fixes will be applied to the original source, which will then be compared against the golden file. 80// In the second case, suggested fixes will be grouped by their messages, and each set of fixes will be applied and tested separately. 81// Each section in the archive corresponds to a single message. 82// 83// A golden file using txtar may look like this: 84// -- turn into single negation -- 85// package pkg 86// 87// func fn(b1, b2 bool) { 88// if !b1 { // want `negating a boolean twice` 89// println() 90// } 91// } 92// 93// -- remove double negation -- 94// package pkg 95// 96// func fn(b1, b2 bool) { 97// if b1 { // want `negating a boolean twice` 98// println() 99// } 100// } 101func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result { 102 r := Run(t, dir, a, patterns...) 103 104 // Process each result (package) separately, matching up the suggested 105 // fixes into a diff, which we will compare to the .golden file. We have 106 // to do this per-result in case a file appears in two packages, such as in 107 // packages with tests, where mypkg/a.go will appear in both mypkg and 108 // mypkg.test. In that case, the analyzer may suggest the same set of 109 // changes to a.go for each package. If we merge all the results, those 110 // changes get doubly applied, which will cause conflicts or mismatches. 111 // Validating the results separately means as long as the two analyses 112 // don't produce conflicting suggestions for a single file, everything 113 // should match up. 114 for _, act := range r { 115 // file -> message -> edits 116 fileEdits := make(map[*token.File]map[string][]diff.TextEdit) 117 fileContents := make(map[*token.File][]byte) 118 119 // Validate edits, prepare the fileEdits map and read the file contents. 120 for _, diag := range act.Diagnostics { 121 for _, sf := range diag.SuggestedFixes { 122 for _, edit := range sf.TextEdits { 123 // Validate the edit. 124 if edit.Pos > edit.End { 125 t.Errorf( 126 "diagnostic for analysis %v contains Suggested Fix with malformed edit: pos (%v) > end (%v)", 127 act.Pass.Analyzer.Name, edit.Pos, edit.End) 128 continue 129 } 130 file, endfile := act.Pass.Fset.File(edit.Pos), act.Pass.Fset.File(edit.End) 131 if file == nil || endfile == nil || file != endfile { 132 t.Errorf( 133 "diagnostic for analysis %v contains Suggested Fix with malformed spanning files %v and %v", 134 act.Pass.Analyzer.Name, file.Name(), endfile.Name()) 135 continue 136 } 137 if _, ok := fileContents[file]; !ok { 138 contents, err := ioutil.ReadFile(file.Name()) 139 if err != nil { 140 t.Errorf("error reading %s: %v", file.Name(), err) 141 } 142 fileContents[file] = contents 143 } 144 spn, err := span.NewRange(act.Pass.Fset, edit.Pos, edit.End).Span() 145 if err != nil { 146 t.Errorf("error converting edit to span %s: %v", file.Name(), err) 147 } 148 149 if _, ok := fileEdits[file]; !ok { 150 fileEdits[file] = make(map[string][]diff.TextEdit) 151 } 152 fileEdits[file][sf.Message] = append(fileEdits[file][sf.Message], diff.TextEdit{ 153 Span: spn, 154 NewText: string(edit.NewText), 155 }) 156 } 157 } 158 } 159 160 for file, fixes := range fileEdits { 161 // Get the original file contents. 162 orig, ok := fileContents[file] 163 if !ok { 164 t.Errorf("could not find file contents for %s", file.Name()) 165 continue 166 } 167 168 // Get the golden file and read the contents. 169 ar, err := txtar.ParseFile(file.Name() + ".golden") 170 if err != nil { 171 t.Errorf("error reading %s.golden: %v", file.Name(), err) 172 continue 173 } 174 175 if len(ar.Files) > 0 { 176 // one virtual file per kind of suggested fix 177 178 if len(ar.Comment) != 0 { 179 // we allow either just the comment, or just virtual 180 // files, not both. it is not clear how "both" should 181 // behave. 182 t.Errorf("%s.golden has leading comment; we don't know what to do with it", file.Name()) 183 continue 184 } 185 186 for sf, edits := range fixes { 187 found := false 188 for _, vf := range ar.Files { 189 if vf.Name == sf { 190 found = true 191 out := diff.ApplyEdits(string(orig), edits) 192 // the file may contain multiple trailing 193 // newlines if the user places empty lines 194 // between files in the archive. normalize 195 // this to a single newline. 196 want := string(bytes.TrimRight(vf.Data, "\n")) + "\n" 197 formatted, err := format.Source([]byte(out)) 198 if err != nil { 199 continue 200 } 201 if want != string(formatted) { 202 d, err := myers.ComputeEdits("", want, string(formatted)) 203 if err != nil { 204 t.Errorf("failed to compute suggested fixes: %v", err) 205 } 206 t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), diff.ToUnified(fmt.Sprintf("%s.golden [%s]", file.Name(), sf), "actual", want, d)) 207 } 208 break 209 } 210 } 211 if !found { 212 t.Errorf("no section for suggested fix %q in %s.golden", sf, file.Name()) 213 } 214 } 215 } else { 216 // all suggested fixes are represented by a single file 217 218 var catchallEdits []diff.TextEdit 219 for _, edits := range fixes { 220 catchallEdits = append(catchallEdits, edits...) 221 } 222 223 out := diff.ApplyEdits(string(orig), catchallEdits) 224 want := string(ar.Comment) 225 226 formatted, err := format.Source([]byte(out)) 227 if err != nil { 228 continue 229 } 230 if want != string(formatted) { 231 d, err := myers.ComputeEdits("", want, string(formatted)) 232 if err != nil { 233 t.Errorf("failed to compute edits: %s", err) 234 } 235 t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), diff.ToUnified(file.Name()+".golden", "actual", want, d)) 236 } 237 } 238 } 239 } 240 return r 241} 242 243// Run applies an analysis to the packages denoted by the "go list" patterns. 244// 245// It loads the packages from the specified GOPATH-style project 246// directory using golang.org/x/tools/go/packages, runs the analysis on 247// them, and checks that each analysis emits the expected diagnostics 248// and facts specified by the contents of '// want ...' comments in the 249// package's source files. 250// 251// An expectation of a Diagnostic is specified by a string literal 252// containing a regular expression that must match the diagnostic 253// message. For example: 254// 255// fmt.Printf("%s", 1) // want `cannot provide int 1 to %s` 256// 257// An expectation of a Fact associated with an object is specified by 258// 'name:"pattern"', where name is the name of the object, which must be 259// declared on the same line as the comment, and pattern is a regular 260// expression that must match the string representation of the fact, 261// fmt.Sprint(fact). For example: 262// 263// func panicf(format string, args interface{}) { // want panicf:"printfWrapper" 264// 265// Package facts are specified by the name "package" and appear on 266// line 1 of the first source file of the package. 267// 268// A single 'want' comment may contain a mixture of diagnostic and fact 269// expectations, including multiple facts about the same object: 270// 271// // want "diag" "diag2" x:"fact1" x:"fact2" y:"fact3" 272// 273// Unexpected diagnostics and facts, and unmatched expectations, are 274// reported as errors to the Testing. 275// 276// Run reports an error to the Testing if loading or analysis failed. 277// Run also returns a Result for each package for which analysis was 278// attempted, even if unsuccessful. It is safe for a test to ignore all 279// the results, but a test may use it to perform additional checks. 280func Run(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result { 281 if t, ok := t.(testenv.Testing); ok { 282 testenv.NeedsGoPackages(t) 283 } 284 285 pkgs, err := loadPackages(dir, patterns...) 286 if err != nil { 287 t.Errorf("loading %s: %v", patterns, err) 288 return nil 289 } 290 291 results := checker.TestAnalyzer(a, pkgs) 292 for _, result := range results { 293 if result.Err != nil { 294 t.Errorf("error analyzing %s: %v", result.Pass, result.Err) 295 } else { 296 check(t, dir, result.Pass, result.Diagnostics, result.Facts) 297 } 298 } 299 return results 300} 301 302// A Result holds the result of applying an analyzer to a package. 303type Result = checker.TestAnalyzerResult 304 305// loadPackages uses go/packages to load a specified packages (from source, with 306// dependencies) from dir, which is the root of a GOPATH-style project 307// tree. It returns an error if any package had an error, or the pattern 308// matched no packages. 309func loadPackages(dir string, patterns ...string) ([]*packages.Package, error) { 310 // packages.Load loads the real standard library, not a minimal 311 // fake version, which would be more efficient, especially if we 312 // have many small tests that import, say, net/http. 313 // However there is no easy way to make go/packages to consume 314 // a list of packages we generate and then do the parsing and 315 // typechecking, though this feature seems to be a recurring need. 316 317 cfg := &packages.Config{ 318 Mode: packages.LoadAllSyntax, 319 Dir: dir, 320 Tests: true, 321 Env: append(os.Environ(), "GOPATH="+dir, "GO111MODULE=off", "GOPROXY=off"), 322 } 323 pkgs, err := packages.Load(cfg, patterns...) 324 if err != nil { 325 return nil, err 326 } 327 328 // Print errors but do not stop: 329 // some Analyzers may be disposed to RunDespiteErrors. 330 packages.PrintErrors(pkgs) 331 332 if len(pkgs) == 0 { 333 return nil, fmt.Errorf("no packages matched %s", patterns) 334 } 335 return pkgs, nil 336} 337 338// check inspects an analysis pass on which the analysis has already 339// been run, and verifies that all reported diagnostics and facts match 340// specified by the contents of "// want ..." comments in the package's 341// source files, which must have been parsed with comments enabled. 342func check(t Testing, gopath string, pass *analysis.Pass, diagnostics []analysis.Diagnostic, facts map[types.Object][]analysis.Fact) { 343 type key struct { 344 file string 345 line int 346 } 347 348 want := make(map[key][]expectation) 349 350 // processComment parses expectations out of comments. 351 processComment := func(filename string, linenum int, text string) { 352 text = strings.TrimSpace(text) 353 354 // Any comment starting with "want" is treated 355 // as an expectation, even without following whitespace. 356 if rest := strings.TrimPrefix(text, "want"); rest != text { 357 lineDelta, expects, err := parseExpectations(rest) 358 if err != nil { 359 t.Errorf("%s:%d: in 'want' comment: %s", filename, linenum, err) 360 return 361 } 362 if expects != nil { 363 want[key{filename, linenum + lineDelta}] = expects 364 } 365 } 366 } 367 368 // Extract 'want' comments from parsed Go files. 369 for _, f := range pass.Files { 370 for _, cgroup := range f.Comments { 371 for _, c := range cgroup.List { 372 373 text := strings.TrimPrefix(c.Text, "//") 374 if text == c.Text { // not a //-comment. 375 text = strings.TrimPrefix(text, "/*") 376 text = strings.TrimSuffix(text, "*/") 377 } 378 379 // Hack: treat a comment of the form "//...// want..." 380 // or "/*...// want... */ 381 // as if it starts at 'want'. 382 // This allows us to add comments on comments, 383 // as required when testing the buildtag analyzer. 384 if i := strings.Index(text, "// want"); i >= 0 { 385 text = text[i+len("// "):] 386 } 387 388 // It's tempting to compute the filename 389 // once outside the loop, but it's 390 // incorrect because it can change due 391 // to //line directives. 392 posn := pass.Fset.Position(c.Pos()) 393 filename := sanitize(gopath, posn.Filename) 394 processComment(filename, posn.Line, text) 395 } 396 } 397 } 398 399 // Extract 'want' comments from non-Go files. 400 // TODO(adonovan): we may need to handle //line directives. 401 for _, filename := range pass.OtherFiles { 402 data, err := ioutil.ReadFile(filename) 403 if err != nil { 404 t.Errorf("can't read '// want' comments from %s: %v", filename, err) 405 continue 406 } 407 filename := sanitize(gopath, filename) 408 linenum := 0 409 for _, line := range strings.Split(string(data), "\n") { 410 linenum++ 411 412 // Hack: treat a comment of the form "//...// want..." 413 // or "/*...// want... */ 414 // as if it starts at 'want'. 415 // This allows us to add comments on comments, 416 // as required when testing the buildtag analyzer. 417 if i := strings.Index(line, "// want"); i >= 0 { 418 line = line[i:] 419 } 420 421 if i := strings.Index(line, "//"); i >= 0 { 422 line = line[i+len("//"):] 423 processComment(filename, linenum, line) 424 } 425 } 426 } 427 428 checkMessage := func(posn token.Position, kind, name, message string) { 429 posn.Filename = sanitize(gopath, posn.Filename) 430 k := key{posn.Filename, posn.Line} 431 expects := want[k] 432 var unmatched []string 433 for i, exp := range expects { 434 if exp.kind == kind && exp.name == name { 435 if exp.rx.MatchString(message) { 436 // matched: remove the expectation. 437 expects[i] = expects[len(expects)-1] 438 expects = expects[:len(expects)-1] 439 want[k] = expects 440 return 441 } 442 unmatched = append(unmatched, fmt.Sprintf("%q", exp.rx)) 443 } 444 } 445 if unmatched == nil { 446 t.Errorf("%v: unexpected %s: %v", posn, kind, message) 447 } else { 448 t.Errorf("%v: %s %q does not match pattern %s", 449 posn, kind, message, strings.Join(unmatched, " or ")) 450 } 451 } 452 453 // Check the diagnostics match expectations. 454 for _, f := range diagnostics { 455 // TODO(matloob): Support ranges in analysistest. 456 posn := pass.Fset.Position(f.Pos) 457 checkMessage(posn, "diagnostic", "", f.Message) 458 } 459 460 // Check the facts match expectations. 461 // Report errors in lexical order for determinism. 462 // (It's only deterministic within each file, not across files, 463 // because go/packages does not guarantee file.Pos is ascending 464 // across the files of a single compilation unit.) 465 var objects []types.Object 466 for obj := range facts { 467 objects = append(objects, obj) 468 } 469 sort.Slice(objects, func(i, j int) bool { 470 // Package facts compare less than object facts. 471 ip, jp := objects[i] == nil, objects[j] == nil // whether i, j is a package fact 472 if ip != jp { 473 return ip && !jp 474 } 475 return objects[i].Pos() < objects[j].Pos() 476 }) 477 for _, obj := range objects { 478 var posn token.Position 479 var name string 480 if obj != nil { 481 // Object facts are reported on the declaring line. 482 name = obj.Name() 483 posn = pass.Fset.Position(obj.Pos()) 484 } else { 485 // Package facts are reported at the start of the file. 486 name = "package" 487 posn = pass.Fset.Position(pass.Files[0].Pos()) 488 posn.Line = 1 489 } 490 491 for _, fact := range facts[obj] { 492 checkMessage(posn, "fact", name, fmt.Sprint(fact)) 493 } 494 } 495 496 // Reject surplus expectations. 497 // 498 // Sometimes an Analyzer reports two similar diagnostics on a 499 // line with only one expectation. The reader may be confused by 500 // the error message. 501 // TODO(adonovan): print a better error: 502 // "got 2 diagnostics here; each one needs its own expectation". 503 var surplus []string 504 for key, expects := range want { 505 for _, exp := range expects { 506 err := fmt.Sprintf("%s:%d: no %s was reported matching %q", key.file, key.line, exp.kind, exp.rx) 507 surplus = append(surplus, err) 508 } 509 } 510 sort.Strings(surplus) 511 for _, err := range surplus { 512 t.Errorf("%s", err) 513 } 514} 515 516type expectation struct { 517 kind string // either "fact" or "diagnostic" 518 name string // name of object to which fact belongs, or "package" ("fact" only) 519 rx *regexp.Regexp 520} 521 522func (ex expectation) String() string { 523 return fmt.Sprintf("%s %s:%q", ex.kind, ex.name, ex.rx) // for debugging 524} 525 526// parseExpectations parses the content of a "// want ..." comment 527// and returns the expectations, a mixture of diagnostics ("rx") and 528// facts (name:"rx"). 529func parseExpectations(text string) (lineDelta int, expects []expectation, err error) { 530 var scanErr string 531 sc := new(scanner.Scanner).Init(strings.NewReader(text)) 532 sc.Error = func(s *scanner.Scanner, msg string) { 533 scanErr = msg // e.g. bad string escape 534 } 535 sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanInts 536 537 scanRegexp := func(tok rune) (*regexp.Regexp, error) { 538 if tok != scanner.String && tok != scanner.RawString { 539 return nil, fmt.Errorf("got %s, want regular expression", 540 scanner.TokenString(tok)) 541 } 542 pattern, _ := strconv.Unquote(sc.TokenText()) // can't fail 543 return regexp.Compile(pattern) 544 } 545 546 for { 547 tok := sc.Scan() 548 switch tok { 549 case '+': 550 tok = sc.Scan() 551 if tok != scanner.Int { 552 return 0, nil, fmt.Errorf("got +%s, want +Int", scanner.TokenString(tok)) 553 } 554 lineDelta, _ = strconv.Atoi(sc.TokenText()) 555 case scanner.String, scanner.RawString: 556 rx, err := scanRegexp(tok) 557 if err != nil { 558 return 0, nil, err 559 } 560 expects = append(expects, expectation{"diagnostic", "", rx}) 561 562 case scanner.Ident: 563 name := sc.TokenText() 564 tok = sc.Scan() 565 if tok != ':' { 566 return 0, nil, fmt.Errorf("got %s after %s, want ':'", 567 scanner.TokenString(tok), name) 568 } 569 tok = sc.Scan() 570 rx, err := scanRegexp(tok) 571 if err != nil { 572 return 0, nil, err 573 } 574 expects = append(expects, expectation{"fact", name, rx}) 575 576 case scanner.EOF: 577 if scanErr != "" { 578 return 0, nil, fmt.Errorf("%s", scanErr) 579 } 580 return lineDelta, expects, nil 581 582 default: 583 return 0, nil, fmt.Errorf("unexpected %s", scanner.TokenString(tok)) 584 } 585 } 586} 587 588// sanitize removes the GOPATH portion of the filename, 589// typically a gnarly /tmp directory, and returns the rest. 590func sanitize(gopath, filename string) string { 591 prefix := gopath + string(os.PathSeparator) + "src" + string(os.PathSeparator) 592 return filepath.ToSlash(strings.TrimPrefix(filename, prefix)) 593} 594