1// Copyright 2017 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//lint:file-ignore U1000 unused fns we might want to use later. 6 7package test 8 9import ( 10 "bytes" 11 "flag" 12 "fmt" 13 "go/format" 14 "io" 15 "io/ioutil" 16 "os" 17 "os/exec" 18 "path/filepath" 19 "regexp" 20 "runtime" 21 "strings" 22 "sync" 23 "testing" 24 25 "github.com/pkg/errors" 26) 27 28var ( 29 // ExeSuffix is the suffix of executable files; ".exe" on Windows. 30 ExeSuffix string 31 mu sync.Mutex 32 // PrintLogs controls logging of test commands. 33 PrintLogs = flag.Bool("logs", false, "log stdin/stdout of test commands") 34 // UpdateGolden controls updating test fixtures. 35 UpdateGolden = flag.Bool("update", false, "update golden files") 36) 37 38const ( 39 manifestName = "Gopkg.toml" 40 lockName = "Gopkg.lock" 41) 42 43func init() { 44 switch runtime.GOOS { 45 case "windows": 46 ExeSuffix = ".exe" 47 } 48} 49 50// Helper with utilities for testing. 51type Helper struct { 52 t *testing.T 53 temps []string 54 wd string 55 origWd string 56 env []string 57 tempdir string 58 ran bool 59 inParallel bool 60 stdout, stderr bytes.Buffer 61} 62 63// NewHelper initializes a new helper for testing. 64func NewHelper(t *testing.T) *Helper { 65 wd, err := os.Getwd() 66 if err != nil { 67 panic(err) 68 } 69 return &Helper{t: t, origWd: wd} 70} 71 72// Must gives a fatal error if err is not nil. 73func (h *Helper) Must(err error) { 74 if err != nil { 75 h.t.Fatalf("%+v", err) 76 } 77} 78 79// check gives a test non-fatal error if err is not nil. 80func (h *Helper) check(err error) { 81 if err != nil { 82 h.t.Errorf("%+v", err) 83 } 84} 85 86// Parallel runs the test in parallel by calling t.Parallel. 87func (h *Helper) Parallel() { 88 if h.ran { 89 h.t.Fatalf("%+v", errors.New("internal testsuite error: call to parallel after run")) 90 } 91 if h.wd != "" { 92 h.t.Fatalf("%+v", errors.New("internal testsuite error: call to parallel after cd")) 93 } 94 for _, e := range h.env { 95 if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { 96 val := e[strings.Index(e, "=")+1:] 97 if strings.HasPrefix(val, "testdata") || strings.HasPrefix(val, "./testdata") { 98 h.t.Fatalf("%+v", errors.Errorf("internal testsuite error: call to parallel with testdata in environment (%s)", e)) 99 } 100 } 101 } 102 h.inParallel = true 103 h.t.Parallel() 104} 105 106// pwd returns the current directory. 107func (h *Helper) pwd() string { 108 wd, err := os.Getwd() 109 if err != nil { 110 h.t.Fatalf("%+v", errors.Wrap(err, "could not get working directory")) 111 } 112 return wd 113} 114 115// Cd changes the current directory to the named directory. Note that 116// using this means that the test must not be run in parallel with any 117// other tests. 118func (h *Helper) Cd(dir string) { 119 if h.inParallel { 120 h.t.Fatalf("%+v", errors.New("internal testsuite error: changing directory when running in parallel")) 121 } 122 if h.wd == "" { 123 h.wd = h.pwd() 124 } 125 abs, err := filepath.Abs(dir) 126 if err == nil { 127 h.Setenv("PWD", abs) 128 } 129 130 err = os.Chdir(dir) 131 h.Must(errors.Wrapf(err, "Unable to cd to %s", dir)) 132} 133 134// Setenv sets an environment variable to use when running the test go 135// command. 136func (h *Helper) Setenv(name, val string) { 137 if h.inParallel && (name == "GOROOT" || name == "GOPATH" || name == "GOBIN") && (strings.HasPrefix(val, "testdata") || strings.HasPrefix(val, "./testdata")) { 138 h.t.Fatalf("%+v", errors.Errorf("internal testsuite error: call to setenv with testdata (%s=%s) after parallel", name, val)) 139 } 140 h.unsetenv(name) 141 h.env = append(h.env, name+"="+val) 142} 143 144// unsetenv removes an environment variable. 145func (h *Helper) unsetenv(name string) { 146 if h.env == nil { 147 h.env = append([]string(nil), os.Environ()...) 148 } 149 for i, v := range h.env { 150 if strings.HasPrefix(v, name+"=") { 151 h.env = append(h.env[:i], h.env[i+1:]...) 152 break 153 } 154 } 155} 156 157// DoRun runs the test go command, recording stdout and stderr and 158// returning exit status. 159func (h *Helper) DoRun(args []string) error { 160 if h.inParallel { 161 for _, arg := range args { 162 if strings.HasPrefix(arg, "testdata") || strings.HasPrefix(arg, "./testdata") { 163 h.t.Fatalf("%+v", errors.New("internal testsuite error: parallel run using testdata")) 164 } 165 } 166 } 167 if *PrintLogs { 168 h.t.Logf("running testdep %v", args) 169 } 170 var prog string 171 if h.wd == "" { 172 prog = "./testdep" + ExeSuffix 173 } else { 174 prog = filepath.Join(h.wd, "testdep"+ExeSuffix) 175 } 176 newargs := args 177 if args[0] != "check" { 178 newargs = append([]string{args[0], "-v"}, args[1:]...) 179 } 180 181 cmd := exec.Command(prog, newargs...) 182 h.stdout.Reset() 183 h.stderr.Reset() 184 cmd.Stdout = &h.stdout 185 cmd.Stderr = &h.stderr 186 cmd.Env = h.env 187 status := cmd.Run() 188 if *PrintLogs { 189 if h.stdout.Len() > 0 { 190 h.t.Log("standard output:") 191 h.t.Log(h.stdout.String()) 192 } 193 if h.stderr.Len() > 0 { 194 h.t.Log("standard error:") 195 h.t.Log(h.stderr.String()) 196 } 197 } 198 h.ran = true 199 return errors.Wrapf(status, "Error running %s\n%s", strings.Join(newargs, " "), h.stderr.String()) 200} 201 202// Run runs the test go command, and expects it to succeed. 203func (h *Helper) Run(args ...string) { 204 if runtime.GOOS == "windows" { 205 mu.Lock() 206 defer mu.Unlock() 207 } 208 if status := h.DoRun(args); status != nil { 209 h.t.Logf("go %v failed unexpectedly: %v", args, status) 210 h.t.FailNow() 211 } 212} 213 214// runFail runs the test go command, and expects it to fail. 215func (h *Helper) runFail(args ...string) { 216 if status := h.DoRun(args); status == nil { 217 h.t.Fatalf("%+v", errors.New("testgo succeeded unexpectedly")) 218 } else { 219 h.t.Log("testgo failed as expected:", status) 220 } 221} 222 223// RunGo runs a go command, and expects it to succeed. 224func (h *Helper) RunGo(args ...string) { 225 cmd := exec.Command("go", args...) 226 h.stdout.Reset() 227 h.stderr.Reset() 228 cmd.Stdout = &h.stdout 229 cmd.Stderr = &h.stderr 230 cmd.Dir = h.wd 231 cmd.Env = h.env 232 status := cmd.Run() 233 if h.stdout.Len() > 0 { 234 h.t.Log("go standard output:") 235 h.t.Log(h.stdout.String()) 236 } 237 if h.stderr.Len() > 0 { 238 h.t.Log("go standard error:") 239 h.t.Log(h.stderr.String()) 240 } 241 if status != nil { 242 h.t.Logf("go %v failed unexpectedly: %v", args, status) 243 h.t.FailNow() 244 } 245} 246 247// NeedsExternalNetwork makes sure the tests needing external network will not 248// be run when executing tests in short mode. 249func NeedsExternalNetwork(t *testing.T) { 250 if testing.Short() { 251 t.Skip("skipping test: no external network in -short mode") 252 } 253} 254 255// NeedsGit will make sure the tests that require git will be skipped if the 256// git binary is not available. 257func NeedsGit(t *testing.T) { 258 if _, err := exec.LookPath("git"); err != nil { 259 t.Skip("skipping because git binary not found") 260 } 261} 262 263// RunGit runs a git command, and expects it to succeed. 264func (h *Helper) RunGit(dir string, args ...string) { 265 cmd := exec.Command("git", args...) 266 h.stdout.Reset() 267 h.stderr.Reset() 268 cmd.Stdout = &h.stdout 269 cmd.Stderr = &h.stderr 270 cmd.Dir = dir 271 cmd.Env = h.env 272 status := cmd.Run() 273 if *PrintLogs { 274 if h.stdout.Len() > 0 { 275 h.t.Logf("git %v standard output:", args) 276 h.t.Log(h.stdout.String()) 277 } 278 if h.stderr.Len() > 0 { 279 h.t.Logf("git %v standard error:", args) 280 h.t.Log(h.stderr.String()) 281 } 282 } 283 if status != nil { 284 h.t.Logf("git %v failed unexpectedly: %v", args, status) 285 h.t.FailNow() 286 } 287} 288 289// getStdout returns standard output of the testgo run as a string. 290func (h *Helper) getStdout() string { 291 if !h.ran { 292 h.t.Fatalf("%+v", errors.New("internal testsuite error: stdout called before run")) 293 } 294 return h.stdout.String() 295} 296 297// getStderr returns standard error of the testgo run as a string. 298func (h *Helper) getStderr() string { 299 if !h.ran { 300 h.t.Fatalf("%+v", errors.New("internal testsuite error: stdout called before run")) 301 } 302 return h.stderr.String() 303} 304 305// doGrepMatch looks for a regular expression in a buffer, and returns 306// whether it is found. The regular expression is matched against 307// each line separately, as with the grep command. 308func (h *Helper) doGrepMatch(match string, b *bytes.Buffer) bool { 309 if !h.ran { 310 h.t.Fatalf("%+v", errors.New("internal testsuite error: grep called before run")) 311 } 312 re := regexp.MustCompile(match) 313 for _, ln := range bytes.Split(b.Bytes(), []byte{'\n'}) { 314 if re.Match(ln) { 315 return true 316 } 317 } 318 return false 319} 320 321// doGrep looks for a regular expression in a buffer and fails if it 322// is not found. The name argument is the name of the output we are 323// searching, "output" or "error". The msg argument is logged on 324// failure. 325func (h *Helper) doGrep(match string, b *bytes.Buffer, name, msg string) { 326 if !h.doGrepMatch(match, b) { 327 h.t.Log(msg) 328 h.t.Logf("pattern %v not found in standard %s", match, name) 329 h.t.FailNow() 330 } 331} 332 333// grepStdout looks for a regular expression in the test run's 334// standard output and fails, logging msg, if it is not found. 335func (h *Helper) grepStdout(match, msg string) { 336 h.doGrep(match, &h.stdout, "output", msg) 337} 338 339// grepStderr looks for a regular expression in the test run's 340// standard error and fails, logging msg, if it is not found. 341func (h *Helper) grepStderr(match, msg string) { 342 h.doGrep(match, &h.stderr, "error", msg) 343} 344 345// grepBoth looks for a regular expression in the test run's standard 346// output or stand error and fails, logging msg, if it is not found. 347func (h *Helper) grepBoth(match, msg string) { 348 if !h.doGrepMatch(match, &h.stdout) && !h.doGrepMatch(match, &h.stderr) { 349 h.t.Log(msg) 350 h.t.Logf("pattern %v not found in standard output or standard error", match) 351 h.t.FailNow() 352 } 353} 354 355// doGrepNot looks for a regular expression in a buffer and fails if 356// it is found. The name and msg arguments are as for doGrep. 357func (h *Helper) doGrepNot(match string, b *bytes.Buffer, name, msg string) { 358 if h.doGrepMatch(match, b) { 359 h.t.Log(msg) 360 h.t.Logf("pattern %v found unexpectedly in standard %s", match, name) 361 h.t.FailNow() 362 } 363} 364 365// grepStdoutNot looks for a regular expression in the test run's 366// standard output and fails, logging msg, if it is found. 367func (h *Helper) grepStdoutNot(match, msg string) { 368 h.doGrepNot(match, &h.stdout, "output", msg) 369} 370 371// grepStderrNot looks for a regular expression in the test run's 372// standard error and fails, logging msg, if it is found. 373func (h *Helper) grepStderrNot(match, msg string) { 374 h.doGrepNot(match, &h.stderr, "error", msg) 375} 376 377// grepBothNot looks for a regular expression in the test run's 378// standard output or stand error and fails, logging msg, if it is 379// found. 380func (h *Helper) grepBothNot(match, msg string) { 381 if h.doGrepMatch(match, &h.stdout) || h.doGrepMatch(match, &h.stderr) { 382 h.t.Log(msg) 383 h.t.Fatalf("%+v", errors.Errorf("pattern %v found unexpectedly in standard output or standard error", match)) 384 } 385} 386 387// doGrepCount counts the number of times a regexp is seen in a buffer. 388func (h *Helper) doGrepCount(match string, b *bytes.Buffer) int { 389 if !h.ran { 390 h.t.Fatalf("%+v", errors.New("internal testsuite error: doGrepCount called before run")) 391 } 392 re := regexp.MustCompile(match) 393 c := 0 394 for _, ln := range bytes.Split(b.Bytes(), []byte{'\n'}) { 395 if re.Match(ln) { 396 c++ 397 } 398 } 399 return c 400} 401 402// grepCountBoth returns the number of times a regexp is seen in both 403// standard output and standard error. 404func (h *Helper) grepCountBoth(match string) int { 405 return h.doGrepCount(match, &h.stdout) + h.doGrepCount(match, &h.stderr) 406} 407 408// creatingTemp records that the test plans to create a temporary file 409// or directory. If the file or directory exists already, it will be 410// removed. When the test completes, the file or directory will be 411// removed if it exists. 412func (h *Helper) creatingTemp(path string) { 413 if filepath.IsAbs(path) && !strings.HasPrefix(path, h.tempdir) { 414 h.t.Fatalf("%+v", errors.Errorf("internal testsuite error: creatingTemp(%q) with absolute path not in temporary directory", path)) 415 } 416 // If we have changed the working directory, make sure we have 417 // an absolute path, because we are going to change directory 418 // back before we remove the temporary. 419 if h.wd != "" && !filepath.IsAbs(path) { 420 path = filepath.Join(h.pwd(), path) 421 } 422 h.Must(os.RemoveAll(path)) 423 h.temps = append(h.temps, path) 424} 425 426// makeTempdir makes a temporary directory for a run of testgo. If 427// the temporary directory was already created, this does nothing. 428func (h *Helper) makeTempdir() { 429 if h.tempdir == "" { 430 var err error 431 h.tempdir, err = ioutil.TempDir("", "gotest") 432 h.Must(err) 433 } 434} 435 436// TempFile adds a temporary file for a run of testgo. 437func (h *Helper) TempFile(path, contents string) { 438 h.makeTempdir() 439 h.Must(os.MkdirAll(filepath.Join(h.tempdir, filepath.Dir(path)), 0755)) 440 bytes := []byte(contents) 441 if strings.HasSuffix(path, ".go") { 442 formatted, err := format.Source(bytes) 443 if err == nil { 444 bytes = formatted 445 } 446 } 447 h.Must(ioutil.WriteFile(filepath.Join(h.tempdir, path), bytes, 0644)) 448} 449 450// WriteTestFile writes a file to the testdata directory from memory. src is 451// relative to ./testdata. 452func (h *Helper) WriteTestFile(src string, content string) error { 453 err := ioutil.WriteFile(filepath.Join(h.origWd, "testdata", src), []byte(content), 0666) 454 return err 455} 456 457// GetFile reads a file into memory 458func (h *Helper) GetFile(path string) io.ReadCloser { 459 content, err := os.Open(path) 460 if err != nil { 461 h.t.Fatalf("%+v", errors.Wrapf(err, "Unable to open file: %s", path)) 462 } 463 return content 464} 465 466// GetTestFile reads a file from the testdata directory into memory. src is 467// relative to ./testdata. 468func (h *Helper) GetTestFile(src string) io.ReadCloser { 469 fullPath := filepath.Join(h.origWd, "testdata", src) 470 return h.GetFile(fullPath) 471} 472 473// GetTestFileString reads a file from the testdata directory into memory. src is 474// relative to ./testdata. 475func (h *Helper) GetTestFileString(src string) string { 476 srcf := h.GetTestFile(src) 477 defer srcf.Close() 478 content, err := ioutil.ReadAll(srcf) 479 if err != nil { 480 h.t.Fatalf("%+v", err) 481 } 482 return string(content) 483} 484 485// TempCopy copies a temporary file from testdata into the temporary directory. 486// dest is relative to the temp directory location, and src is relative to 487// ./testdata. 488func (h *Helper) TempCopy(dest, src string) { 489 in := h.GetTestFile(src) 490 defer in.Close() 491 h.TempDir(filepath.Dir(dest)) 492 out, err := os.Create(filepath.Join(h.tempdir, dest)) 493 if err != nil { 494 panic(err) 495 } 496 defer out.Close() 497 io.Copy(out, in) 498} 499 500// TempDir adds a temporary directory for a run of testgo. 501func (h *Helper) TempDir(path string) { 502 h.makeTempdir() 503 fullPath := filepath.Join(h.tempdir, path) 504 if err := os.MkdirAll(fullPath, 0755); err != nil && !os.IsExist(err) { 505 h.t.Fatalf("%+v", errors.Errorf("Unable to create temp directory: %s", fullPath)) 506 } 507} 508 509// Path returns the absolute pathname to file with the temporary 510// directory. 511func (h *Helper) Path(name string) string { 512 if h.tempdir == "" { 513 h.t.Fatalf("%+v", errors.Errorf("internal testsuite error: path(%q) with no tempdir", name)) 514 } 515 516 var joined string 517 if name == "." { 518 joined = h.tempdir 519 } else { 520 joined = filepath.Join(h.tempdir, name) 521 } 522 523 // Ensure it's the absolute, symlink-less path we're returning 524 abs, err := filepath.EvalSymlinks(joined) 525 if err != nil { 526 h.t.Fatalf("%+v", errors.Wrapf(err, "internal testsuite error: could not get absolute path for dir(%q)", joined)) 527 } 528 return abs 529} 530 531// MustExist fails if path does not exist. 532func (h *Helper) MustExist(path string) { 533 if err := h.ShouldExist(path); err != nil { 534 h.t.Fatalf("%+v", err) 535 } 536} 537 538// ShouldExist returns an error if path does not exist. 539func (h *Helper) ShouldExist(path string) error { 540 if !h.Exist(path) { 541 return errors.Errorf("%s does not exist but should", path) 542 } 543 544 return nil 545} 546 547// Exist returns whether or not a path exists 548func (h *Helper) Exist(path string) bool { 549 if _, err := os.Stat(path); err != nil { 550 if os.IsNotExist(err) { 551 return false 552 } 553 h.t.Fatalf("%+v", errors.Wrapf(err, "Error checking if path exists: %s", path)) 554 } 555 556 return true 557} 558 559// MustNotExist fails if path exists. 560func (h *Helper) MustNotExist(path string) { 561 if err := h.ShouldNotExist(path); err != nil { 562 h.t.Fatalf("%+v", err) 563 } 564} 565 566// ShouldNotExist returns an error if path exists. 567func (h *Helper) ShouldNotExist(path string) error { 568 if h.Exist(path) { 569 return errors.Errorf("%s exists but should not", path) 570 } 571 572 return nil 573} 574 575// Cleanup cleans up a test that runs testgo. 576func (h *Helper) Cleanup() { 577 if h.wd != "" { 578 if err := os.Chdir(h.wd); err != nil { 579 // We are unlikely to be able to continue. 580 fmt.Fprintln(os.Stderr, "could not restore working directory, crashing:", err) 581 os.Exit(2) 582 } 583 } 584 // NOTE(mattn): It seems that sometimes git.exe is not dead 585 // when cleanup() is called. But we do not know any way to wait for it. 586 if runtime.GOOS == "windows" { 587 mu.Lock() 588 exec.Command(`taskkill`, `/F`, `/IM`, `git.exe`).Run() 589 mu.Unlock() 590 } 591 for _, path := range h.temps { 592 h.check(os.RemoveAll(path)) 593 } 594 if h.tempdir != "" { 595 h.check(os.RemoveAll(h.tempdir)) 596 } 597} 598 599// ReadManifest returns the manifest in the current directory. 600func (h *Helper) ReadManifest() string { 601 m := filepath.Join(h.pwd(), manifestName) 602 h.MustExist(m) 603 604 f, err := ioutil.ReadFile(m) 605 h.Must(err) 606 return string(f) 607} 608 609// ReadLock returns the lock in the current directory. 610func (h *Helper) ReadLock() string { 611 l := filepath.Join(h.pwd(), lockName) 612 h.MustExist(l) 613 614 f, err := ioutil.ReadFile(l) 615 h.Must(err) 616 return string(f) 617} 618 619// GetCommit treats repo as a path to a git repository and returns the current 620// revision. 621func (h *Helper) GetCommit(repo string) string { 622 repoPath := h.Path("pkg/dep/sources/https---" + strings.Replace(repo, "/", "-", -1)) 623 cmd := exec.Command("git", "rev-parse", "HEAD") 624 cmd.Dir = repoPath 625 out, err := cmd.CombinedOutput() 626 if err != nil { 627 h.t.Fatalf("%+v", errors.Wrapf(err, "git commit failed: out -> %s", string(out))) 628 } 629 return strings.TrimSpace(string(out)) 630} 631