1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package packages 6 7import ( 8 "bytes" 9 "context" 10 "encoding/json" 11 "fmt" 12 "go/types" 13 "log" 14 "os" 15 "os/exec" 16 "path" 17 "path/filepath" 18 "reflect" 19 "sort" 20 "strconv" 21 "strings" 22 "sync" 23 "unicode" 24 25 "github.com/ooni/psiphon/oopsi/golang.org/x/tools/go/internal/packagesdriver" 26 "github.com/ooni/psiphon/oopsi/golang.org/x/tools/internal/gocommand" 27 "github.com/ooni/psiphon/oopsi/golang.org/x/tools/internal/packagesinternal" 28 "github.com/ooni/psiphon/oopsi/golang.org/x/xerrors" 29) 30 31// debug controls verbose logging. 32var debug, _ = strconv.ParseBool(os.Getenv("GOPACKAGESDEBUG")) 33 34// A goTooOldError reports that the go command 35// found by exec.LookPath is too old to use the new go list behavior. 36type goTooOldError struct { 37 error 38} 39 40// responseDeduper wraps a driverResponse, deduplicating its contents. 41type responseDeduper struct { 42 seenRoots map[string]bool 43 seenPackages map[string]*Package 44 dr *driverResponse 45} 46 47func newDeduper() *responseDeduper { 48 return &responseDeduper{ 49 dr: &driverResponse{}, 50 seenRoots: map[string]bool{}, 51 seenPackages: map[string]*Package{}, 52 } 53} 54 55// addAll fills in r with a driverResponse. 56func (r *responseDeduper) addAll(dr *driverResponse) { 57 for _, pkg := range dr.Packages { 58 r.addPackage(pkg) 59 } 60 for _, root := range dr.Roots { 61 r.addRoot(root) 62 } 63} 64 65func (r *responseDeduper) addPackage(p *Package) { 66 if r.seenPackages[p.ID] != nil { 67 return 68 } 69 r.seenPackages[p.ID] = p 70 r.dr.Packages = append(r.dr.Packages, p) 71} 72 73func (r *responseDeduper) addRoot(id string) { 74 if r.seenRoots[id] { 75 return 76 } 77 r.seenRoots[id] = true 78 r.dr.Roots = append(r.dr.Roots, id) 79} 80 81type golistState struct { 82 cfg *Config 83 ctx context.Context 84 85 envOnce sync.Once 86 goEnvError error 87 goEnv map[string]string 88 89 rootsOnce sync.Once 90 rootDirsError error 91 rootDirs map[string]string 92 93 // vendorDirs caches the (non)existence of vendor directories. 94 vendorDirs map[string]bool 95} 96 97// getEnv returns Go environment variables. Only specific variables are 98// populated -- computing all of them is slow. 99func (state *golistState) getEnv() (map[string]string, error) { 100 state.envOnce.Do(func() { 101 var b *bytes.Buffer 102 b, state.goEnvError = state.invokeGo("env", "-json", "GOMOD", "GOPATH") 103 if state.goEnvError != nil { 104 return 105 } 106 107 state.goEnv = make(map[string]string) 108 decoder := json.NewDecoder(b) 109 if state.goEnvError = decoder.Decode(&state.goEnv); state.goEnvError != nil { 110 return 111 } 112 }) 113 return state.goEnv, state.goEnvError 114} 115 116// mustGetEnv is a convenience function that can be used if getEnv has already succeeded. 117func (state *golistState) mustGetEnv() map[string]string { 118 env, err := state.getEnv() 119 if err != nil { 120 panic(fmt.Sprintf("mustGetEnv: %v", err)) 121 } 122 return env 123} 124 125// goListDriver uses the go list command to interpret the patterns and produce 126// the build system package structure. 127// See driver for more details. 128func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) { 129 // Make sure that any asynchronous go commands are killed when we return. 130 parentCtx := cfg.Context 131 if parentCtx == nil { 132 parentCtx = context.Background() 133 } 134 ctx, cancel := context.WithCancel(parentCtx) 135 defer cancel() 136 137 response := newDeduper() 138 139 // Fill in response.Sizes asynchronously if necessary. 140 var sizeserr error 141 var sizeswg sync.WaitGroup 142 if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 { 143 sizeswg.Add(1) 144 go func() { 145 var sizes types.Sizes 146 sizes, sizeserr = packagesdriver.GetSizesGolist(ctx, cfg.BuildFlags, cfg.Env, cfg.gocmdRunner, cfg.Dir) 147 // types.SizesFor always returns nil or a *types.StdSizes. 148 response.dr.Sizes, _ = sizes.(*types.StdSizes) 149 sizeswg.Done() 150 }() 151 } 152 153 state := &golistState{ 154 cfg: cfg, 155 ctx: ctx, 156 vendorDirs: map[string]bool{}, 157 } 158 159 // Determine files requested in contains patterns 160 var containFiles []string 161 restPatterns := make([]string, 0, len(patterns)) 162 // Extract file= and other [querytype]= patterns. Report an error if querytype 163 // doesn't exist. 164extractQueries: 165 for _, pattern := range patterns { 166 eqidx := strings.Index(pattern, "=") 167 if eqidx < 0 { 168 restPatterns = append(restPatterns, pattern) 169 } else { 170 query, value := pattern[:eqidx], pattern[eqidx+len("="):] 171 switch query { 172 case "file": 173 containFiles = append(containFiles, value) 174 case "pattern": 175 restPatterns = append(restPatterns, value) 176 case "": // not a reserved query 177 restPatterns = append(restPatterns, pattern) 178 default: 179 for _, rune := range query { 180 if rune < 'a' || rune > 'z' { // not a reserved query 181 restPatterns = append(restPatterns, pattern) 182 continue extractQueries 183 } 184 } 185 // Reject all other patterns containing "=" 186 return nil, fmt.Errorf("invalid query type %q in query pattern %q", query, pattern) 187 } 188 } 189 } 190 191 // See if we have any patterns to pass through to go list. Zero initial 192 // patterns also requires a go list call, since it's the equivalent of 193 // ".". 194 if len(restPatterns) > 0 || len(patterns) == 0 { 195 dr, err := state.createDriverResponse(restPatterns...) 196 if err != nil { 197 return nil, err 198 } 199 response.addAll(dr) 200 } 201 202 if len(containFiles) != 0 { 203 if err := state.runContainsQueries(response, containFiles); err != nil { 204 return nil, err 205 } 206 } 207 208 modifiedPkgs, needPkgs, err := state.processGolistOverlay(response) 209 if err != nil { 210 return nil, err 211 } 212 213 var containsCandidates []string 214 if len(containFiles) > 0 { 215 containsCandidates = append(containsCandidates, modifiedPkgs...) 216 containsCandidates = append(containsCandidates, needPkgs...) 217 } 218 if err := state.addNeededOverlayPackages(response, needPkgs); err != nil { 219 return nil, err 220 } 221 // Check candidate packages for containFiles. 222 if len(containFiles) > 0 { 223 for _, id := range containsCandidates { 224 pkg, ok := response.seenPackages[id] 225 if !ok { 226 response.addPackage(&Package{ 227 ID: id, 228 Errors: []Error{ 229 { 230 Kind: ListError, 231 Msg: fmt.Sprintf("package %s expected but not seen", id), 232 }, 233 }, 234 }) 235 continue 236 } 237 for _, f := range containFiles { 238 for _, g := range pkg.GoFiles { 239 if sameFile(f, g) { 240 response.addRoot(id) 241 } 242 } 243 } 244 } 245 } 246 247 sizeswg.Wait() 248 if sizeserr != nil { 249 return nil, sizeserr 250 } 251 return response.dr, nil 252} 253 254func (state *golistState) addNeededOverlayPackages(response *responseDeduper, pkgs []string) error { 255 if len(pkgs) == 0 { 256 return nil 257 } 258 dr, err := state.createDriverResponse(pkgs...) 259 if err != nil { 260 return err 261 } 262 for _, pkg := range dr.Packages { 263 response.addPackage(pkg) 264 } 265 _, needPkgs, err := state.processGolistOverlay(response) 266 if err != nil { 267 return err 268 } 269 return state.addNeededOverlayPackages(response, needPkgs) 270} 271 272func (state *golistState) runContainsQueries(response *responseDeduper, queries []string) error { 273 for _, query := range queries { 274 // TODO(matloob): Do only one query per directory. 275 fdir := filepath.Dir(query) 276 // Pass absolute path of directory to go list so that it knows to treat it as a directory, 277 // not a package path. 278 pattern, err := filepath.Abs(fdir) 279 if err != nil { 280 return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err) 281 } 282 dirResponse, err := state.createDriverResponse(pattern) 283 284 // If there was an error loading the package, or the package is returned 285 // with errors, try to load the file as an ad-hoc package. 286 // Usually the error will appear in a returned package, but may not if we're 287 // in module mode and the ad-hoc is located outside a module. 288 if err != nil || len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].GoFiles) == 0 && 289 len(dirResponse.Packages[0].Errors) == 1 { 290 var queryErr error 291 if dirResponse, queryErr = state.adhocPackage(pattern, query); queryErr != nil { 292 return err // return the original error 293 } 294 } 295 isRoot := make(map[string]bool, len(dirResponse.Roots)) 296 for _, root := range dirResponse.Roots { 297 isRoot[root] = true 298 } 299 for _, pkg := range dirResponse.Packages { 300 // Add any new packages to the main set 301 // We don't bother to filter packages that will be dropped by the changes of roots, 302 // that will happen anyway during graph construction outside this function. 303 // Over-reporting packages is not a problem. 304 response.addPackage(pkg) 305 // if the package was not a root one, it cannot have the file 306 if !isRoot[pkg.ID] { 307 continue 308 } 309 for _, pkgFile := range pkg.GoFiles { 310 if filepath.Base(query) == filepath.Base(pkgFile) { 311 response.addRoot(pkg.ID) 312 break 313 } 314 } 315 } 316 } 317 return nil 318} 319 320// adhocPackage attempts to load or construct an ad-hoc package for a given 321// query, if the original call to the driver produced inadequate results. 322func (state *golistState) adhocPackage(pattern, query string) (*driverResponse, error) { 323 response, err := state.createDriverResponse(query) 324 if err != nil { 325 return nil, err 326 } 327 // If we get nothing back from `go list`, 328 // try to make this file into its own ad-hoc package. 329 // TODO(rstambler): Should this check against the original response? 330 if len(response.Packages) == 0 { 331 response.Packages = append(response.Packages, &Package{ 332 ID: "command-line-arguments", 333 PkgPath: query, 334 GoFiles: []string{query}, 335 CompiledGoFiles: []string{query}, 336 Imports: make(map[string]*Package), 337 }) 338 response.Roots = append(response.Roots, "command-line-arguments") 339 } 340 // Handle special cases. 341 if len(response.Packages) == 1 { 342 // golang/go#33482: If this is a file= query for ad-hoc packages where 343 // the file only exists on an overlay, and exists outside of a module, 344 // add the file to the package and remove the errors. 345 if response.Packages[0].ID == "command-line-arguments" || 346 filepath.ToSlash(response.Packages[0].PkgPath) == filepath.ToSlash(query) { 347 if len(response.Packages[0].GoFiles) == 0 { 348 filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath 349 // TODO(matloob): check if the file is outside of a root dir? 350 for path := range state.cfg.Overlay { 351 if path == filename { 352 response.Packages[0].Errors = nil 353 response.Packages[0].GoFiles = []string{path} 354 response.Packages[0].CompiledGoFiles = []string{path} 355 } 356 } 357 } 358 } 359 } 360 return response, nil 361} 362 363// Fields must match go list; 364// see $GOROOT/src/cmd/go/internal/load/pkg.go. 365type jsonPackage struct { 366 ImportPath string 367 Dir string 368 Name string 369 Export string 370 GoFiles []string 371 CompiledGoFiles []string 372 CFiles []string 373 CgoFiles []string 374 CXXFiles []string 375 MFiles []string 376 HFiles []string 377 FFiles []string 378 SFiles []string 379 SwigFiles []string 380 SwigCXXFiles []string 381 SysoFiles []string 382 Imports []string 383 ImportMap map[string]string 384 Deps []string 385 Module *packagesinternal.Module 386 TestGoFiles []string 387 TestImports []string 388 XTestGoFiles []string 389 XTestImports []string 390 ForTest string // q in a "p [q.test]" package, else "" 391 DepOnly bool 392 393 Error *jsonPackageError 394} 395 396type jsonPackageError struct { 397 ImportStack []string 398 Pos string 399 Err string 400} 401 402func otherFiles(p *jsonPackage) [][]string { 403 return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles} 404} 405 406// createDriverResponse uses the "go list" command to expand the pattern 407// words and return a response for the specified packages. 408func (state *golistState) createDriverResponse(words ...string) (*driverResponse, error) { 409 // go list uses the following identifiers in ImportPath and Imports: 410 // 411 // "p" -- importable package or main (command) 412 // "q.test" -- q's test executable 413 // "p [q.test]" -- variant of p as built for q's test executable 414 // "q_test [q.test]" -- q's external test package 415 // 416 // The packages p that are built differently for a test q.test 417 // are q itself, plus any helpers used by the external test q_test, 418 // typically including "testing" and all its dependencies. 419 420 // Run "go list" for complete 421 // information on the specified packages. 422 buf, err := state.invokeGo("list", golistargs(state.cfg, words)...) 423 if err != nil { 424 return nil, err 425 } 426 seen := make(map[string]*jsonPackage) 427 pkgs := make(map[string]*Package) 428 additionalErrors := make(map[string][]Error) 429 // Decode the JSON and convert it to Package form. 430 var response driverResponse 431 for dec := json.NewDecoder(buf); dec.More(); { 432 p := new(jsonPackage) 433 if err := dec.Decode(p); err != nil { 434 return nil, fmt.Errorf("JSON decoding failed: %v", err) 435 } 436 437 if p.ImportPath == "" { 438 // The documentation for go list says that “[e]rroneous packages will have 439 // a non-empty ImportPath”. If for some reason it comes back empty, we 440 // prefer to error out rather than silently discarding data or handing 441 // back a package without any way to refer to it. 442 if p.Error != nil { 443 return nil, Error{ 444 Pos: p.Error.Pos, 445 Msg: p.Error.Err, 446 } 447 } 448 return nil, fmt.Errorf("package missing import path: %+v", p) 449 } 450 451 // Work around https://github.com/ooni/psiphon/oopsi/golang.org/issue/33157: 452 // go list -e, when given an absolute path, will find the package contained at 453 // that directory. But when no package exists there, it will return a fake package 454 // with an error and the ImportPath set to the absolute path provided to go list. 455 // Try to convert that absolute path to what its package path would be if it's 456 // contained in a known module or GOPATH entry. This will allow the package to be 457 // properly "reclaimed" when overlays are processed. 458 if filepath.IsAbs(p.ImportPath) && p.Error != nil { 459 pkgPath, ok, err := state.getPkgPath(p.ImportPath) 460 if err != nil { 461 return nil, err 462 } 463 if ok { 464 p.ImportPath = pkgPath 465 } 466 } 467 468 if old, found := seen[p.ImportPath]; found { 469 // If one version of the package has an error, and the other doesn't, assume 470 // that this is a case where go list is reporting a fake dependency variant 471 // of the imported package: When a package tries to invalidly import another 472 // package, go list emits a variant of the imported package (with the same 473 // import path, but with an error on it, and the package will have a 474 // DepError set on it). An example of when this can happen is for imports of 475 // main packages: main packages can not be imported, but they may be 476 // separately matched and listed by another pattern. 477 // See github.com/ooni/psiphon/oopsi/golang.org/issue/36188 for more details. 478 479 // The plan is that eventually, hopefully in Go 1.15, the error will be 480 // reported on the importing package rather than the duplicate "fake" 481 // version of the imported package. Once all supported versions of Go 482 // have the new behavior this logic can be deleted. 483 // TODO(matloob): delete the workaround logic once all supported versions of 484 // Go return the errors on the proper package. 485 486 // There should be exactly one version of a package that doesn't have an 487 // error. 488 if old.Error == nil && p.Error == nil { 489 if !reflect.DeepEqual(p, old) { 490 return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath) 491 } 492 continue 493 } 494 495 // Determine if this package's error needs to be bubbled up. 496 // This is a hack, and we expect for go list to eventually set the error 497 // on the package. 498 if old.Error != nil { 499 var errkind string 500 if strings.Contains(old.Error.Err, "not an importable package") { 501 errkind = "not an importable package" 502 } else if strings.Contains(old.Error.Err, "use of internal package") && strings.Contains(old.Error.Err, "not allowed") { 503 errkind = "use of internal package not allowed" 504 } 505 if errkind != "" { 506 if len(old.Error.ImportStack) < 1 { 507 return nil, fmt.Errorf(`internal error: go list gave a %q error with empty import stack`, errkind) 508 } 509 importingPkg := old.Error.ImportStack[len(old.Error.ImportStack)-1] 510 if importingPkg == old.ImportPath { 511 // Using an older version of Go which put this package itself on top of import 512 // stack, instead of the importer. Look for importer in second from top 513 // position. 514 if len(old.Error.ImportStack) < 2 { 515 return nil, fmt.Errorf(`internal error: go list gave a %q error with an import stack without importing package`, errkind) 516 } 517 importingPkg = old.Error.ImportStack[len(old.Error.ImportStack)-2] 518 } 519 additionalErrors[importingPkg] = append(additionalErrors[importingPkg], Error{ 520 Pos: old.Error.Pos, 521 Msg: old.Error.Err, 522 Kind: ListError, 523 }) 524 } 525 } 526 527 // Make sure that if there's a version of the package without an error, 528 // that's the one reported to the user. 529 if old.Error == nil { 530 continue 531 } 532 533 // This package will replace the old one at the end of the loop. 534 } 535 seen[p.ImportPath] = p 536 537 pkg := &Package{ 538 Name: p.Name, 539 ID: p.ImportPath, 540 GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), 541 CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles), 542 OtherFiles: absJoin(p.Dir, otherFiles(p)...), 543 forTest: p.ForTest, 544 module: p.Module, 545 } 546 547 // Work around https://github.com/ooni/psiphon/oopsi/golang.org/issue/28749: 548 // cmd/go puts assembly, C, and C++ files in CompiledGoFiles. 549 // Filter out any elements of CompiledGoFiles that are also in OtherFiles. 550 // We have to keep this workaround in place until go1.12 is a distant memory. 551 if len(pkg.OtherFiles) > 0 { 552 other := make(map[string]bool, len(pkg.OtherFiles)) 553 for _, f := range pkg.OtherFiles { 554 other[f] = true 555 } 556 557 out := pkg.CompiledGoFiles[:0] 558 for _, f := range pkg.CompiledGoFiles { 559 if other[f] { 560 continue 561 } 562 out = append(out, f) 563 } 564 pkg.CompiledGoFiles = out 565 } 566 567 // Extract the PkgPath from the package's ID. 568 if i := strings.IndexByte(pkg.ID, ' '); i >= 0 { 569 pkg.PkgPath = pkg.ID[:i] 570 } else { 571 pkg.PkgPath = pkg.ID 572 } 573 574 if pkg.PkgPath == "unsafe" { 575 pkg.GoFiles = nil // ignore fake unsafe.go file 576 } 577 578 // Assume go list emits only absolute paths for Dir. 579 if p.Dir != "" && !filepath.IsAbs(p.Dir) { 580 log.Fatalf("internal error: go list returned non-absolute Package.Dir: %s", p.Dir) 581 } 582 583 if p.Export != "" && !filepath.IsAbs(p.Export) { 584 pkg.ExportFile = filepath.Join(p.Dir, p.Export) 585 } else { 586 pkg.ExportFile = p.Export 587 } 588 589 // imports 590 // 591 // Imports contains the IDs of all imported packages. 592 // ImportsMap records (path, ID) only where they differ. 593 ids := make(map[string]bool) 594 for _, id := range p.Imports { 595 ids[id] = true 596 } 597 pkg.Imports = make(map[string]*Package) 598 for path, id := range p.ImportMap { 599 pkg.Imports[path] = &Package{ID: id} // non-identity import 600 delete(ids, id) 601 } 602 for id := range ids { 603 if id == "C" { 604 continue 605 } 606 607 pkg.Imports[id] = &Package{ID: id} // identity import 608 } 609 if !p.DepOnly { 610 response.Roots = append(response.Roots, pkg.ID) 611 } 612 613 // Work around for pre-go.1.11 versions of go list. 614 // TODO(matloob): they should be handled by the fallback. 615 // Can we delete this? 616 if len(pkg.CompiledGoFiles) == 0 { 617 pkg.CompiledGoFiles = pkg.GoFiles 618 } 619 620 if p.Error != nil { 621 msg := strings.TrimSpace(p.Error.Err) // Trim to work around github.com/ooni/psiphon/oopsi/golang.org/issue/32363. 622 // Address github.com/ooni/psiphon/oopsi/golang.org/issue/35964 by appending import stack to error message. 623 if msg == "import cycle not allowed" && len(p.Error.ImportStack) != 0 { 624 msg += fmt.Sprintf(": import stack: %v", p.Error.ImportStack) 625 } 626 pkg.Errors = append(pkg.Errors, Error{ 627 Pos: p.Error.Pos, 628 Msg: msg, 629 Kind: ListError, 630 }) 631 } 632 633 pkgs[pkg.ID] = pkg 634 } 635 636 for id, errs := range additionalErrors { 637 if p, ok := pkgs[id]; ok { 638 p.Errors = append(p.Errors, errs...) 639 } 640 } 641 for _, pkg := range pkgs { 642 response.Packages = append(response.Packages, pkg) 643 } 644 sort.Slice(response.Packages, func(i, j int) bool { return response.Packages[i].ID < response.Packages[j].ID }) 645 646 return &response, nil 647} 648 649// getPkgPath finds the package path of a directory if it's relative to a root directory. 650func (state *golistState) getPkgPath(dir string) (string, bool, error) { 651 absDir, err := filepath.Abs(dir) 652 if err != nil { 653 return "", false, err 654 } 655 roots, err := state.determineRootDirs() 656 if err != nil { 657 return "", false, err 658 } 659 660 for rdir, rpath := range roots { 661 // Make sure that the directory is in the module, 662 // to avoid creating a path relative to another module. 663 if !strings.HasPrefix(absDir, rdir) { 664 continue 665 } 666 // TODO(matloob): This doesn't properly handle symlinks. 667 r, err := filepath.Rel(rdir, dir) 668 if err != nil { 669 continue 670 } 671 if rpath != "" { 672 // We choose only one root even though the directory even it can belong in multiple modules 673 // or GOPATH entries. This is okay because we only need to work with absolute dirs when a 674 // file is missing from disk, for instance when gopls calls go/packages in an overlay. 675 // Once the file is saved, gopls, or the next invocation of the tool will get the correct 676 // result straight from golist. 677 // TODO(matloob): Implement module tiebreaking? 678 return path.Join(rpath, filepath.ToSlash(r)), true, nil 679 } 680 return filepath.ToSlash(r), true, nil 681 } 682 return "", false, nil 683} 684 685// absJoin absolutizes and flattens the lists of files. 686func absJoin(dir string, fileses ...[]string) (res []string) { 687 for _, files := range fileses { 688 for _, file := range files { 689 if !filepath.IsAbs(file) { 690 file = filepath.Join(dir, file) 691 } 692 res = append(res, file) 693 } 694 } 695 return res 696} 697 698func golistargs(cfg *Config, words []string) []string { 699 const findFlags = NeedImports | NeedTypes | NeedSyntax | NeedTypesInfo 700 fullargs := []string{ 701 "-e", "-json", 702 fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypes|NeedTypesInfo|NeedTypesSizes) != 0), 703 fmt.Sprintf("-test=%t", cfg.Tests), 704 fmt.Sprintf("-export=%t", usesExportData(cfg)), 705 fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0), 706 // go list doesn't let you pass -test and -find together, 707 // probably because you'd just get the TestMain. 708 fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0), 709 } 710 fullargs = append(fullargs, cfg.BuildFlags...) 711 fullargs = append(fullargs, "--") 712 fullargs = append(fullargs, words...) 713 return fullargs 714} 715 716// invokeGo returns the stdout of a go command invocation. 717func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, error) { 718 cfg := state.cfg 719 720 inv := gocommand.Invocation{ 721 Verb: verb, 722 Args: args, 723 BuildFlags: cfg.BuildFlags, 724 Env: cfg.Env, 725 Logf: cfg.Logf, 726 WorkingDir: cfg.Dir, 727 } 728 gocmdRunner := cfg.gocmdRunner 729 if gocmdRunner == nil { 730 gocmdRunner = &gocommand.Runner{} 731 } 732 stdout, stderr, _, err := gocmdRunner.RunRaw(cfg.Context, inv) 733 if err != nil { 734 // Check for 'go' executable not being found. 735 if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { 736 return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound) 737 } 738 739 exitErr, ok := err.(*exec.ExitError) 740 if !ok { 741 // Catastrophic error: 742 // - context cancellation 743 return nil, xerrors.Errorf("couldn't run 'go': %w", err) 744 } 745 746 // Old go version? 747 if strings.Contains(stderr.String(), "flag provided but not defined") { 748 return nil, goTooOldError{fmt.Errorf("unsupported version of go: %s: %s", exitErr, stderr)} 749 } 750 751 // Related to #24854 752 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "unexpected directory layout") { 753 return nil, fmt.Errorf("%s", stderr.String()) 754 } 755 756 // Is there an error running the C compiler in cgo? This will be reported in the "Error" field 757 // and should be suppressed by go list -e. 758 // 759 // This condition is not perfect yet because the error message can include other error messages than runtime/cgo. 760 isPkgPathRune := func(r rune) bool { 761 // From https://github.com/ooni/psiphon/oopsi/golang.org/ref/spec#Import_declarations: 762 // Implementation restriction: A compiler may restrict ImportPaths to non-empty strings 763 // using only characters belonging to Unicode's L, M, N, P, and S general categories 764 // (the Graphic characters without spaces) and may also exclude the 765 // characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD. 766 return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) && 767 !strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r) 768 } 769 if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") { 770 msg := stderr.String()[len("# "):] 771 if strings.HasPrefix(strings.TrimLeftFunc(msg, isPkgPathRune), "\n") { 772 return stdout, nil 773 } 774 // Treat pkg-config errors as a special case (github.com/ooni/psiphon/oopsi/golang.org/issue/36770). 775 if strings.HasPrefix(msg, "pkg-config") { 776 return stdout, nil 777 } 778 } 779 780 // This error only appears in stderr. See github.com/ooni/psiphon/oopsi/golang.org/cl/166398 for a fix in go list to show 781 // the error in the Err section of stdout in case -e option is provided. 782 // This fix is provided for backwards compatibility. 783 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must be .go files") { 784 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 785 strings.Trim(stderr.String(), "\n")) 786 return bytes.NewBufferString(output), nil 787 } 788 789 // Similar to the previous error, but currently lacks a fix in Go. 790 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must all be in one directory") { 791 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 792 strings.Trim(stderr.String(), "\n")) 793 return bytes.NewBufferString(output), nil 794 } 795 796 // Backwards compatibility for Go 1.11 because 1.12 and 1.13 put the directory in the ImportPath. 797 // If the package doesn't exist, put the absolute path of the directory into the error message, 798 // as Go 1.13 list does. 799 const noSuchDirectory = "no such directory" 800 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), noSuchDirectory) { 801 errstr := stderr.String() 802 abspath := strings.TrimSpace(errstr[strings.Index(errstr, noSuchDirectory)+len(noSuchDirectory):]) 803 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 804 abspath, strings.Trim(stderr.String(), "\n")) 805 return bytes.NewBufferString(output), nil 806 } 807 808 // Workaround for #29280: go list -e has incorrect behavior when an ad-hoc package doesn't exist. 809 // Note that the error message we look for in this case is different that the one looked for above. 810 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no such file or directory") { 811 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 812 strings.Trim(stderr.String(), "\n")) 813 return bytes.NewBufferString(output), nil 814 } 815 816 // Workaround for #34273. go list -e with GO111MODULE=on has incorrect behavior when listing a 817 // directory outside any module. 818 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside available modules") { 819 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 820 // TODO(matloob): command-line-arguments isn't correct here. 821 "command-line-arguments", strings.Trim(stderr.String(), "\n")) 822 return bytes.NewBufferString(output), nil 823 } 824 825 // Another variation of the previous error 826 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") { 827 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 828 // TODO(matloob): command-line-arguments isn't correct here. 829 "command-line-arguments", strings.Trim(stderr.String(), "\n")) 830 return bytes.NewBufferString(output), nil 831 } 832 833 // Workaround for an instance of github.com/ooni/psiphon/oopsi/golang.org/issue/26755: go list -e will return a non-zero exit 834 // status if there's a dependency on a package that doesn't exist. But it should return 835 // a zero exit status and set an error on that package. 836 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") { 837 // Don't clobber stdout if `go list` actually returned something. 838 if len(stdout.String()) > 0 { 839 return stdout, nil 840 } 841 // try to extract package name from string 842 stderrStr := stderr.String() 843 var importPath string 844 colon := strings.Index(stderrStr, ":") 845 if colon > 0 && strings.HasPrefix(stderrStr, "go build ") { 846 importPath = stderrStr[len("go build "):colon] 847 } 848 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 849 importPath, strings.Trim(stderrStr, "\n")) 850 return bytes.NewBufferString(output), nil 851 } 852 853 // Export mode entails a build. 854 // If that build fails, errors appear on stderr 855 // (despite the -e flag) and the Export field is blank. 856 // Do not fail in that case. 857 // The same is true if an ad-hoc package given to go list doesn't exist. 858 // TODO(matloob): Remove these once we can depend on go list to exit with a zero status with -e even when 859 // packages don't exist or a build fails. 860 if !usesExportData(cfg) && !containsGoFile(args) { 861 return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr) 862 } 863 } 864 return stdout, nil 865} 866 867func containsGoFile(s []string) bool { 868 for _, f := range s { 869 if strings.HasSuffix(f, ".go") { 870 return true 871 } 872 } 873 return false 874} 875 876func cmdDebugStr(cmd *exec.Cmd, args ...string) string { 877 env := make(map[string]string) 878 for _, kv := range cmd.Env { 879 split := strings.Split(kv, "=") 880 k, v := split[0], split[1] 881 env[k] = v 882 } 883 var quotedArgs []string 884 for _, arg := range args { 885 quotedArgs = append(quotedArgs, strconv.Quote(arg)) 886 } 887 888 return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v PWD=%v go %s", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["PWD"], strings.Join(quotedArgs, " ")) 889} 890