1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package packages 6 7import ( 8 "bytes" 9 "encoding/json" 10 "fmt" 11 "go/types" 12 "io/ioutil" 13 "log" 14 "os" 15 "os/exec" 16 "path" 17 "path/filepath" 18 "reflect" 19 "regexp" 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24 "unicode" 25 26 "golang.org/x/tools/go/internal/packagesdriver" 27 "golang.org/x/tools/internal/gopathwalk" 28 "golang.org/x/tools/internal/semver" 29 "golang.org/x/tools/internal/span" 30) 31 32// debug controls verbose logging. 33var debug, _ = strconv.ParseBool(os.Getenv("GOPACKAGESDEBUG")) 34 35// A goTooOldError reports that the go command 36// found by exec.LookPath is too old to use the new go list behavior. 37type goTooOldError struct { 38 error 39} 40 41// responseDeduper wraps a driverResponse, deduplicating its contents. 42type responseDeduper struct { 43 seenRoots map[string]bool 44 seenPackages map[string]*Package 45 dr *driverResponse 46} 47 48// init fills in r with a driverResponse. 49func (r *responseDeduper) init(dr *driverResponse) { 50 r.dr = dr 51 r.seenRoots = map[string]bool{} 52 r.seenPackages = map[string]*Package{} 53 for _, pkg := range dr.Packages { 54 r.seenPackages[pkg.ID] = pkg 55 } 56 for _, root := range dr.Roots { 57 r.seenRoots[root] = true 58 } 59} 60 61func (r *responseDeduper) addPackage(p *Package) { 62 if r.seenPackages[p.ID] != nil { 63 return 64 } 65 r.seenPackages[p.ID] = p 66 r.dr.Packages = append(r.dr.Packages, p) 67} 68 69func (r *responseDeduper) addRoot(id string) { 70 if r.seenRoots[id] { 71 return 72 } 73 r.seenRoots[id] = true 74 r.dr.Roots = append(r.dr.Roots, id) 75} 76 77// goInfo contains global information from the go tool. 78type goInfo struct { 79 rootDirs map[string]string 80 env goEnv 81} 82 83type goEnv struct { 84 modulesOn bool 85} 86 87func determineEnv(cfg *Config) goEnv { 88 buf, err := invokeGo(cfg, "env", "GOMOD") 89 if err != nil { 90 return goEnv{} 91 } 92 gomod := bytes.TrimSpace(buf.Bytes()) 93 94 env := goEnv{} 95 env.modulesOn = len(gomod) > 0 96 return env 97} 98 99// goListDriver uses the go list command to interpret the patterns and produce 100// the build system package structure. 101// See driver for more details. 102func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) { 103 var sizes types.Sizes 104 var sizeserr error 105 var sizeswg sync.WaitGroup 106 if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 { 107 sizeswg.Add(1) 108 go func() { 109 sizes, sizeserr = getSizes(cfg) 110 sizeswg.Done() 111 }() 112 } 113 defer sizeswg.Wait() 114 115 // start fetching rootDirs 116 var info goInfo 117 var rootDirsReady, envReady = make(chan struct{}), make(chan struct{}) 118 go func() { 119 info.rootDirs = determineRootDirs(cfg) 120 close(rootDirsReady) 121 }() 122 go func() { 123 info.env = determineEnv(cfg) 124 close(envReady) 125 }() 126 getGoInfo := func() *goInfo { 127 <-rootDirsReady 128 <-envReady 129 return &info 130 } 131 132 // Ensure that we don't leak goroutines: Load is synchronous, so callers will 133 // not expect it to access the fields of cfg after the call returns. 134 defer getGoInfo() 135 136 // always pass getGoInfo to golistDriver 137 golistDriver := func(cfg *Config, patterns ...string) (*driverResponse, error) { 138 return golistDriver(cfg, getGoInfo, patterns...) 139 } 140 141 // Determine files requested in contains patterns 142 var containFiles []string 143 var packagesNamed []string 144 restPatterns := make([]string, 0, len(patterns)) 145 // Extract file= and other [querytype]= patterns. Report an error if querytype 146 // doesn't exist. 147extractQueries: 148 for _, pattern := range patterns { 149 eqidx := strings.Index(pattern, "=") 150 if eqidx < 0 { 151 restPatterns = append(restPatterns, pattern) 152 } else { 153 query, value := pattern[:eqidx], pattern[eqidx+len("="):] 154 switch query { 155 case "file": 156 containFiles = append(containFiles, value) 157 case "pattern": 158 restPatterns = append(restPatterns, value) 159 case "iamashamedtousethedisabledqueryname": 160 packagesNamed = append(packagesNamed, value) 161 case "": // not a reserved query 162 restPatterns = append(restPatterns, pattern) 163 default: 164 for _, rune := range query { 165 if rune < 'a' || rune > 'z' { // not a reserved query 166 restPatterns = append(restPatterns, pattern) 167 continue extractQueries 168 } 169 } 170 // Reject all other patterns containing "=" 171 return nil, fmt.Errorf("invalid query type %q in query pattern %q", query, pattern) 172 } 173 } 174 } 175 176 response := &responseDeduper{} 177 var err error 178 179 // See if we have any patterns to pass through to go list. Zero initial 180 // patterns also requires a go list call, since it's the equivalent of 181 // ".". 182 if len(restPatterns) > 0 || len(patterns) == 0 { 183 dr, err := golistDriver(cfg, restPatterns...) 184 if err != nil { 185 return nil, err 186 } 187 response.init(dr) 188 } else { 189 response.init(&driverResponse{}) 190 } 191 192 sizeswg.Wait() 193 if sizeserr != nil { 194 return nil, sizeserr 195 } 196 // types.SizesFor always returns nil or a *types.StdSizes 197 response.dr.Sizes, _ = sizes.(*types.StdSizes) 198 199 var containsCandidates []string 200 201 if len(containFiles) != 0 { 202 if err := runContainsQueries(cfg, golistDriver, response, containFiles, getGoInfo); err != nil { 203 return nil, err 204 } 205 } 206 207 if len(packagesNamed) != 0 { 208 if err := runNamedQueries(cfg, golistDriver, response, packagesNamed); err != nil { 209 return nil, err 210 } 211 } 212 213 modifiedPkgs, needPkgs, err := processGolistOverlay(cfg, response, getGoInfo) 214 if err != nil { 215 return nil, err 216 } 217 if len(containFiles) > 0 { 218 containsCandidates = append(containsCandidates, modifiedPkgs...) 219 containsCandidates = append(containsCandidates, needPkgs...) 220 } 221 if err := addNeededOverlayPackages(cfg, golistDriver, response, needPkgs, getGoInfo); err != nil { 222 return nil, err 223 } 224 // Check candidate packages for containFiles. 225 if len(containFiles) > 0 { 226 for _, id := range containsCandidates { 227 pkg, ok := response.seenPackages[id] 228 if !ok { 229 response.addPackage(&Package{ 230 ID: id, 231 Errors: []Error{ 232 { 233 Kind: ListError, 234 Msg: fmt.Sprintf("package %s expected but not seen", id), 235 }, 236 }, 237 }) 238 continue 239 } 240 for _, f := range containFiles { 241 for _, g := range pkg.GoFiles { 242 if sameFile(f, g) { 243 response.addRoot(id) 244 } 245 } 246 } 247 } 248 } 249 250 return response.dr, nil 251} 252 253func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDeduper, pkgs []string, getGoInfo func() *goInfo) error { 254 if len(pkgs) == 0 { 255 return nil 256 } 257 drivercfg := *cfg 258 if getGoInfo().env.modulesOn { 259 drivercfg.BuildFlags = append(drivercfg.BuildFlags, "-mod=readonly") 260 } 261 dr, err := driver(&drivercfg, pkgs...) 262 263 if err != nil { 264 return err 265 } 266 for _, pkg := range dr.Packages { 267 response.addPackage(pkg) 268 } 269 _, needPkgs, err := processGolistOverlay(cfg, response, getGoInfo) 270 if err != nil { 271 return err 272 } 273 return addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo) 274} 275 276func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, queries []string, goInfo func() *goInfo) error { 277 for _, query := range queries { 278 // TODO(matloob): Do only one query per directory. 279 fdir := filepath.Dir(query) 280 // Pass absolute path of directory to go list so that it knows to treat it as a directory, 281 // not a package path. 282 pattern, err := filepath.Abs(fdir) 283 if err != nil { 284 return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err) 285 } 286 dirResponse, err := driver(cfg, pattern) 287 if err != nil { 288 var queryErr error 289 if dirResponse, queryErr = adHocPackage(cfg, driver, pattern, query); queryErr != nil { 290 return err // return the original error 291 } 292 } 293 // `go list` can report errors for files that are not listed as part of a package's GoFiles. 294 // In the case of an invalid Go file, we should assume that it is part of package if only 295 // one package is in the response. The file may have valid contents in an overlay. 296 if len(dirResponse.Packages) == 1 { 297 pkg := dirResponse.Packages[0] 298 for i, err := range pkg.Errors { 299 s := errorSpan(err) 300 if !s.IsValid() { 301 break 302 } 303 if len(pkg.CompiledGoFiles) == 0 { 304 break 305 } 306 dir := filepath.Dir(pkg.CompiledGoFiles[0]) 307 filename := filepath.Join(dir, filepath.Base(s.URI().Filename())) 308 if info, err := os.Stat(filename); err != nil || info.IsDir() { 309 break 310 } 311 if !contains(pkg.CompiledGoFiles, filename) { 312 pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, filename) 313 pkg.GoFiles = append(pkg.GoFiles, filename) 314 pkg.Errors = append(pkg.Errors[:i], pkg.Errors[i+1:]...) 315 } 316 } 317 } 318 // A final attempt to construct an ad-hoc package. 319 if len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].Errors) == 1 { 320 var queryErr error 321 if dirResponse, queryErr = adHocPackage(cfg, driver, pattern, query); queryErr != nil { 322 return err // return the original error 323 } 324 } 325 isRoot := make(map[string]bool, len(dirResponse.Roots)) 326 for _, root := range dirResponse.Roots { 327 isRoot[root] = true 328 } 329 for _, pkg := range dirResponse.Packages { 330 // Add any new packages to the main set 331 // We don't bother to filter packages that will be dropped by the changes of roots, 332 // that will happen anyway during graph construction outside this function. 333 // Over-reporting packages is not a problem. 334 response.addPackage(pkg) 335 // if the package was not a root one, it cannot have the file 336 if !isRoot[pkg.ID] { 337 continue 338 } 339 for _, pkgFile := range pkg.GoFiles { 340 if filepath.Base(query) == filepath.Base(pkgFile) { 341 response.addRoot(pkg.ID) 342 break 343 } 344 } 345 } 346 } 347 return nil 348} 349 350// adHocPackage attempts to construct an ad-hoc package given a query that failed. 351func adHocPackage(cfg *Config, driver driver, pattern, query string) (*driverResponse, error) { 352 // There was an error loading the package. Try to load the file as an ad-hoc package. 353 // Usually the error will appear in a returned package, but may not if we're in modules mode 354 // and the ad-hoc is located outside a module. 355 dirResponse, err := driver(cfg, query) 356 if err != nil { 357 return nil, err 358 } 359 // If we get nothing back from `go list`, try to make this file into its own ad-hoc package. 360 if len(dirResponse.Packages) == 0 && err == nil { 361 dirResponse.Packages = append(dirResponse.Packages, &Package{ 362 ID: "command-line-arguments", 363 PkgPath: query, 364 GoFiles: []string{query}, 365 CompiledGoFiles: []string{query}, 366 Imports: make(map[string]*Package), 367 }) 368 dirResponse.Roots = append(dirResponse.Roots, "command-line-arguments") 369 } 370 // Special case to handle issue #33482: 371 // If this is a file= query for ad-hoc packages where the file only exists on an overlay, 372 // and exists outside of a module, add the file in for the package. 373 if len(dirResponse.Packages) == 1 && (dirResponse.Packages[0].ID == "command-line-arguments" || 374 filepath.ToSlash(dirResponse.Packages[0].PkgPath) == filepath.ToSlash(query)) { 375 if len(dirResponse.Packages[0].GoFiles) == 0 { 376 filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath 377 // TODO(matloob): check if the file is outside of a root dir? 378 for path := range cfg.Overlay { 379 if path == filename { 380 dirResponse.Packages[0].Errors = nil 381 dirResponse.Packages[0].GoFiles = []string{path} 382 dirResponse.Packages[0].CompiledGoFiles = []string{path} 383 } 384 } 385 } 386 } 387 return dirResponse, nil 388} 389 390func contains(files []string, filename string) bool { 391 for _, f := range files { 392 if f == filename { 393 return true 394 } 395 } 396 return false 397} 398 399// errorSpan attempts to parse a standard `go list` error message 400// by stripping off the trailing error message. 401// 402// It works only on errors whose message is prefixed by colon, 403// followed by a space (": "). For example: 404// 405// attributes.go:13:1: expected 'package', found 'type' 406// 407func errorSpan(err Error) span.Span { 408 if err.Pos == "" { 409 input := strings.TrimSpace(err.Msg) 410 msgIndex := strings.Index(input, ": ") 411 if msgIndex < 0 { 412 return span.Parse(input) 413 } 414 return span.Parse(input[:msgIndex]) 415 } 416 return span.Parse(err.Pos) 417} 418 419// modCacheRegexp splits a path in a module cache into module, module version, and package. 420var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`) 421 422func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, queries []string) error { 423 // calling `go env` isn't free; bail out if there's nothing to do. 424 if len(queries) == 0 { 425 return nil 426 } 427 // Determine which directories are relevant to scan. 428 roots, modRoot, err := roots(cfg) 429 if err != nil { 430 return err 431 } 432 433 // Scan the selected directories. Simple matches, from GOPATH/GOROOT 434 // or the local module, can simply be "go list"ed. Matches from the 435 // module cache need special treatment. 436 var matchesMu sync.Mutex 437 var simpleMatches, modCacheMatches []string 438 add := func(root gopathwalk.Root, dir string) { 439 // Walk calls this concurrently; protect the result slices. 440 matchesMu.Lock() 441 defer matchesMu.Unlock() 442 443 path := dir 444 if dir != root.Path { 445 path = dir[len(root.Path)+1:] 446 } 447 if pathMatchesQueries(path, queries) { 448 switch root.Type { 449 case gopathwalk.RootModuleCache: 450 modCacheMatches = append(modCacheMatches, path) 451 case gopathwalk.RootCurrentModule: 452 // We'd need to read go.mod to find the full 453 // import path. Relative's easier. 454 rel, err := filepath.Rel(cfg.Dir, dir) 455 if err != nil { 456 // This ought to be impossible, since 457 // we found dir in the current module. 458 panic(err) 459 } 460 simpleMatches = append(simpleMatches, "./"+rel) 461 case gopathwalk.RootGOPATH, gopathwalk.RootGOROOT: 462 simpleMatches = append(simpleMatches, path) 463 } 464 } 465 } 466 467 startWalk := time.Now() 468 gopathwalk.Walk(roots, add, gopathwalk.Options{ModulesEnabled: modRoot != "", Debug: debug}) 469 cfg.Logf("%v for walk", time.Since(startWalk)) 470 471 // Weird special case: the top-level package in a module will be in 472 // whatever directory the user checked the repository out into. It's 473 // more reasonable for that to not match the package name. So, if there 474 // are any Go files in the mod root, query it just to be safe. 475 if modRoot != "" { 476 rel, err := filepath.Rel(cfg.Dir, modRoot) 477 if err != nil { 478 panic(err) // See above. 479 } 480 481 files, err := ioutil.ReadDir(modRoot) 482 if err != nil { 483 panic(err) // See above. 484 } 485 486 for _, f := range files { 487 if strings.HasSuffix(f.Name(), ".go") { 488 simpleMatches = append(simpleMatches, rel) 489 break 490 } 491 } 492 } 493 494 addResponse := func(r *driverResponse) { 495 for _, pkg := range r.Packages { 496 response.addPackage(pkg) 497 for _, name := range queries { 498 if pkg.Name == name { 499 response.addRoot(pkg.ID) 500 break 501 } 502 } 503 } 504 } 505 506 if len(simpleMatches) != 0 { 507 resp, err := driver(cfg, simpleMatches...) 508 if err != nil { 509 return err 510 } 511 addResponse(resp) 512 } 513 514 // Module cache matches are tricky. We want to avoid downloading new 515 // versions of things, so we need to use the ones present in the cache. 516 // go list doesn't accept version specifiers, so we have to write out a 517 // temporary module, and do the list in that module. 518 if len(modCacheMatches) != 0 { 519 // Collect all the matches, deduplicating by major version 520 // and preferring the newest. 521 type modInfo struct { 522 mod string 523 major string 524 } 525 mods := make(map[modInfo]string) 526 var imports []string 527 for _, modPath := range modCacheMatches { 528 matches := modCacheRegexp.FindStringSubmatch(modPath) 529 mod, ver := filepath.ToSlash(matches[1]), matches[2] 530 importPath := filepath.ToSlash(filepath.Join(matches[1], matches[3])) 531 532 major := semver.Major(ver) 533 if prevVer, ok := mods[modInfo{mod, major}]; !ok || semver.Compare(ver, prevVer) > 0 { 534 mods[modInfo{mod, major}] = ver 535 } 536 537 imports = append(imports, importPath) 538 } 539 540 // Build the temporary module. 541 var gomod bytes.Buffer 542 gomod.WriteString("module modquery\nrequire (\n") 543 for mod, version := range mods { 544 gomod.WriteString("\t" + mod.mod + " " + version + "\n") 545 } 546 gomod.WriteString(")\n") 547 548 tmpCfg := *cfg 549 550 // We're only trying to look at stuff in the module cache, so 551 // disable the network. This should speed things up, and has 552 // prevented errors in at least one case, #28518. 553 tmpCfg.Env = append([]string{"GOPROXY=off"}, cfg.Env...) 554 555 var err error 556 tmpCfg.Dir, err = ioutil.TempDir("", "gopackages-modquery") 557 if err != nil { 558 return err 559 } 560 defer os.RemoveAll(tmpCfg.Dir) 561 562 if err := ioutil.WriteFile(filepath.Join(tmpCfg.Dir, "go.mod"), gomod.Bytes(), 0777); err != nil { 563 return fmt.Errorf("writing go.mod for module cache query: %v", err) 564 } 565 566 // Run the query, using the import paths calculated from the matches above. 567 resp, err := driver(&tmpCfg, imports...) 568 if err != nil { 569 return fmt.Errorf("querying module cache matches: %v", err) 570 } 571 addResponse(resp) 572 } 573 574 return nil 575} 576 577func getSizes(cfg *Config) (types.Sizes, error) { 578 return packagesdriver.GetSizesGolist(cfg.Context, cfg.BuildFlags, cfg.Env, cfg.Dir, usesExportData(cfg)) 579} 580 581// roots selects the appropriate paths to walk based on the passed-in configuration, 582// particularly the environment and the presence of a go.mod in cfg.Dir's parents. 583func roots(cfg *Config) ([]gopathwalk.Root, string, error) { 584 stdout, err := invokeGo(cfg, "env", "GOROOT", "GOPATH", "GOMOD") 585 if err != nil { 586 return nil, "", err 587 } 588 589 fields := strings.Split(stdout.String(), "\n") 590 if len(fields) != 4 || len(fields[3]) != 0 { 591 return nil, "", fmt.Errorf("go env returned unexpected output: %q", stdout.String()) 592 } 593 goroot, gopath, gomod := fields[0], filepath.SplitList(fields[1]), fields[2] 594 var modDir string 595 if gomod != "" { 596 modDir = filepath.Dir(gomod) 597 } 598 599 var roots []gopathwalk.Root 600 // Always add GOROOT. 601 roots = append(roots, gopathwalk.Root{ 602 Path: filepath.Join(goroot, "/src"), 603 Type: gopathwalk.RootGOROOT, 604 }) 605 // If modules are enabled, scan the module dir. 606 if modDir != "" { 607 roots = append(roots, gopathwalk.Root{ 608 Path: modDir, 609 Type: gopathwalk.RootCurrentModule, 610 }) 611 } 612 // Add either GOPATH/src or GOPATH/pkg/mod, depending on module mode. 613 for _, p := range gopath { 614 if modDir != "" { 615 roots = append(roots, gopathwalk.Root{ 616 Path: filepath.Join(p, "/pkg/mod"), 617 Type: gopathwalk.RootModuleCache, 618 }) 619 } else { 620 roots = append(roots, gopathwalk.Root{ 621 Path: filepath.Join(p, "/src"), 622 Type: gopathwalk.RootGOPATH, 623 }) 624 } 625 } 626 627 return roots, modDir, nil 628} 629 630// These functions were copied from goimports. See further documentation there. 631 632// pathMatchesQueries is adapted from pkgIsCandidate. 633// TODO: is it reasonable to do Contains here, rather than an exact match on a path component? 634func pathMatchesQueries(path string, queries []string) bool { 635 lastTwo := lastTwoComponents(path) 636 for _, query := range queries { 637 if strings.Contains(lastTwo, query) { 638 return true 639 } 640 if hasHyphenOrUpperASCII(lastTwo) && !hasHyphenOrUpperASCII(query) { 641 lastTwo = lowerASCIIAndRemoveHyphen(lastTwo) 642 if strings.Contains(lastTwo, query) { 643 return true 644 } 645 } 646 } 647 return false 648} 649 650// lastTwoComponents returns at most the last two path components 651// of v, using either / or \ as the path separator. 652func lastTwoComponents(v string) string { 653 nslash := 0 654 for i := len(v) - 1; i >= 0; i-- { 655 if v[i] == '/' || v[i] == '\\' { 656 nslash++ 657 if nslash == 2 { 658 return v[i:] 659 } 660 } 661 } 662 return v 663} 664 665func hasHyphenOrUpperASCII(s string) bool { 666 for i := 0; i < len(s); i++ { 667 b := s[i] 668 if b == '-' || ('A' <= b && b <= 'Z') { 669 return true 670 } 671 } 672 return false 673} 674 675func lowerASCIIAndRemoveHyphen(s string) (ret string) { 676 buf := make([]byte, 0, len(s)) 677 for i := 0; i < len(s); i++ { 678 b := s[i] 679 switch { 680 case b == '-': 681 continue 682 case 'A' <= b && b <= 'Z': 683 buf = append(buf, b+('a'-'A')) 684 default: 685 buf = append(buf, b) 686 } 687 } 688 return string(buf) 689} 690 691// Fields must match go list; 692// see $GOROOT/src/cmd/go/internal/load/pkg.go. 693type jsonPackage struct { 694 ImportPath string 695 Dir string 696 Name string 697 Export string 698 GoFiles []string 699 CompiledGoFiles []string 700 CFiles []string 701 CgoFiles []string 702 CXXFiles []string 703 MFiles []string 704 HFiles []string 705 FFiles []string 706 SFiles []string 707 SwigFiles []string 708 SwigCXXFiles []string 709 SysoFiles []string 710 Imports []string 711 ImportMap map[string]string 712 Deps []string 713 TestGoFiles []string 714 TestImports []string 715 XTestGoFiles []string 716 XTestImports []string 717 ForTest string // q in a "p [q.test]" package, else "" 718 DepOnly bool 719 720 Error *jsonPackageError 721} 722 723type jsonPackageError struct { 724 ImportStack []string 725 Pos string 726 Err string 727} 728 729func otherFiles(p *jsonPackage) [][]string { 730 return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles} 731} 732 733// golistDriver uses the "go list" command to expand the pattern 734// words and return metadata for the specified packages. dir may be 735// "" and env may be nil, as per os/exec.Command. 736func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driverResponse, error) { 737 // go list uses the following identifiers in ImportPath and Imports: 738 // 739 // "p" -- importable package or main (command) 740 // "q.test" -- q's test executable 741 // "p [q.test]" -- variant of p as built for q's test executable 742 // "q_test [q.test]" -- q's external test package 743 // 744 // The packages p that are built differently for a test q.test 745 // are q itself, plus any helpers used by the external test q_test, 746 // typically including "testing" and all its dependencies. 747 748 // Run "go list" for complete 749 // information on the specified packages. 750 buf, err := invokeGo(cfg, golistargs(cfg, words)...) 751 if err != nil { 752 return nil, err 753 } 754 seen := make(map[string]*jsonPackage) 755 // Decode the JSON and convert it to Package form. 756 var response driverResponse 757 for dec := json.NewDecoder(buf); dec.More(); { 758 p := new(jsonPackage) 759 if err := dec.Decode(p); err != nil { 760 return nil, fmt.Errorf("JSON decoding failed: %v", err) 761 } 762 763 if p.ImportPath == "" { 764 // The documentation for go list says that “[e]rroneous packages will have 765 // a non-empty ImportPath”. If for some reason it comes back empty, we 766 // prefer to error out rather than silently discarding data or handing 767 // back a package without any way to refer to it. 768 if p.Error != nil { 769 return nil, Error{ 770 Pos: p.Error.Pos, 771 Msg: p.Error.Err, 772 } 773 } 774 return nil, fmt.Errorf("package missing import path: %+v", p) 775 } 776 777 // Work around https://golang.org/issue/33157: 778 // go list -e, when given an absolute path, will find the package contained at 779 // that directory. But when no package exists there, it will return a fake package 780 // with an error and the ImportPath set to the absolute path provided to go list. 781 // Try to convert that absolute path to what its package path would be if it's 782 // contained in a known module or GOPATH entry. This will allow the package to be 783 // properly "reclaimed" when overlays are processed. 784 if filepath.IsAbs(p.ImportPath) && p.Error != nil { 785 pkgPath, ok := getPkgPath(cfg, p.ImportPath, rootsDirs) 786 if ok { 787 p.ImportPath = pkgPath 788 } 789 } 790 791 if old, found := seen[p.ImportPath]; found { 792 if !reflect.DeepEqual(p, old) { 793 return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath) 794 } 795 // skip the duplicate 796 continue 797 } 798 seen[p.ImportPath] = p 799 800 pkg := &Package{ 801 Name: p.Name, 802 ID: p.ImportPath, 803 GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), 804 CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles), 805 OtherFiles: absJoin(p.Dir, otherFiles(p)...), 806 } 807 808 // Work around https://golang.org/issue/28749: 809 // cmd/go puts assembly, C, and C++ files in CompiledGoFiles. 810 // Filter out any elements of CompiledGoFiles that are also in OtherFiles. 811 // We have to keep this workaround in place until go1.12 is a distant memory. 812 if len(pkg.OtherFiles) > 0 { 813 other := make(map[string]bool, len(pkg.OtherFiles)) 814 for _, f := range pkg.OtherFiles { 815 other[f] = true 816 } 817 818 out := pkg.CompiledGoFiles[:0] 819 for _, f := range pkg.CompiledGoFiles { 820 if other[f] { 821 continue 822 } 823 out = append(out, f) 824 } 825 pkg.CompiledGoFiles = out 826 } 827 828 // Extract the PkgPath from the package's ID. 829 if i := strings.IndexByte(pkg.ID, ' '); i >= 0 { 830 pkg.PkgPath = pkg.ID[:i] 831 } else { 832 pkg.PkgPath = pkg.ID 833 } 834 835 if pkg.PkgPath == "unsafe" { 836 pkg.GoFiles = nil // ignore fake unsafe.go file 837 } 838 839 // Assume go list emits only absolute paths for Dir. 840 if p.Dir != "" && !filepath.IsAbs(p.Dir) { 841 log.Fatalf("internal error: go list returned non-absolute Package.Dir: %s", p.Dir) 842 } 843 844 if p.Export != "" && !filepath.IsAbs(p.Export) { 845 pkg.ExportFile = filepath.Join(p.Dir, p.Export) 846 } else { 847 pkg.ExportFile = p.Export 848 } 849 850 // imports 851 // 852 // Imports contains the IDs of all imported packages. 853 // ImportsMap records (path, ID) only where they differ. 854 ids := make(map[string]bool) 855 for _, id := range p.Imports { 856 ids[id] = true 857 } 858 pkg.Imports = make(map[string]*Package) 859 for path, id := range p.ImportMap { 860 pkg.Imports[path] = &Package{ID: id} // non-identity import 861 delete(ids, id) 862 } 863 for id := range ids { 864 if id == "C" { 865 continue 866 } 867 868 pkg.Imports[id] = &Package{ID: id} // identity import 869 } 870 if !p.DepOnly { 871 response.Roots = append(response.Roots, pkg.ID) 872 } 873 874 // Work around for pre-go.1.11 versions of go list. 875 // TODO(matloob): they should be handled by the fallback. 876 // Can we delete this? 877 if len(pkg.CompiledGoFiles) == 0 { 878 pkg.CompiledGoFiles = pkg.GoFiles 879 } 880 881 if p.Error != nil { 882 pkg.Errors = append(pkg.Errors, Error{ 883 Pos: p.Error.Pos, 884 Msg: strings.TrimSpace(p.Error.Err), // Trim to work around golang.org/issue/32363. 885 }) 886 } 887 888 response.Packages = append(response.Packages, pkg) 889 } 890 891 return &response, nil 892} 893 894// getPkgPath finds the package path of a directory if it's relative to a root directory. 895func getPkgPath(cfg *Config, dir string, goInfo func() *goInfo) (string, bool) { 896 absDir, err := filepath.Abs(dir) 897 if err != nil { 898 cfg.Logf("error getting absolute path of %s: %v", dir, err) 899 return "", false 900 } 901 for rdir, rpath := range goInfo().rootDirs { 902 absRdir, err := filepath.Abs(rdir) 903 if err != nil { 904 cfg.Logf("error getting absolute path of %s: %v", rdir, err) 905 continue 906 } 907 // Make sure that the directory is in the module, 908 // to avoid creating a path relative to another module. 909 if !strings.HasPrefix(absDir, absRdir) { 910 cfg.Logf("%s does not have prefix %s", absDir, absRdir) 911 continue 912 } 913 // TODO(matloob): This doesn't properly handle symlinks. 914 r, err := filepath.Rel(rdir, dir) 915 if err != nil { 916 continue 917 } 918 if rpath != "" { 919 // We choose only one root even though the directory even it can belong in multiple modules 920 // or GOPATH entries. This is okay because we only need to work with absolute dirs when a 921 // file is missing from disk, for instance when gopls calls go/packages in an overlay. 922 // Once the file is saved, gopls, or the next invocation of the tool will get the correct 923 // result straight from golist. 924 // TODO(matloob): Implement module tiebreaking? 925 return path.Join(rpath, filepath.ToSlash(r)), true 926 } 927 return filepath.ToSlash(r), true 928 } 929 return "", false 930} 931 932// absJoin absolutizes and flattens the lists of files. 933func absJoin(dir string, fileses ...[]string) (res []string) { 934 for _, files := range fileses { 935 for _, file := range files { 936 if !filepath.IsAbs(file) { 937 file = filepath.Join(dir, file) 938 } 939 res = append(res, file) 940 } 941 } 942 return res 943} 944 945func golistargs(cfg *Config, words []string) []string { 946 const findFlags = NeedImports | NeedTypes | NeedSyntax | NeedTypesInfo 947 fullargs := []string{ 948 "list", "-e", "-json", 949 fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypesInfo|NeedTypesSizes) != 0), 950 fmt.Sprintf("-test=%t", cfg.Tests), 951 fmt.Sprintf("-export=%t", usesExportData(cfg)), 952 fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0), 953 // go list doesn't let you pass -test and -find together, 954 // probably because you'd just get the TestMain. 955 fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0), 956 } 957 fullargs = append(fullargs, cfg.BuildFlags...) 958 fullargs = append(fullargs, "--") 959 fullargs = append(fullargs, words...) 960 return fullargs 961} 962 963// invokeGo returns the stdout of a go command invocation. 964func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) { 965 stdout := new(bytes.Buffer) 966 stderr := new(bytes.Buffer) 967 cmd := exec.CommandContext(cfg.Context, "go", args...) 968 // On darwin the cwd gets resolved to the real path, which breaks anything that 969 // expects the working directory to keep the original path, including the 970 // go command when dealing with modules. 971 // The Go stdlib has a special feature where if the cwd and the PWD are the 972 // same node then it trusts the PWD, so by setting it in the env for the child 973 // process we fix up all the paths returned by the go command. 974 cmd.Env = append(append([]string{}, cfg.Env...), "PWD="+cfg.Dir) 975 cmd.Dir = cfg.Dir 976 cmd.Stdout = stdout 977 cmd.Stderr = stderr 978 defer func(start time.Time) { 979 cfg.Logf("%s for %v, stderr: <<%s>> stdout: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, args...), stderr, stdout) 980 }(time.Now()) 981 982 if err := cmd.Run(); err != nil { 983 // Check for 'go' executable not being found. 984 if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { 985 return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound) 986 } 987 988 exitErr, ok := err.(*exec.ExitError) 989 if !ok { 990 // Catastrophic error: 991 // - context cancellation 992 return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err) 993 } 994 995 // Old go version? 996 if strings.Contains(stderr.String(), "flag provided but not defined") { 997 return nil, goTooOldError{fmt.Errorf("unsupported version of go: %s: %s", exitErr, stderr)} 998 } 999 1000 // Related to #24854 1001 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "unexpected directory layout") { 1002 return nil, fmt.Errorf("%s", stderr.String()) 1003 } 1004 1005 // Is there an error running the C compiler in cgo? This will be reported in the "Error" field 1006 // and should be suppressed by go list -e. 1007 // 1008 // This condition is not perfect yet because the error message can include other error messages than runtime/cgo. 1009 isPkgPathRune := func(r rune) bool { 1010 // From https://golang.org/ref/spec#Import_declarations: 1011 // Implementation restriction: A compiler may restrict ImportPaths to non-empty strings 1012 // using only characters belonging to Unicode's L, M, N, P, and S general categories 1013 // (the Graphic characters without spaces) and may also exclude the 1014 // characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD. 1015 return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) && 1016 !strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r) 1017 } 1018 if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") { 1019 if strings.HasPrefix(strings.TrimLeftFunc(stderr.String()[len("# "):], isPkgPathRune), "\n") { 1020 return stdout, nil 1021 } 1022 } 1023 1024 // This error only appears in stderr. See golang.org/cl/166398 for a fix in go list to show 1025 // the error in the Err section of stdout in case -e option is provided. 1026 // This fix is provided for backwards compatibility. 1027 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must be .go files") { 1028 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1029 strings.Trim(stderr.String(), "\n")) 1030 return bytes.NewBufferString(output), nil 1031 } 1032 1033 // Similar to the previous error, but currently lacks a fix in Go. 1034 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must all be in one directory") { 1035 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1036 strings.Trim(stderr.String(), "\n")) 1037 return bytes.NewBufferString(output), nil 1038 } 1039 1040 // Backwards compatibility for Go 1.11 because 1.12 and 1.13 put the directory in the ImportPath. 1041 // If the package doesn't exist, put the absolute path of the directory into the error message, 1042 // as Go 1.13 list does. 1043 const noSuchDirectory = "no such directory" 1044 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), noSuchDirectory) { 1045 errstr := stderr.String() 1046 abspath := strings.TrimSpace(errstr[strings.Index(errstr, noSuchDirectory)+len(noSuchDirectory):]) 1047 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1048 abspath, strings.Trim(stderr.String(), "\n")) 1049 return bytes.NewBufferString(output), nil 1050 } 1051 1052 // Workaround for #29280: go list -e has incorrect behavior when an ad-hoc package doesn't exist. 1053 // Note that the error message we look for in this case is different that the one looked for above. 1054 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no such file or directory") { 1055 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1056 strings.Trim(stderr.String(), "\n")) 1057 return bytes.NewBufferString(output), nil 1058 } 1059 1060 // Workaround for #34273. go list -e with GO111MODULE=on has incorrect behavior when listing a 1061 // directory outside any module. 1062 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside available modules") { 1063 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1064 // TODO(matloob): command-line-arguments isn't correct here. 1065 "command-line-arguments", strings.Trim(stderr.String(), "\n")) 1066 return bytes.NewBufferString(output), nil 1067 } 1068 1069 // Another variation of the previous error 1070 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") { 1071 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1072 // TODO(matloob): command-line-arguments isn't correct here. 1073 "command-line-arguments", strings.Trim(stderr.String(), "\n")) 1074 return bytes.NewBufferString(output), nil 1075 } 1076 1077 // Workaround for an instance of golang.org/issue/26755: go list -e will return a non-zero exit 1078 // status if there's a dependency on a package that doesn't exist. But it should return 1079 // a zero exit status and set an error on that package. 1080 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") { 1081 // Don't clobber stdout if `go list` actually returned something. 1082 if len(stdout.String()) > 0 { 1083 return stdout, nil 1084 } 1085 // try to extract package name from string 1086 stderrStr := stderr.String() 1087 var importPath string 1088 colon := strings.Index(stderrStr, ":") 1089 if colon > 0 && strings.HasPrefix(stderrStr, "go build ") { 1090 importPath = stderrStr[len("go build "):colon] 1091 } 1092 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1093 importPath, strings.Trim(stderrStr, "\n")) 1094 return bytes.NewBufferString(output), nil 1095 } 1096 1097 // Export mode entails a build. 1098 // If that build fails, errors appear on stderr 1099 // (despite the -e flag) and the Export field is blank. 1100 // Do not fail in that case. 1101 // The same is true if an ad-hoc package given to go list doesn't exist. 1102 // TODO(matloob): Remove these once we can depend on go list to exit with a zero status with -e even when 1103 // packages don't exist or a build fails. 1104 if !usesExportData(cfg) && !containsGoFile(args) { 1105 return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr) 1106 } 1107 } 1108 1109 // As of writing, go list -export prints some non-fatal compilation 1110 // errors to stderr, even with -e set. We would prefer that it put 1111 // them in the Package.Error JSON (see https://golang.org/issue/26319). 1112 // In the meantime, there's nowhere good to put them, but they can 1113 // be useful for debugging. Print them if $GOPACKAGESPRINTGOLISTERRORS 1114 // is set. 1115 if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" { 1116 fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, args...), stderr) 1117 } 1118 return stdout, nil 1119} 1120 1121func containsGoFile(s []string) bool { 1122 for _, f := range s { 1123 if strings.HasSuffix(f, ".go") { 1124 return true 1125 } 1126 } 1127 return false 1128} 1129 1130func cmdDebugStr(cmd *exec.Cmd, args ...string) string { 1131 env := make(map[string]string) 1132 for _, kv := range cmd.Env { 1133 split := strings.Split(kv, "=") 1134 k, v := split[0], split[1] 1135 env[k] = v 1136 } 1137 var quotedArgs []string 1138 for _, arg := range args { 1139 quotedArgs = append(quotedArgs, strconv.Quote(arg)) 1140 } 1141 1142 return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v PWD=%v go %s", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["PWD"], strings.Join(quotedArgs, " ")) 1143} 1144