1// Copyright 2013 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package loader 6 7// See doc.go for package documentation and implementation notes. 8 9import ( 10 "errors" 11 "fmt" 12 "go/ast" 13 "go/build" 14 "go/parser" 15 "go/token" 16 "go/types" 17 "os" 18 "path/filepath" 19 "sort" 20 "strings" 21 "sync" 22 "time" 23 24 "golang.org/x/tools/go/ast/astutil" 25) 26 27var ignoreVendor build.ImportMode 28 29const trace = false // show timing info for type-checking 30 31// Config specifies the configuration for loading a whole program from 32// Go source code. 33// The zero value for Config is a ready-to-use default configuration. 34type Config struct { 35 // Fset is the file set for the parser to use when loading the 36 // program. If nil, it may be lazily initialized by any 37 // method of Config. 38 Fset *token.FileSet 39 40 // ParserMode specifies the mode to be used by the parser when 41 // loading source packages. 42 ParserMode parser.Mode 43 44 // TypeChecker contains options relating to the type checker. 45 // 46 // The supplied IgnoreFuncBodies is not used; the effective 47 // value comes from the TypeCheckFuncBodies func below. 48 // The supplied Import function is not used either. 49 TypeChecker types.Config 50 51 // TypeCheckFuncBodies is a predicate over package paths. 52 // A package for which the predicate is false will 53 // have its package-level declarations type checked, but not 54 // its function bodies; this can be used to quickly load 55 // dependencies from source. If nil, all func bodies are type 56 // checked. 57 TypeCheckFuncBodies func(path string) bool 58 59 // If Build is non-nil, it is used to locate source packages. 60 // Otherwise &build.Default is used. 61 // 62 // By default, cgo is invoked to preprocess Go files that 63 // import the fake package "C". This behaviour can be 64 // disabled by setting CGO_ENABLED=0 in the environment prior 65 // to startup, or by setting Build.CgoEnabled=false. 66 Build *build.Context 67 68 // The current directory, used for resolving relative package 69 // references such as "./go/loader". If empty, os.Getwd will be 70 // used instead. 71 Cwd string 72 73 // If DisplayPath is non-nil, it is used to transform each 74 // file name obtained from Build.Import(). This can be used 75 // to prevent a virtualized build.Config's file names from 76 // leaking into the user interface. 77 DisplayPath func(path string) string 78 79 // If AllowErrors is true, Load will return a Program even 80 // if some of the its packages contained I/O, parser or type 81 // errors; such errors are accessible via PackageInfo.Errors. If 82 // false, Load will fail if any package had an error. 83 AllowErrors bool 84 85 // CreatePkgs specifies a list of non-importable initial 86 // packages to create. The resulting packages will appear in 87 // the corresponding elements of the Program.Created slice. 88 CreatePkgs []PkgSpec 89 90 // ImportPkgs specifies a set of initial packages to load. 91 // The map keys are package paths. 92 // 93 // The map value indicates whether to load tests. If true, Load 94 // will add and type-check two lists of files to the package: 95 // non-test files followed by in-package *_test.go files. In 96 // addition, it will append the external test package (if any) 97 // to Program.Created. 98 ImportPkgs map[string]bool 99 100 // FindPackage is called during Load to create the build.Package 101 // for a given import path from a given directory. 102 // If FindPackage is nil, (*build.Context).Import is used. 103 // A client may use this hook to adapt to a proprietary build 104 // system that does not follow the "go build" layout 105 // conventions, for example. 106 // 107 // It must be safe to call concurrently from multiple goroutines. 108 FindPackage func(ctxt *build.Context, importPath, fromDir string, mode build.ImportMode) (*build.Package, error) 109 110 // AfterTypeCheck is called immediately after a list of files 111 // has been type-checked and appended to info.Files. 112 // 113 // This optional hook function is the earliest opportunity for 114 // the client to observe the output of the type checker, 115 // which may be useful to reduce analysis latency when loading 116 // a large program. 117 // 118 // The function is permitted to modify info.Info, for instance 119 // to clear data structures that are no longer needed, which can 120 // dramatically reduce peak memory consumption. 121 // 122 // The function may be called twice for the same PackageInfo: 123 // once for the files of the package and again for the 124 // in-package test files. 125 // 126 // It must be safe to call concurrently from multiple goroutines. 127 AfterTypeCheck func(info *PackageInfo, files []*ast.File) 128} 129 130// A PkgSpec specifies a non-importable package to be created by Load. 131// Files are processed first, but typically only one of Files and 132// Filenames is provided. The path needn't be globally unique. 133// 134// For vendoring purposes, the package's directory is the one that 135// contains the first file. 136type PkgSpec struct { 137 Path string // package path ("" => use package declaration) 138 Files []*ast.File // ASTs of already-parsed files 139 Filenames []string // names of files to be parsed 140} 141 142// A Program is a Go program loaded from source as specified by a Config. 143type Program struct { 144 Fset *token.FileSet // the file set for this program 145 146 // Created[i] contains the initial package whose ASTs or 147 // filenames were supplied by Config.CreatePkgs[i], followed by 148 // the external test package, if any, of each package in 149 // Config.ImportPkgs ordered by ImportPath. 150 // 151 // NOTE: these files must not import "C". Cgo preprocessing is 152 // only performed on imported packages, not ad hoc packages. 153 // 154 // TODO(adonovan): we need to copy and adapt the logic of 155 // goFilesPackage (from $GOROOT/src/cmd/go/build.go) and make 156 // Config.Import and Config.Create methods return the same kind 157 // of entity, essentially a build.Package. 158 // Perhaps we can even reuse that type directly. 159 Created []*PackageInfo 160 161 // Imported contains the initially imported packages, 162 // as specified by Config.ImportPkgs. 163 Imported map[string]*PackageInfo 164 165 // AllPackages contains the PackageInfo of every package 166 // encountered by Load: all initial packages and all 167 // dependencies, including incomplete ones. 168 AllPackages map[*types.Package]*PackageInfo 169 170 // importMap is the canonical mapping of package paths to 171 // packages. It contains all Imported initial packages, but not 172 // Created ones, and all imported dependencies. 173 importMap map[string]*types.Package 174} 175 176// PackageInfo holds the ASTs and facts derived by the type-checker 177// for a single package. 178// 179// Not mutated once exposed via the API. 180// 181type PackageInfo struct { 182 Pkg *types.Package 183 Importable bool // true if 'import "Pkg.Path()"' would resolve to this 184 TransitivelyErrorFree bool // true if Pkg and all its dependencies are free of errors 185 Files []*ast.File // syntax trees for the package's files 186 Errors []error // non-nil if the package had errors 187 types.Info // type-checker deductions. 188 dir string // package directory 189 190 checker *types.Checker // transient type-checker state 191 errorFunc func(error) 192} 193 194func (info *PackageInfo) String() string { return info.Pkg.Path() } 195 196func (info *PackageInfo) appendError(err error) { 197 if info.errorFunc != nil { 198 info.errorFunc(err) 199 } else { 200 fmt.Fprintln(os.Stderr, err) 201 } 202 info.Errors = append(info.Errors, err) 203} 204 205func (conf *Config) fset() *token.FileSet { 206 if conf.Fset == nil { 207 conf.Fset = token.NewFileSet() 208 } 209 return conf.Fset 210} 211 212// ParseFile is a convenience function (intended for testing) that invokes 213// the parser using the Config's FileSet, which is initialized if nil. 214// 215// src specifies the parser input as a string, []byte, or io.Reader, and 216// filename is its apparent name. If src is nil, the contents of 217// filename are read from the file system. 218// 219func (conf *Config) ParseFile(filename string, src interface{}) (*ast.File, error) { 220 // TODO(adonovan): use conf.build() etc like parseFiles does. 221 return parser.ParseFile(conf.fset(), filename, src, conf.ParserMode) 222} 223 224// FromArgsUsage is a partial usage message that applications calling 225// FromArgs may wish to include in their -help output. 226const FromArgsUsage = ` 227<args> is a list of arguments denoting a set of initial packages. 228It may take one of two forms: 229 2301. A list of *.go source files. 231 232 All of the specified files are loaded, parsed and type-checked 233 as a single package. All the files must belong to the same directory. 234 2352. A list of import paths, each denoting a package. 236 237 The package's directory is found relative to the $GOROOT and 238 $GOPATH using similar logic to 'go build', and the *.go files in 239 that directory are loaded, parsed and type-checked as a single 240 package. 241 242 In addition, all *_test.go files in the directory are then loaded 243 and parsed. Those files whose package declaration equals that of 244 the non-*_test.go files are included in the primary package. Test 245 files whose package declaration ends with "_test" are type-checked 246 as another package, the 'external' test package, so that a single 247 import path may denote two packages. (Whether this behaviour is 248 enabled is tool-specific, and may depend on additional flags.) 249 250A '--' argument terminates the list of packages. 251` 252 253// FromArgs interprets args as a set of initial packages to load from 254// source and updates the configuration. It returns the list of 255// unconsumed arguments. 256// 257// It is intended for use in command-line interfaces that require a 258// set of initial packages to be specified; see FromArgsUsage message 259// for details. 260// 261// Only superficial errors are reported at this stage; errors dependent 262// on I/O are detected during Load. 263// 264func (conf *Config) FromArgs(args []string, xtest bool) ([]string, error) { 265 var rest []string 266 for i, arg := range args { 267 if arg == "--" { 268 rest = args[i+1:] 269 args = args[:i] 270 break // consume "--" and return the remaining args 271 } 272 } 273 274 if len(args) > 0 && strings.HasSuffix(args[0], ".go") { 275 // Assume args is a list of a *.go files 276 // denoting a single ad hoc package. 277 for _, arg := range args { 278 if !strings.HasSuffix(arg, ".go") { 279 return nil, fmt.Errorf("named files must be .go files: %s", arg) 280 } 281 } 282 conf.CreateFromFilenames("", args...) 283 } else { 284 // Assume args are directories each denoting a 285 // package and (perhaps) an external test, iff xtest. 286 for _, arg := range args { 287 if xtest { 288 conf.ImportWithTests(arg) 289 } else { 290 conf.Import(arg) 291 } 292 } 293 } 294 295 return rest, nil 296} 297 298// CreateFromFilenames is a convenience function that adds 299// a conf.CreatePkgs entry to create a package of the specified *.go 300// files. 301// 302func (conf *Config) CreateFromFilenames(path string, filenames ...string) { 303 conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Filenames: filenames}) 304} 305 306// CreateFromFiles is a convenience function that adds a conf.CreatePkgs 307// entry to create package of the specified path and parsed files. 308// 309func (conf *Config) CreateFromFiles(path string, files ...*ast.File) { 310 conf.CreatePkgs = append(conf.CreatePkgs, PkgSpec{Path: path, Files: files}) 311} 312 313// ImportWithTests is a convenience function that adds path to 314// ImportPkgs, the set of initial source packages located relative to 315// $GOPATH. The package will be augmented by any *_test.go files in 316// its directory that contain a "package x" (not "package x_test") 317// declaration. 318// 319// In addition, if any *_test.go files contain a "package x_test" 320// declaration, an additional package comprising just those files will 321// be added to CreatePkgs. 322// 323func (conf *Config) ImportWithTests(path string) { conf.addImport(path, true) } 324 325// Import is a convenience function that adds path to ImportPkgs, the 326// set of initial packages that will be imported from source. 327// 328func (conf *Config) Import(path string) { conf.addImport(path, false) } 329 330func (conf *Config) addImport(path string, tests bool) { 331 if path == "C" { 332 return // ignore; not a real package 333 } 334 if conf.ImportPkgs == nil { 335 conf.ImportPkgs = make(map[string]bool) 336 } 337 conf.ImportPkgs[path] = conf.ImportPkgs[path] || tests 338} 339 340// PathEnclosingInterval returns the PackageInfo and ast.Node that 341// contain source interval [start, end), and all the node's ancestors 342// up to the AST root. It searches all ast.Files of all packages in prog. 343// exact is defined as for astutil.PathEnclosingInterval. 344// 345// The zero value is returned if not found. 346// 347func (prog *Program) PathEnclosingInterval(start, end token.Pos) (pkg *PackageInfo, path []ast.Node, exact bool) { 348 for _, info := range prog.AllPackages { 349 for _, f := range info.Files { 350 if f.Pos() == token.NoPos { 351 // This can happen if the parser saw 352 // too many errors and bailed out. 353 // (Use parser.AllErrors to prevent that.) 354 continue 355 } 356 if !tokenFileContainsPos(prog.Fset.File(f.Pos()), start) { 357 continue 358 } 359 if path, exact := astutil.PathEnclosingInterval(f, start, end); path != nil { 360 return info, path, exact 361 } 362 } 363 } 364 return nil, nil, false 365} 366 367// InitialPackages returns a new slice containing the set of initial 368// packages (Created + Imported) in unspecified order. 369// 370func (prog *Program) InitialPackages() []*PackageInfo { 371 infos := make([]*PackageInfo, 0, len(prog.Created)+len(prog.Imported)) 372 infos = append(infos, prog.Created...) 373 for _, info := range prog.Imported { 374 infos = append(infos, info) 375 } 376 return infos 377} 378 379// Package returns the ASTs and results of type checking for the 380// specified package. 381func (prog *Program) Package(path string) *PackageInfo { 382 if info, ok := prog.AllPackages[prog.importMap[path]]; ok { 383 return info 384 } 385 for _, info := range prog.Created { 386 if path == info.Pkg.Path() { 387 return info 388 } 389 } 390 return nil 391} 392 393// ---------- Implementation ---------- 394 395// importer holds the working state of the algorithm. 396type importer struct { 397 conf *Config // the client configuration 398 start time.Time // for logging 399 400 progMu sync.Mutex // guards prog 401 prog *Program // the resulting program 402 403 // findpkg is a memoization of FindPackage. 404 findpkgMu sync.Mutex // guards findpkg 405 findpkg map[findpkgKey]*findpkgValue 406 407 importedMu sync.Mutex // guards imported 408 imported map[string]*importInfo // all imported packages (incl. failures) by import path 409 410 // import dependency graph: graph[x][y] => x imports y 411 // 412 // Since non-importable packages cannot be cyclic, we ignore 413 // their imports, thus we only need the subgraph over importable 414 // packages. Nodes are identified by their import paths. 415 graphMu sync.Mutex 416 graph map[string]map[string]bool 417} 418 419type findpkgKey struct { 420 importPath string 421 fromDir string 422 mode build.ImportMode 423} 424 425type findpkgValue struct { 426 ready chan struct{} // closed to broadcast readiness 427 bp *build.Package 428 err error 429} 430 431// importInfo tracks the success or failure of a single import. 432// 433// Upon completion, exactly one of info and err is non-nil: 434// info on successful creation of a package, err otherwise. 435// A successful package may still contain type errors. 436// 437type importInfo struct { 438 path string // import path 439 info *PackageInfo // results of typechecking (including errors) 440 complete chan struct{} // closed to broadcast that info is set. 441} 442 443// awaitCompletion blocks until ii is complete, 444// i.e. the info field is safe to inspect. 445func (ii *importInfo) awaitCompletion() { 446 <-ii.complete // wait for close 447} 448 449// Complete marks ii as complete. 450// Its info and err fields will not be subsequently updated. 451func (ii *importInfo) Complete(info *PackageInfo) { 452 if info == nil { 453 panic("info == nil") 454 } 455 ii.info = info 456 close(ii.complete) 457} 458 459type importError struct { 460 path string // import path 461 err error // reason for failure to create a package 462} 463 464// Load creates the initial packages specified by conf.{Create,Import}Pkgs, 465// loading their dependencies packages as needed. 466// 467// On success, Load returns a Program containing a PackageInfo for 468// each package. On failure, it returns an error. 469// 470// If AllowErrors is true, Load will return a Program even if some 471// packages contained I/O, parser or type errors, or if dependencies 472// were missing. (Such errors are accessible via PackageInfo.Errors. If 473// false, Load will fail if any package had an error. 474// 475// It is an error if no packages were loaded. 476// 477func (conf *Config) Load() (*Program, error) { 478 // Create a simple default error handler for parse/type errors. 479 if conf.TypeChecker.Error == nil { 480 conf.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) } 481 } 482 483 // Set default working directory for relative package references. 484 if conf.Cwd == "" { 485 var err error 486 conf.Cwd, err = os.Getwd() 487 if err != nil { 488 return nil, err 489 } 490 } 491 492 // Install default FindPackage hook using go/build logic. 493 if conf.FindPackage == nil { 494 conf.FindPackage = (*build.Context).Import 495 } 496 497 prog := &Program{ 498 Fset: conf.fset(), 499 Imported: make(map[string]*PackageInfo), 500 importMap: make(map[string]*types.Package), 501 AllPackages: make(map[*types.Package]*PackageInfo), 502 } 503 504 imp := importer{ 505 conf: conf, 506 prog: prog, 507 findpkg: make(map[findpkgKey]*findpkgValue), 508 imported: make(map[string]*importInfo), 509 start: time.Now(), 510 graph: make(map[string]map[string]bool), 511 } 512 513 // -- loading proper (concurrent phase) -------------------------------- 514 515 var errpkgs []string // packages that contained errors 516 517 // Load the initially imported packages and their dependencies, 518 // in parallel. 519 // No vendor check on packages imported from the command line. 520 infos, importErrors := imp.importAll("", conf.Cwd, conf.ImportPkgs, ignoreVendor) 521 for _, ie := range importErrors { 522 conf.TypeChecker.Error(ie.err) // failed to create package 523 errpkgs = append(errpkgs, ie.path) 524 } 525 for _, info := range infos { 526 prog.Imported[info.Pkg.Path()] = info 527 } 528 529 // Augment the designated initial packages by their tests. 530 // Dependencies are loaded in parallel. 531 var xtestPkgs []*build.Package 532 for importPath, augment := range conf.ImportPkgs { 533 if !augment { 534 continue 535 } 536 537 // No vendor check on packages imported from command line. 538 bp, err := imp.findPackage(importPath, conf.Cwd, ignoreVendor) 539 if err != nil { 540 // Package not found, or can't even parse package declaration. 541 // Already reported by previous loop; ignore it. 542 continue 543 } 544 545 // Needs external test package? 546 if len(bp.XTestGoFiles) > 0 { 547 xtestPkgs = append(xtestPkgs, bp) 548 } 549 550 // Consult the cache using the canonical package path. 551 path := bp.ImportPath 552 imp.importedMu.Lock() // (unnecessary, we're sequential here) 553 ii, ok := imp.imported[path] 554 // Paranoid checks added due to issue #11012. 555 if !ok { 556 // Unreachable. 557 // The previous loop called importAll and thus 558 // startLoad for each path in ImportPkgs, which 559 // populates imp.imported[path] with a non-zero value. 560 panic(fmt.Sprintf("imported[%q] not found", path)) 561 } 562 if ii == nil { 563 // Unreachable. 564 // The ii values in this loop are the same as in 565 // the previous loop, which enforced the invariant 566 // that at least one of ii.err and ii.info is non-nil. 567 panic(fmt.Sprintf("imported[%q] == nil", path)) 568 } 569 if ii.info == nil { 570 // Unreachable. 571 // awaitCompletion has the postcondition 572 // ii.info != nil. 573 panic(fmt.Sprintf("imported[%q].info = nil", path)) 574 } 575 info := ii.info 576 imp.importedMu.Unlock() 577 578 // Parse the in-package test files. 579 files, errs := imp.conf.parsePackageFiles(bp, 't') 580 for _, err := range errs { 581 info.appendError(err) 582 } 583 584 // The test files augmenting package P cannot be imported, 585 // but may import packages that import P, 586 // so we must disable the cycle check. 587 imp.addFiles(info, files, false) 588 } 589 590 createPkg := func(path, dir string, files []*ast.File, errs []error) { 591 info := imp.newPackageInfo(path, dir) 592 for _, err := range errs { 593 info.appendError(err) 594 } 595 596 // Ad hoc packages are non-importable, 597 // so no cycle check is needed. 598 // addFiles loads dependencies in parallel. 599 imp.addFiles(info, files, false) 600 prog.Created = append(prog.Created, info) 601 } 602 603 // Create packages specified by conf.CreatePkgs. 604 for _, cp := range conf.CreatePkgs { 605 files, errs := parseFiles(conf.fset(), conf.build(), nil, conf.Cwd, cp.Filenames, conf.ParserMode) 606 files = append(files, cp.Files...) 607 608 path := cp.Path 609 if path == "" { 610 if len(files) > 0 { 611 path = files[0].Name.Name 612 } else { 613 path = "(unnamed)" 614 } 615 } 616 617 dir := conf.Cwd 618 if len(files) > 0 && files[0].Pos().IsValid() { 619 dir = filepath.Dir(conf.fset().File(files[0].Pos()).Name()) 620 } 621 createPkg(path, dir, files, errs) 622 } 623 624 // Create external test packages. 625 sort.Sort(byImportPath(xtestPkgs)) 626 for _, bp := range xtestPkgs { 627 files, errs := imp.conf.parsePackageFiles(bp, 'x') 628 createPkg(bp.ImportPath+"_test", bp.Dir, files, errs) 629 } 630 631 // -- finishing up (sequential) ---------------------------------------- 632 633 if len(prog.Imported)+len(prog.Created) == 0 { 634 return nil, errors.New("no initial packages were loaded") 635 } 636 637 // Create infos for indirectly imported packages. 638 // e.g. incomplete packages without syntax, loaded from export data. 639 for _, obj := range prog.importMap { 640 info := prog.AllPackages[obj] 641 if info == nil { 642 prog.AllPackages[obj] = &PackageInfo{Pkg: obj, Importable: true} 643 } else { 644 // finished 645 info.checker = nil 646 info.errorFunc = nil 647 } 648 } 649 650 if !conf.AllowErrors { 651 // Report errors in indirectly imported packages. 652 for _, info := range prog.AllPackages { 653 if len(info.Errors) > 0 { 654 errpkgs = append(errpkgs, info.Pkg.Path()) 655 } 656 } 657 if errpkgs != nil { 658 var more string 659 if len(errpkgs) > 3 { 660 more = fmt.Sprintf(" and %d more", len(errpkgs)-3) 661 errpkgs = errpkgs[:3] 662 } 663 return nil, fmt.Errorf("couldn't load packages due to errors: %s%s", 664 strings.Join(errpkgs, ", "), more) 665 } 666 } 667 668 markErrorFreePackages(prog.AllPackages) 669 670 return prog, nil 671} 672 673type byImportPath []*build.Package 674 675func (b byImportPath) Len() int { return len(b) } 676func (b byImportPath) Less(i, j int) bool { return b[i].ImportPath < b[j].ImportPath } 677func (b byImportPath) Swap(i, j int) { b[i], b[j] = b[j], b[i] } 678 679// markErrorFreePackages sets the TransitivelyErrorFree flag on all 680// applicable packages. 681func markErrorFreePackages(allPackages map[*types.Package]*PackageInfo) { 682 // Build the transpose of the import graph. 683 importedBy := make(map[*types.Package]map[*types.Package]bool) 684 for P := range allPackages { 685 for _, Q := range P.Imports() { 686 clients, ok := importedBy[Q] 687 if !ok { 688 clients = make(map[*types.Package]bool) 689 importedBy[Q] = clients 690 } 691 clients[P] = true 692 } 693 } 694 695 // Find all packages reachable from some error package. 696 reachable := make(map[*types.Package]bool) 697 var visit func(*types.Package) 698 visit = func(p *types.Package) { 699 if !reachable[p] { 700 reachable[p] = true 701 for q := range importedBy[p] { 702 visit(q) 703 } 704 } 705 } 706 for _, info := range allPackages { 707 if len(info.Errors) > 0 { 708 visit(info.Pkg) 709 } 710 } 711 712 // Mark the others as "transitively error-free". 713 for _, info := range allPackages { 714 if !reachable[info.Pkg] { 715 info.TransitivelyErrorFree = true 716 } 717 } 718} 719 720// build returns the effective build context. 721func (conf *Config) build() *build.Context { 722 if conf.Build != nil { 723 return conf.Build 724 } 725 return &build.Default 726} 727 728// parsePackageFiles enumerates the files belonging to package path, 729// then loads, parses and returns them, plus a list of I/O or parse 730// errors that were encountered. 731// 732// 'which' indicates which files to include: 733// 'g': include non-test *.go source files (GoFiles + processed CgoFiles) 734// 't': include in-package *_test.go source files (TestGoFiles) 735// 'x': include external *_test.go source files. (XTestGoFiles) 736// 737func (conf *Config) parsePackageFiles(bp *build.Package, which rune) ([]*ast.File, []error) { 738 if bp.ImportPath == "unsafe" { 739 return nil, nil 740 } 741 var filenames []string 742 switch which { 743 case 'g': 744 filenames = bp.GoFiles 745 case 't': 746 filenames = bp.TestGoFiles 747 case 'x': 748 filenames = bp.XTestGoFiles 749 default: 750 panic(which) 751 } 752 753 files, errs := parseFiles(conf.fset(), conf.build(), conf.DisplayPath, bp.Dir, filenames, conf.ParserMode) 754 755 // Preprocess CgoFiles and parse the outputs (sequentially). 756 if which == 'g' && bp.CgoFiles != nil { 757 cgofiles, err := processCgoFiles(bp, conf.fset(), conf.DisplayPath, conf.ParserMode) 758 if err != nil { 759 errs = append(errs, err) 760 } else { 761 files = append(files, cgofiles...) 762 } 763 } 764 765 return files, errs 766} 767 768// doImport imports the package denoted by path. 769// It implements the types.Importer signature. 770// 771// It returns an error if a package could not be created 772// (e.g. go/build or parse error), but type errors are reported via 773// the types.Config.Error callback (the first of which is also saved 774// in the package's PackageInfo). 775// 776// Idempotent. 777// 778func (imp *importer) doImport(from *PackageInfo, to string) (*types.Package, error) { 779 if to == "C" { 780 // This should be unreachable, but ad hoc packages are 781 // not currently subject to cgo preprocessing. 782 // See https://github.com/golang/go/issues/11627. 783 return nil, fmt.Errorf(`the loader doesn't cgo-process ad hoc packages like %q; see Go issue 11627`, 784 from.Pkg.Path()) 785 } 786 787 bp, err := imp.findPackage(to, from.dir, 0) 788 if err != nil { 789 return nil, err 790 } 791 792 // The standard unsafe package is handled specially, 793 // and has no PackageInfo. 794 if bp.ImportPath == "unsafe" { 795 return types.Unsafe, nil 796 } 797 798 // Look for the package in the cache using its canonical path. 799 path := bp.ImportPath 800 imp.importedMu.Lock() 801 ii := imp.imported[path] 802 imp.importedMu.Unlock() 803 if ii == nil { 804 panic("internal error: unexpected import: " + path) 805 } 806 if ii.info != nil { 807 return ii.info.Pkg, nil 808 } 809 810 // Import of incomplete package: this indicates a cycle. 811 fromPath := from.Pkg.Path() 812 if cycle := imp.findPath(path, fromPath); cycle != nil { 813 cycle = append([]string{fromPath}, cycle...) 814 return nil, fmt.Errorf("import cycle: %s", strings.Join(cycle, " -> ")) 815 } 816 817 panic("internal error: import of incomplete (yet acyclic) package: " + fromPath) 818} 819 820// findPackage locates the package denoted by the importPath in the 821// specified directory. 822func (imp *importer) findPackage(importPath, fromDir string, mode build.ImportMode) (*build.Package, error) { 823 // We use a non-blocking duplicate-suppressing cache (gopl.io §9.7) 824 // to avoid holding the lock around FindPackage. 825 key := findpkgKey{importPath, fromDir, mode} 826 imp.findpkgMu.Lock() 827 v, ok := imp.findpkg[key] 828 if ok { 829 // cache hit 830 imp.findpkgMu.Unlock() 831 832 <-v.ready // wait for entry to become ready 833 } else { 834 // Cache miss: this goroutine becomes responsible for 835 // populating the map entry and broadcasting its readiness. 836 v = &findpkgValue{ready: make(chan struct{})} 837 imp.findpkg[key] = v 838 imp.findpkgMu.Unlock() 839 840 ioLimit <- true 841 v.bp, v.err = imp.conf.FindPackage(imp.conf.build(), importPath, fromDir, mode) 842 <-ioLimit 843 844 if _, ok := v.err.(*build.NoGoError); ok { 845 v.err = nil // empty directory is not an error 846 } 847 848 close(v.ready) // broadcast ready condition 849 } 850 return v.bp, v.err 851} 852 853// importAll loads, parses, and type-checks the specified packages in 854// parallel and returns their completed importInfos in unspecified order. 855// 856// fromPath is the package path of the importing package, if it is 857// importable, "" otherwise. It is used for cycle detection. 858// 859// fromDir is the directory containing the import declaration that 860// caused these imports. 861// 862func (imp *importer) importAll(fromPath, fromDir string, imports map[string]bool, mode build.ImportMode) (infos []*PackageInfo, errors []importError) { 863 // TODO(adonovan): opt: do the loop in parallel once 864 // findPackage is non-blocking. 865 var pending []*importInfo 866 for importPath := range imports { 867 bp, err := imp.findPackage(importPath, fromDir, mode) 868 if err != nil { 869 errors = append(errors, importError{ 870 path: importPath, 871 err: err, 872 }) 873 continue 874 } 875 pending = append(pending, imp.startLoad(bp)) 876 } 877 878 if fromPath != "" { 879 // We're loading a set of imports. 880 // 881 // We must record graph edges from the importing package 882 // to its dependencies, and check for cycles. 883 imp.graphMu.Lock() 884 deps, ok := imp.graph[fromPath] 885 if !ok { 886 deps = make(map[string]bool) 887 imp.graph[fromPath] = deps 888 } 889 for _, ii := range pending { 890 deps[ii.path] = true 891 } 892 imp.graphMu.Unlock() 893 } 894 895 for _, ii := range pending { 896 if fromPath != "" { 897 if cycle := imp.findPath(ii.path, fromPath); cycle != nil { 898 // Cycle-forming import: we must not await its 899 // completion since it would deadlock. 900 // 901 // We don't record the error in ii since 902 // the error is really associated with the 903 // cycle-forming edge, not the package itself. 904 // (Also it would complicate the 905 // invariants of importPath completion.) 906 if trace { 907 fmt.Fprintf(os.Stderr, "import cycle: %q\n", cycle) 908 } 909 continue 910 } 911 } 912 ii.awaitCompletion() 913 infos = append(infos, ii.info) 914 } 915 916 return infos, errors 917} 918 919// findPath returns an arbitrary path from 'from' to 'to' in the import 920// graph, or nil if there was none. 921func (imp *importer) findPath(from, to string) []string { 922 imp.graphMu.Lock() 923 defer imp.graphMu.Unlock() 924 925 seen := make(map[string]bool) 926 var search func(stack []string, importPath string) []string 927 search = func(stack []string, importPath string) []string { 928 if !seen[importPath] { 929 seen[importPath] = true 930 stack = append(stack, importPath) 931 if importPath == to { 932 return stack 933 } 934 for x := range imp.graph[importPath] { 935 if p := search(stack, x); p != nil { 936 return p 937 } 938 } 939 } 940 return nil 941 } 942 return search(make([]string, 0, 20), from) 943} 944 945// startLoad initiates the loading, parsing and type-checking of the 946// specified package and its dependencies, if it has not already begun. 947// 948// It returns an importInfo, not necessarily in a completed state. The 949// caller must call awaitCompletion() before accessing its info field. 950// 951// startLoad is concurrency-safe and idempotent. 952// 953func (imp *importer) startLoad(bp *build.Package) *importInfo { 954 path := bp.ImportPath 955 imp.importedMu.Lock() 956 ii, ok := imp.imported[path] 957 if !ok { 958 ii = &importInfo{path: path, complete: make(chan struct{})} 959 imp.imported[path] = ii 960 go func() { 961 info := imp.load(bp) 962 ii.Complete(info) 963 }() 964 } 965 imp.importedMu.Unlock() 966 967 return ii 968} 969 970// load implements package loading by parsing Go source files 971// located by go/build. 972func (imp *importer) load(bp *build.Package) *PackageInfo { 973 info := imp.newPackageInfo(bp.ImportPath, bp.Dir) 974 info.Importable = true 975 files, errs := imp.conf.parsePackageFiles(bp, 'g') 976 for _, err := range errs { 977 info.appendError(err) 978 } 979 980 imp.addFiles(info, files, true) 981 982 imp.progMu.Lock() 983 imp.prog.importMap[bp.ImportPath] = info.Pkg 984 imp.progMu.Unlock() 985 986 return info 987} 988 989// addFiles adds and type-checks the specified files to info, loading 990// their dependencies if needed. The order of files determines the 991// package initialization order. It may be called multiple times on the 992// same package. Errors are appended to the info.Errors field. 993// 994// cycleCheck determines whether the imports within files create 995// dependency edges that should be checked for potential cycles. 996// 997func (imp *importer) addFiles(info *PackageInfo, files []*ast.File, cycleCheck bool) { 998 // Ensure the dependencies are loaded, in parallel. 999 var fromPath string 1000 if cycleCheck { 1001 fromPath = info.Pkg.Path() 1002 } 1003 // TODO(adonovan): opt: make the caller do scanImports. 1004 // Callers with a build.Package can skip it. 1005 imp.importAll(fromPath, info.dir, scanImports(files), 0) 1006 1007 if trace { 1008 fmt.Fprintf(os.Stderr, "%s: start %q (%d)\n", 1009 time.Since(imp.start), info.Pkg.Path(), len(files)) 1010 } 1011 1012 // Don't call checker.Files on Unsafe, even with zero files, 1013 // because it would mutate the package, which is a global. 1014 if info.Pkg == types.Unsafe { 1015 if len(files) > 0 { 1016 panic(`"unsafe" package contains unexpected files`) 1017 } 1018 } else { 1019 // Ignore the returned (first) error since we 1020 // already collect them all in the PackageInfo. 1021 info.checker.Files(files) 1022 info.Files = append(info.Files, files...) 1023 } 1024 1025 if imp.conf.AfterTypeCheck != nil { 1026 imp.conf.AfterTypeCheck(info, files) 1027 } 1028 1029 if trace { 1030 fmt.Fprintf(os.Stderr, "%s: stop %q\n", 1031 time.Since(imp.start), info.Pkg.Path()) 1032 } 1033} 1034 1035func (imp *importer) newPackageInfo(path, dir string) *PackageInfo { 1036 var pkg *types.Package 1037 if path == "unsafe" { 1038 pkg = types.Unsafe 1039 } else { 1040 pkg = types.NewPackage(path, "") 1041 } 1042 info := &PackageInfo{ 1043 Pkg: pkg, 1044 Info: types.Info{ 1045 Types: make(map[ast.Expr]types.TypeAndValue), 1046 Defs: make(map[*ast.Ident]types.Object), 1047 Uses: make(map[*ast.Ident]types.Object), 1048 Implicits: make(map[ast.Node]types.Object), 1049 Scopes: make(map[ast.Node]*types.Scope), 1050 Selections: make(map[*ast.SelectorExpr]*types.Selection), 1051 }, 1052 errorFunc: imp.conf.TypeChecker.Error, 1053 dir: dir, 1054 } 1055 1056 // Copy the types.Config so we can vary it across PackageInfos. 1057 tc := imp.conf.TypeChecker 1058 tc.IgnoreFuncBodies = false 1059 if f := imp.conf.TypeCheckFuncBodies; f != nil { 1060 tc.IgnoreFuncBodies = !f(path) 1061 } 1062 tc.Importer = closure{imp, info} 1063 tc.Error = info.appendError // appendError wraps the user's Error function 1064 1065 info.checker = types.NewChecker(&tc, imp.conf.fset(), pkg, &info.Info) 1066 imp.progMu.Lock() 1067 imp.prog.AllPackages[pkg] = info 1068 imp.progMu.Unlock() 1069 return info 1070} 1071 1072type closure struct { 1073 imp *importer 1074 info *PackageInfo 1075} 1076 1077func (c closure) Import(to string) (*types.Package, error) { return c.imp.doImport(c.info, to) } 1078