1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package packages 6 7// See doc.go for package documentation and implementation notes. 8 9import ( 10 "context" 11 "encoding/json" 12 "fmt" 13 "go/ast" 14 "go/parser" 15 "go/scanner" 16 "go/token" 17 "go/types" 18 "io/ioutil" 19 "log" 20 "os" 21 "path/filepath" 22 "strings" 23 "sync" 24 25 "golang.org/x/tools/go/gcexportdata" 26) 27 28// A LoadMode specifies the amount of detail to return when loading. 29// Higher-numbered modes cause Load to return more information, 30// but may be slower. Load may return more information than requested. 31type LoadMode int 32 33const ( 34 // LoadFiles finds the packages and computes their source file lists. 35 // Package fields: ID, Name, Errors, GoFiles, and OtherFiles. 36 LoadFiles LoadMode = iota 37 38 // LoadImports adds import information for each package 39 // and its dependencies. 40 // Package fields added: Imports. 41 LoadImports 42 43 // LoadTypes adds type information for package-level 44 // declarations in the packages matching the patterns. 45 // Package fields added: Types, Fset, and IllTyped. 46 // This mode uses type information provided by the build system when 47 // possible, and may fill in the ExportFile field. 48 LoadTypes 49 50 // LoadSyntax adds typed syntax trees for the packages matching the patterns. 51 // Package fields added: Syntax, and TypesInfo, for direct pattern matches only. 52 LoadSyntax 53 54 // LoadAllSyntax adds typed syntax trees for the packages matching the patterns 55 // and all dependencies. 56 // Package fields added: Types, Fset, IllTyped, Syntax, and TypesInfo, 57 // for all packages in the import graph. 58 LoadAllSyntax 59) 60 61// A Config specifies details about how packages should be loaded. 62// The zero value is a valid configuration. 63// Calls to Load do not modify this struct. 64type Config struct { 65 // Mode controls the level of information returned for each package. 66 Mode LoadMode 67 68 // Context specifies the context for the load operation. 69 // If the context is cancelled, the loader may stop early 70 // and return an ErrCancelled error. 71 // If Context is nil, the load cannot be cancelled. 72 Context context.Context 73 74 // Dir is the directory in which to run the build system's query tool 75 // that provides information about the packages. 76 // If Dir is empty, the tool is run in the current directory. 77 Dir string 78 79 // Env is the environment to use when invoking the build system's query tool. 80 // If Env is nil, the current environment is used. 81 // As in os/exec's Cmd, only the last value in the slice for 82 // each environment key is used. To specify the setting of only 83 // a few variables, append to the current environment, as in: 84 // 85 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386") 86 // 87 Env []string 88 89 // BuildFlags is a list of command-line flags to be passed through to 90 // the build system's query tool. 91 BuildFlags []string 92 93 // Fset provides source position information for syntax trees and types. 94 // If Fset is nil, the loader will create a new FileSet. 95 Fset *token.FileSet 96 97 // ParseFile is called to read and parse each file 98 // when preparing a package's type-checked syntax tree. 99 // It must be safe to call ParseFile simultaneously from multiple goroutines. 100 // If ParseFile is nil, the loader will uses parser.ParseFile. 101 // 102 // ParseFile should parse the source from src and use filename only for 103 // recording position information. 104 // 105 // An application may supply a custom implementation of ParseFile 106 // to change the effective file contents or the behavior of the parser, 107 // or to modify the syntax tree. For example, selectively eliminating 108 // unwanted function bodies can significantly accelerate type checking. 109 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) 110 111 // If Tests is set, the loader includes not just the packages 112 // matching a particular pattern but also any related test packages, 113 // including test-only variants of the package and the test executable. 114 // 115 // For example, when using the go command, loading "fmt" with Tests=true 116 // returns four packages, with IDs "fmt" (the standard package), 117 // "fmt [fmt.test]" (the package as compiled for the test), 118 // "fmt_test" (the test functions from source files in package fmt_test), 119 // and "fmt.test" (the test binary). 120 // 121 // In build systems with explicit names for tests, 122 // setting Tests may have no effect. 123 Tests bool 124 125 // Overlay provides a mapping of absolute file paths to file contents. 126 // If the file with the given path already exists, the parser will use the 127 // alternative file contents provided by the map. 128 // 129 // Overlays provide incomplete support for when a given file doesn't 130 // already exist on disk. See the package doc above for more details. 131 Overlay map[string][]byte 132} 133 134// driver is the type for functions that query the build system for the 135// packages named by the patterns. 136type driver func(cfg *Config, patterns ...string) (*driverResponse, error) 137 138// driverResponse contains the results for a driver query. 139type driverResponse struct { 140 // Sizes, if not nil, is the types.Sizes to use when type checking. 141 Sizes *types.StdSizes 142 143 // Roots is the set of package IDs that make up the root packages. 144 // We have to encode this separately because when we encode a single package 145 // we cannot know if it is one of the roots as that requires knowledge of the 146 // graph it is part of. 147 Roots []string `json:",omitempty"` 148 149 // Packages is the full set of packages in the graph. 150 // The packages are not connected into a graph. 151 // The Imports if populated will be stubs that only have their ID set. 152 // Imports will be connected and then type and syntax information added in a 153 // later pass (see refine). 154 Packages []*Package 155} 156 157// Load loads and returns the Go packages named by the given patterns. 158// 159// Config specifies loading options; 160// nil behaves the same as an empty Config. 161// 162// Load returns an error if any of the patterns was invalid 163// as defined by the underlying build system. 164// It may return an empty list of packages without an error, 165// for instance for an empty expansion of a valid wildcard. 166// Errors associated with a particular package are recorded in the 167// corresponding Package's Errors list, and do not cause Load to 168// return an error. Clients may need to handle such errors before 169// proceeding with further analysis. The PrintErrors function is 170// provided for convenient display of all errors. 171func Load(cfg *Config, patterns ...string) ([]*Package, error) { 172 l := newLoader(cfg) 173 response, err := defaultDriver(&l.Config, patterns...) 174 if err != nil { 175 return nil, err 176 } 177 l.sizes = response.Sizes 178 return l.refine(response.Roots, response.Packages...) 179} 180 181// defaultDriver is a driver that looks for an external driver binary, and if 182// it does not find it falls back to the built in go list driver. 183func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) { 184 driver := findExternalDriver(cfg) 185 if driver == nil { 186 driver = goListDriver 187 } 188 return driver(cfg, patterns...) 189} 190 191// A Package describes a loaded Go package. 192type Package struct { 193 // ID is a unique identifier for a package, 194 // in a syntax provided by the underlying build system. 195 // 196 // Because the syntax varies based on the build system, 197 // clients should treat IDs as opaque and not attempt to 198 // interpret them. 199 ID string 200 201 // Name is the package name as it appears in the package source code. 202 Name string 203 204 // PkgPath is the package path as used by the go/types package. 205 PkgPath string 206 207 // Errors contains any errors encountered querying the metadata 208 // of the package, or while parsing or type-checking its files. 209 Errors []Error 210 211 // GoFiles lists the absolute file paths of the package's Go source files. 212 GoFiles []string 213 214 // CompiledGoFiles lists the absolute file paths of the package's source 215 // files that were presented to the compiler. 216 // This may differ from GoFiles if files are processed before compilation. 217 CompiledGoFiles []string 218 219 // OtherFiles lists the absolute file paths of the package's non-Go source files, 220 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on. 221 OtherFiles []string 222 223 // ExportFile is the absolute path to a file containing type 224 // information for the package as provided by the build system. 225 ExportFile string 226 227 // Imports maps import paths appearing in the package's Go source files 228 // to corresponding loaded Packages. 229 Imports map[string]*Package 230 231 // Types provides type information for the package. 232 // Modes LoadTypes and above set this field for packages matching the 233 // patterns; type information for dependencies may be missing or incomplete. 234 // Mode LoadAllSyntax sets this field for all packages, including dependencies. 235 Types *types.Package 236 237 // Fset provides position information for Types, TypesInfo, and Syntax. 238 // It is set only when Types is set. 239 Fset *token.FileSet 240 241 // IllTyped indicates whether the package or any dependency contains errors. 242 // It is set only when Types is set. 243 IllTyped bool 244 245 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles. 246 // 247 // Mode LoadSyntax sets this field for packages matching the patterns. 248 // Mode LoadAllSyntax sets this field for all packages, including dependencies. 249 Syntax []*ast.File 250 251 // TypesInfo provides type information about the package's syntax trees. 252 // It is set only when Syntax is set. 253 TypesInfo *types.Info 254 255 // TypesSizes provides the effective size function for types in TypesInfo. 256 TypesSizes types.Sizes 257} 258 259// An Error describes a problem with a package's metadata, syntax, or types. 260type Error struct { 261 Pos string // "file:line:col" or "file:line" or "" or "-" 262 Msg string 263 Kind ErrorKind 264} 265 266// ErrorKind describes the source of the error, allowing the user to 267// differentiate between errors generated by the driver, the parser, or the 268// type-checker. 269type ErrorKind int 270 271const ( 272 UnknownError ErrorKind = iota 273 ListError 274 ParseError 275 TypeError 276) 277 278func (err Error) Error() string { 279 pos := err.Pos 280 if pos == "" { 281 pos = "-" // like token.Position{}.String() 282 } 283 return pos + ": " + err.Msg 284} 285 286// flatPackage is the JSON form of Package 287// It drops all the type and syntax fields, and transforms the Imports 288// 289// TODO(adonovan): identify this struct with Package, effectively 290// publishing the JSON protocol. 291type flatPackage struct { 292 ID string 293 Name string `json:",omitempty"` 294 PkgPath string `json:",omitempty"` 295 Errors []Error `json:",omitempty"` 296 GoFiles []string `json:",omitempty"` 297 CompiledGoFiles []string `json:",omitempty"` 298 OtherFiles []string `json:",omitempty"` 299 ExportFile string `json:",omitempty"` 300 Imports map[string]string `json:",omitempty"` 301} 302 303// MarshalJSON returns the Package in its JSON form. 304// For the most part, the structure fields are written out unmodified, and 305// the type and syntax fields are skipped. 306// The imports are written out as just a map of path to package id. 307// The errors are written using a custom type that tries to preserve the 308// structure of error types we know about. 309// 310// This method exists to enable support for additional build systems. It is 311// not intended for use by clients of the API and we may change the format. 312func (p *Package) MarshalJSON() ([]byte, error) { 313 flat := &flatPackage{ 314 ID: p.ID, 315 Name: p.Name, 316 PkgPath: p.PkgPath, 317 Errors: p.Errors, 318 GoFiles: p.GoFiles, 319 CompiledGoFiles: p.CompiledGoFiles, 320 OtherFiles: p.OtherFiles, 321 ExportFile: p.ExportFile, 322 } 323 if len(p.Imports) > 0 { 324 flat.Imports = make(map[string]string, len(p.Imports)) 325 for path, ipkg := range p.Imports { 326 flat.Imports[path] = ipkg.ID 327 } 328 } 329 return json.Marshal(flat) 330} 331 332// UnmarshalJSON reads in a Package from its JSON format. 333// See MarshalJSON for details about the format accepted. 334func (p *Package) UnmarshalJSON(b []byte) error { 335 flat := &flatPackage{} 336 if err := json.Unmarshal(b, &flat); err != nil { 337 return err 338 } 339 *p = Package{ 340 ID: flat.ID, 341 Name: flat.Name, 342 PkgPath: flat.PkgPath, 343 Errors: flat.Errors, 344 GoFiles: flat.GoFiles, 345 CompiledGoFiles: flat.CompiledGoFiles, 346 OtherFiles: flat.OtherFiles, 347 ExportFile: flat.ExportFile, 348 } 349 if len(flat.Imports) > 0 { 350 p.Imports = make(map[string]*Package, len(flat.Imports)) 351 for path, id := range flat.Imports { 352 p.Imports[path] = &Package{ID: id} 353 } 354 } 355 return nil 356} 357 358func (p *Package) String() string { return p.ID } 359 360// loaderPackage augments Package with state used during the loading phase 361type loaderPackage struct { 362 *Package 363 importErrors map[string]error // maps each bad import to its error 364 loadOnce sync.Once 365 color uint8 // for cycle detection 366 needsrc bool // load from source (Mode >= LoadTypes) 367 needtypes bool // type information is either requested or depended on 368 initial bool // package was matched by a pattern 369} 370 371// loader holds the working state of a single call to load. 372type loader struct { 373 pkgs map[string]*loaderPackage 374 Config 375 sizes types.Sizes 376 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations 377} 378 379func newLoader(cfg *Config) *loader { 380 ld := &loader{} 381 if cfg != nil { 382 ld.Config = *cfg 383 } 384 if ld.Config.Env == nil { 385 ld.Config.Env = os.Environ() 386 } 387 if ld.Context == nil { 388 ld.Context = context.Background() 389 } 390 if ld.Dir == "" { 391 if dir, err := os.Getwd(); err == nil { 392 ld.Dir = dir 393 } 394 } 395 396 if ld.Mode >= LoadTypes { 397 if ld.Fset == nil { 398 ld.Fset = token.NewFileSet() 399 } 400 401 // ParseFile is required even in LoadTypes mode 402 // because we load source if export data is missing. 403 if ld.ParseFile == nil { 404 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) { 405 var isrc interface{} 406 if src != nil { 407 isrc = src 408 } 409 const mode = parser.AllErrors | parser.ParseComments 410 return parser.ParseFile(fset, filename, isrc, mode) 411 } 412 } 413 } 414 return ld 415} 416 417// refine connects the supplied packages into a graph and then adds type and 418// and syntax information as requested by the LoadMode. 419func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) { 420 rootMap := make(map[string]int, len(roots)) 421 for i, root := range roots { 422 rootMap[root] = i 423 } 424 ld.pkgs = make(map[string]*loaderPackage) 425 // first pass, fixup and build the map and roots 426 var initial = make([]*loaderPackage, len(roots)) 427 for _, pkg := range list { 428 rootIndex := -1 429 if i, found := rootMap[pkg.ID]; found { 430 rootIndex = i 431 } 432 lpkg := &loaderPackage{ 433 Package: pkg, 434 needtypes: ld.Mode >= LoadAllSyntax || 435 ld.Mode >= LoadTypes && rootIndex >= 0, 436 needsrc: ld.Mode >= LoadAllSyntax || 437 ld.Mode >= LoadSyntax && rootIndex >= 0 || 438 len(ld.Overlay) > 0 || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files 439 pkg.ExportFile == "" && pkg.PkgPath != "unsafe", 440 } 441 ld.pkgs[lpkg.ID] = lpkg 442 if rootIndex >= 0 { 443 initial[rootIndex] = lpkg 444 lpkg.initial = true 445 } 446 } 447 for i, root := range roots { 448 if initial[i] == nil { 449 return nil, fmt.Errorf("root package %v is missing", root) 450 } 451 } 452 453 // Materialize the import graph. 454 455 const ( 456 white = 0 // new 457 grey = 1 // in progress 458 black = 2 // complete 459 ) 460 461 // visit traverses the import graph, depth-first, 462 // and materializes the graph as Packages.Imports. 463 // 464 // Valid imports are saved in the Packages.Import map. 465 // Invalid imports (cycles and missing nodes) are saved in the importErrors map. 466 // Thus, even in the presence of both kinds of errors, the Import graph remains a DAG. 467 // 468 // visit returns whether the package needs src or has a transitive 469 // dependency on a package that does. These are the only packages 470 // for which we load source code. 471 var stack []*loaderPackage 472 var visit func(lpkg *loaderPackage) bool 473 var srcPkgs []*loaderPackage 474 visit = func(lpkg *loaderPackage) bool { 475 switch lpkg.color { 476 case black: 477 return lpkg.needsrc 478 case grey: 479 panic("internal error: grey node") 480 } 481 lpkg.color = grey 482 stack = append(stack, lpkg) // push 483 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports 484 lpkg.Imports = make(map[string]*Package, len(stubs)) 485 for importPath, ipkg := range stubs { 486 var importErr error 487 imp := ld.pkgs[ipkg.ID] 488 if imp == nil { 489 // (includes package "C" when DisableCgo) 490 importErr = fmt.Errorf("missing package: %q", ipkg.ID) 491 } else if imp.color == grey { 492 importErr = fmt.Errorf("import cycle: %s", stack) 493 } 494 if importErr != nil { 495 if lpkg.importErrors == nil { 496 lpkg.importErrors = make(map[string]error) 497 } 498 lpkg.importErrors[importPath] = importErr 499 continue 500 } 501 502 if visit(imp) { 503 lpkg.needsrc = true 504 } 505 lpkg.Imports[importPath] = imp.Package 506 } 507 if lpkg.needsrc { 508 srcPkgs = append(srcPkgs, lpkg) 509 } 510 stack = stack[:len(stack)-1] // pop 511 lpkg.color = black 512 513 return lpkg.needsrc 514 } 515 516 if ld.Mode < LoadImports { 517 //we do this to drop the stub import packages that we are not even going to try to resolve 518 for _, lpkg := range initial { 519 lpkg.Imports = nil 520 } 521 } else { 522 // For each initial package, create its import DAG. 523 for _, lpkg := range initial { 524 visit(lpkg) 525 } 526 } 527 for _, lpkg := range srcPkgs { 528 // Complete type information is required for the 529 // immediate dependencies of each source package. 530 for _, ipkg := range lpkg.Imports { 531 imp := ld.pkgs[ipkg.ID] 532 imp.needtypes = true 533 } 534 } 535 // Load type data if needed, starting at 536 // the initial packages (roots of the import DAG). 537 if ld.Mode >= LoadTypes { 538 var wg sync.WaitGroup 539 for _, lpkg := range initial { 540 wg.Add(1) 541 go func(lpkg *loaderPackage) { 542 ld.loadRecursive(lpkg) 543 wg.Done() 544 }(lpkg) 545 } 546 wg.Wait() 547 } 548 549 result := make([]*Package, len(initial)) 550 for i, lpkg := range initial { 551 result[i] = lpkg.Package 552 } 553 return result, nil 554} 555 556// loadRecursive loads the specified package and its dependencies, 557// recursively, in parallel, in topological order. 558// It is atomic and idempotent. 559// Precondition: ld.Mode >= LoadTypes. 560func (ld *loader) loadRecursive(lpkg *loaderPackage) { 561 lpkg.loadOnce.Do(func() { 562 // Load the direct dependencies, in parallel. 563 var wg sync.WaitGroup 564 for _, ipkg := range lpkg.Imports { 565 imp := ld.pkgs[ipkg.ID] 566 wg.Add(1) 567 go func(imp *loaderPackage) { 568 ld.loadRecursive(imp) 569 wg.Done() 570 }(imp) 571 } 572 wg.Wait() 573 574 ld.loadPackage(lpkg) 575 }) 576} 577 578// loadPackage loads the specified package. 579// It must be called only once per Package, 580// after immediate dependencies are loaded. 581// Precondition: ld.Mode >= LoadTypes. 582func (ld *loader) loadPackage(lpkg *loaderPackage) { 583 if lpkg.PkgPath == "unsafe" { 584 // Fill in the blanks to avoid surprises. 585 lpkg.Types = types.Unsafe 586 lpkg.Fset = ld.Fset 587 lpkg.Syntax = []*ast.File{} 588 lpkg.TypesInfo = new(types.Info) 589 lpkg.TypesSizes = ld.sizes 590 return 591 } 592 593 // Call NewPackage directly with explicit name. 594 // This avoids skew between golist and go/types when the files' 595 // package declarations are inconsistent. 596 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name) 597 lpkg.Fset = ld.Fset 598 599 // Subtle: we populate all Types fields with an empty Package 600 // before loading export data so that export data processing 601 // never has to create a types.Package for an indirect dependency, 602 // which would then require that such created packages be explicitly 603 // inserted back into the Import graph as a final step after export data loading. 604 // The Diamond test exercises this case. 605 if !lpkg.needtypes { 606 return 607 } 608 if !lpkg.needsrc { 609 ld.loadFromExportData(lpkg) 610 return // not a source package, don't get syntax trees 611 } 612 613 appendError := func(err error) { 614 // Convert various error types into the one true Error. 615 var errs []Error 616 switch err := err.(type) { 617 case Error: 618 // from driver 619 errs = append(errs, err) 620 621 case *os.PathError: 622 // from parser 623 errs = append(errs, Error{ 624 Pos: err.Path + ":1", 625 Msg: err.Err.Error(), 626 Kind: ParseError, 627 }) 628 629 case scanner.ErrorList: 630 // from parser 631 for _, err := range err { 632 errs = append(errs, Error{ 633 Pos: err.Pos.String(), 634 Msg: err.Msg, 635 Kind: ParseError, 636 }) 637 } 638 639 case types.Error: 640 // from type checker 641 errs = append(errs, Error{ 642 Pos: err.Fset.Position(err.Pos).String(), 643 Msg: err.Msg, 644 Kind: TypeError, 645 }) 646 647 default: 648 // unexpected impoverished error from parser? 649 errs = append(errs, Error{ 650 Pos: "-", 651 Msg: err.Error(), 652 Kind: UnknownError, 653 }) 654 655 // If you see this error message, please file a bug. 656 log.Printf("internal error: error %q (%T) without position", err, err) 657 } 658 659 lpkg.Errors = append(lpkg.Errors, errs...) 660 } 661 662 files, errs := ld.parseFiles(lpkg.CompiledGoFiles) 663 for _, err := range errs { 664 appendError(err) 665 } 666 667 lpkg.Syntax = files 668 669 lpkg.TypesInfo = &types.Info{ 670 Types: make(map[ast.Expr]types.TypeAndValue), 671 Defs: make(map[*ast.Ident]types.Object), 672 Uses: make(map[*ast.Ident]types.Object), 673 Implicits: make(map[ast.Node]types.Object), 674 Scopes: make(map[ast.Node]*types.Scope), 675 Selections: make(map[*ast.SelectorExpr]*types.Selection), 676 } 677 lpkg.TypesSizes = ld.sizes 678 679 importer := importerFunc(func(path string) (*types.Package, error) { 680 if path == "unsafe" { 681 return types.Unsafe, nil 682 } 683 684 // The imports map is keyed by import path. 685 ipkg := lpkg.Imports[path] 686 if ipkg == nil { 687 if err := lpkg.importErrors[path]; err != nil { 688 return nil, err 689 } 690 // There was skew between the metadata and the 691 // import declarations, likely due to an edit 692 // race, or because the ParseFile feature was 693 // used to supply alternative file contents. 694 return nil, fmt.Errorf("no metadata for %s", path) 695 } 696 697 if ipkg.Types != nil && ipkg.Types.Complete() { 698 return ipkg.Types, nil 699 } 700 log.Fatalf("internal error: nil Pkg importing %q from %q", path, lpkg) 701 panic("unreachable") 702 }) 703 704 // type-check 705 tc := &types.Config{ 706 Importer: importer, 707 708 // Type-check bodies of functions only in non-initial packages. 709 // Example: for import graph A->B->C and initial packages {A,C}, 710 // we can ignore function bodies in B. 711 IgnoreFuncBodies: ld.Mode < LoadAllSyntax && !lpkg.initial, 712 713 Error: appendError, 714 Sizes: ld.sizes, 715 } 716 types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax) 717 718 lpkg.importErrors = nil // no longer needed 719 720 // If !Cgo, the type-checker uses FakeImportC mode, so 721 // it doesn't invoke the importer for import "C", 722 // nor report an error for the import, 723 // or for any undefined C.f reference. 724 // We must detect this explicitly and correctly 725 // mark the package as IllTyped (by reporting an error). 726 // TODO(adonovan): if these errors are annoying, 727 // we could just set IllTyped quietly. 728 if tc.FakeImportC { 729 outer: 730 for _, f := range lpkg.Syntax { 731 for _, imp := range f.Imports { 732 if imp.Path.Value == `"C"` { 733 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`} 734 appendError(err) 735 break outer 736 } 737 } 738 } 739 } 740 741 // Record accumulated errors. 742 illTyped := len(lpkg.Errors) > 0 743 if !illTyped { 744 for _, imp := range lpkg.Imports { 745 if imp.IllTyped { 746 illTyped = true 747 break 748 } 749 } 750 } 751 lpkg.IllTyped = illTyped 752} 753 754// An importFunc is an implementation of the single-method 755// types.Importer interface based on a function value. 756type importerFunc func(path string) (*types.Package, error) 757 758func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } 759 760// We use a counting semaphore to limit 761// the number of parallel I/O calls per process. 762var ioLimit = make(chan bool, 20) 763 764// parseFiles reads and parses the Go source files and returns the ASTs 765// of the ones that could be at least partially parsed, along with a 766// list of I/O and parse errors encountered. 767// 768// Because files are scanned in parallel, the token.Pos 769// positions of the resulting ast.Files are not ordered. 770// 771func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { 772 var wg sync.WaitGroup 773 n := len(filenames) 774 parsed := make([]*ast.File, n) 775 errors := make([]error, n) 776 for i, file := range filenames { 777 if ld.Config.Context.Err() != nil { 778 parsed[i] = nil 779 errors[i] = ld.Config.Context.Err() 780 continue 781 } 782 wg.Add(1) 783 go func(i int, filename string) { 784 ioLimit <- true // wait 785 // ParseFile may return both an AST and an error. 786 var src []byte 787 for f, contents := range ld.Config.Overlay { 788 if sameFile(f, filename) { 789 src = contents 790 } 791 } 792 var err error 793 if src == nil { 794 src, err = ioutil.ReadFile(filename) 795 } 796 if err != nil { 797 parsed[i], errors[i] = nil, err 798 } else { 799 parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename, src) 800 } 801 <-ioLimit // signal 802 wg.Done() 803 }(i, file) 804 } 805 wg.Wait() 806 807 // Eliminate nils, preserving order. 808 var o int 809 for _, f := range parsed { 810 if f != nil { 811 parsed[o] = f 812 o++ 813 } 814 } 815 parsed = parsed[:o] 816 817 o = 0 818 for _, err := range errors { 819 if err != nil { 820 errors[o] = err 821 o++ 822 } 823 } 824 errors = errors[:o] 825 826 return parsed, errors 827} 828 829// sameFile returns true if x and y have the same basename and denote 830// the same file. 831// 832func sameFile(x, y string) bool { 833 if x == y { 834 // It could be the case that y doesn't exist. 835 // For instance, it may be an overlay file that 836 // hasn't been written to disk. To handle that case 837 // let x == y through. (We added the exact absolute path 838 // string to the CompiledGoFiles list, so the unwritten 839 // overlay case implies x==y.) 840 return true 841 } 842 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation) 843 if xi, err := os.Stat(x); err == nil { 844 if yi, err := os.Stat(y); err == nil { 845 return os.SameFile(xi, yi) 846 } 847 } 848 } 849 return false 850} 851 852// loadFromExportData returns type information for the specified 853// package, loading it from an export data file on the first request. 854func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) { 855 if lpkg.PkgPath == "" { 856 log.Fatalf("internal error: Package %s has no PkgPath", lpkg) 857 } 858 859 // Because gcexportdata.Read has the potential to create or 860 // modify the types.Package for each node in the transitive 861 // closure of dependencies of lpkg, all exportdata operations 862 // must be sequential. (Finer-grained locking would require 863 // changes to the gcexportdata API.) 864 // 865 // The exportMu lock guards the Package.Pkg field and the 866 // types.Package it points to, for each Package in the graph. 867 // 868 // Not all accesses to Package.Pkg need to be protected by exportMu: 869 // graph ordering ensures that direct dependencies of source 870 // packages are fully loaded before the importer reads their Pkg field. 871 ld.exportMu.Lock() 872 defer ld.exportMu.Unlock() 873 874 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() { 875 return tpkg, nil // cache hit 876 } 877 878 lpkg.IllTyped = true // fail safe 879 880 if lpkg.ExportFile == "" { 881 // Errors while building export data will have been printed to stderr. 882 return nil, fmt.Errorf("no export data file") 883 } 884 f, err := os.Open(lpkg.ExportFile) 885 if err != nil { 886 return nil, err 887 } 888 defer f.Close() 889 890 // Read gc export data. 891 // 892 // We don't currently support gccgo export data because all 893 // underlying workspaces use the gc toolchain. (Even build 894 // systems that support gccgo don't use it for workspace 895 // queries.) 896 r, err := gcexportdata.NewReader(f) 897 if err != nil { 898 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 899 } 900 901 // Build the view. 902 // 903 // The gcexportdata machinery has no concept of package ID. 904 // It identifies packages by their PkgPath, which although not 905 // globally unique is unique within the scope of one invocation 906 // of the linker, type-checker, or gcexportdata. 907 // 908 // So, we must build a PkgPath-keyed view of the global 909 // (conceptually ID-keyed) cache of packages and pass it to 910 // gcexportdata. The view must contain every existing 911 // package that might possibly be mentioned by the 912 // current package---its transitive closure. 913 // 914 // In loadPackage, we unconditionally create a types.Package for 915 // each dependency so that export data loading does not 916 // create new ones. 917 // 918 // TODO(adonovan): it would be simpler and more efficient 919 // if the export data machinery invoked a callback to 920 // get-or-create a package instead of a map. 921 // 922 view := make(map[string]*types.Package) // view seen by gcexportdata 923 seen := make(map[*loaderPackage]bool) // all visited packages 924 var visit func(pkgs map[string]*Package) 925 visit = func(pkgs map[string]*Package) { 926 for _, p := range pkgs { 927 lpkg := ld.pkgs[p.ID] 928 if !seen[lpkg] { 929 seen[lpkg] = true 930 view[lpkg.PkgPath] = lpkg.Types 931 visit(lpkg.Imports) 932 } 933 } 934 } 935 visit(lpkg.Imports) 936 937 viewLen := len(view) + 1 // adding the self package 938 // Parse the export data. 939 // (May modify incomplete packages in view but not create new ones.) 940 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath) 941 if err != nil { 942 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 943 } 944 if viewLen != len(view) { 945 log.Fatalf("Unexpected package creation during export data loading") 946 } 947 948 lpkg.Types = tpkg 949 lpkg.IllTyped = false 950 951 return tpkg, nil 952} 953 954func usesExportData(cfg *Config) bool { 955 return LoadTypes <= cfg.Mode && cfg.Mode < LoadAllSyntax 956} 957