1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package packages 6 7// See doc.go for package documentation and implementation notes. 8 9import ( 10 "context" 11 "encoding/json" 12 "fmt" 13 "go/ast" 14 "go/parser" 15 "go/scanner" 16 "go/token" 17 "go/types" 18 "io/ioutil" 19 "log" 20 "os" 21 "path/filepath" 22 "strings" 23 "sync" 24 "time" 25 26 "golang.org/x/tools/go/gcexportdata" 27 "golang.org/x/tools/internal/gocommand" 28 "golang.org/x/tools/internal/packagesinternal" 29 "golang.org/x/tools/internal/typesinternal" 30) 31 32// A LoadMode controls the amount of detail to return when loading. 33// The bits below can be combined to specify which fields should be 34// filled in the result packages. 35// The zero value is a special case, equivalent to combining 36// the NeedName, NeedFiles, and NeedCompiledGoFiles bits. 37// ID and Errors (if present) will always be filled. 38// Load may return more information than requested. 39type LoadMode int 40 41// TODO(matloob): When a V2 of go/packages is released, rename NeedExportsFile to 42// NeedExportFile to make it consistent with the Package field it's adding. 43 44const ( 45 // NeedName adds Name and PkgPath. 46 NeedName LoadMode = 1 << iota 47 48 // NeedFiles adds GoFiles and OtherFiles. 49 NeedFiles 50 51 // NeedCompiledGoFiles adds CompiledGoFiles. 52 NeedCompiledGoFiles 53 54 // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain 55 // "placeholder" Packages with only the ID set. 56 NeedImports 57 58 // NeedDeps adds the fields requested by the LoadMode in the packages in Imports. 59 NeedDeps 60 61 // NeedExportsFile adds ExportFile. 62 NeedExportsFile 63 64 // NeedTypes adds Types, Fset, and IllTyped. 65 NeedTypes 66 67 // NeedSyntax adds Syntax. 68 NeedSyntax 69 70 // NeedTypesInfo adds TypesInfo. 71 NeedTypesInfo 72 73 // NeedTypesSizes adds TypesSizes. 74 NeedTypesSizes 75 76 // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+. 77 // Modifies CompiledGoFiles and Types, and has no effect on its own. 78 typecheckCgo 79 80 // NeedModule adds Module. 81 NeedModule 82) 83 84const ( 85 // Deprecated: LoadFiles exists for historical compatibility 86 // and should not be used. Please directly specify the needed fields using the Need values. 87 LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles 88 89 // Deprecated: LoadImports exists for historical compatibility 90 // and should not be used. Please directly specify the needed fields using the Need values. 91 LoadImports = LoadFiles | NeedImports 92 93 // Deprecated: LoadTypes exists for historical compatibility 94 // and should not be used. Please directly specify the needed fields using the Need values. 95 LoadTypes = LoadImports | NeedTypes | NeedTypesSizes 96 97 // Deprecated: LoadSyntax exists for historical compatibility 98 // and should not be used. Please directly specify the needed fields using the Need values. 99 LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo 100 101 // Deprecated: LoadAllSyntax exists for historical compatibility 102 // and should not be used. Please directly specify the needed fields using the Need values. 103 LoadAllSyntax = LoadSyntax | NeedDeps 104) 105 106// A Config specifies details about how packages should be loaded. 107// The zero value is a valid configuration. 108// Calls to Load do not modify this struct. 109type Config struct { 110 // Mode controls the level of information returned for each package. 111 Mode LoadMode 112 113 // Context specifies the context for the load operation. 114 // If the context is cancelled, the loader may stop early 115 // and return an ErrCancelled error. 116 // If Context is nil, the load cannot be cancelled. 117 Context context.Context 118 119 // Logf is the logger for the config. 120 // If the user provides a logger, debug logging is enabled. 121 // If the GOPACKAGESDEBUG environment variable is set to true, 122 // but the logger is nil, default to log.Printf. 123 Logf func(format string, args ...interface{}) 124 125 // Dir is the directory in which to run the build system's query tool 126 // that provides information about the packages. 127 // If Dir is empty, the tool is run in the current directory. 128 Dir string 129 130 // Env is the environment to use when invoking the build system's query tool. 131 // If Env is nil, the current environment is used. 132 // As in os/exec's Cmd, only the last value in the slice for 133 // each environment key is used. To specify the setting of only 134 // a few variables, append to the current environment, as in: 135 // 136 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386") 137 // 138 Env []string 139 140 // gocmdRunner guards go command calls from concurrency errors. 141 gocmdRunner *gocommand.Runner 142 143 // BuildFlags is a list of command-line flags to be passed through to 144 // the build system's query tool. 145 BuildFlags []string 146 147 // modFile will be used for -modfile in go command invocations. 148 modFile string 149 150 // modFlag will be used for -modfile in go command invocations. 151 modFlag string 152 153 // Fset provides source position information for syntax trees and types. 154 // If Fset is nil, Load will use a new fileset, but preserve Fset's value. 155 Fset *token.FileSet 156 157 // ParseFile is called to read and parse each file 158 // when preparing a package's type-checked syntax tree. 159 // It must be safe to call ParseFile simultaneously from multiple goroutines. 160 // If ParseFile is nil, the loader will uses parser.ParseFile. 161 // 162 // ParseFile should parse the source from src and use filename only for 163 // recording position information. 164 // 165 // An application may supply a custom implementation of ParseFile 166 // to change the effective file contents or the behavior of the parser, 167 // or to modify the syntax tree. For example, selectively eliminating 168 // unwanted function bodies can significantly accelerate type checking. 169 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) 170 171 // If Tests is set, the loader includes not just the packages 172 // matching a particular pattern but also any related test packages, 173 // including test-only variants of the package and the test executable. 174 // 175 // For example, when using the go command, loading "fmt" with Tests=true 176 // returns four packages, with IDs "fmt" (the standard package), 177 // "fmt [fmt.test]" (the package as compiled for the test), 178 // "fmt_test" (the test functions from source files in package fmt_test), 179 // and "fmt.test" (the test binary). 180 // 181 // In build systems with explicit names for tests, 182 // setting Tests may have no effect. 183 Tests bool 184 185 // Overlay provides a mapping of absolute file paths to file contents. 186 // If the file with the given path already exists, the parser will use the 187 // alternative file contents provided by the map. 188 // 189 // Overlays provide incomplete support for when a given file doesn't 190 // already exist on disk. See the package doc above for more details. 191 Overlay map[string][]byte 192} 193 194// driver is the type for functions that query the build system for the 195// packages named by the patterns. 196type driver func(cfg *Config, patterns ...string) (*driverResponse, error) 197 198// driverResponse contains the results for a driver query. 199type driverResponse struct { 200 // NotHandled is returned if the request can't be handled by the current 201 // driver. If an external driver returns a response with NotHandled, the 202 // rest of the driverResponse is ignored, and go/packages will fallback 203 // to the next driver. If go/packages is extended in the future to support 204 // lists of multiple drivers, go/packages will fall back to the next driver. 205 NotHandled bool 206 207 // Sizes, if not nil, is the types.Sizes to use when type checking. 208 Sizes *types.StdSizes 209 210 // Roots is the set of package IDs that make up the root packages. 211 // We have to encode this separately because when we encode a single package 212 // we cannot know if it is one of the roots as that requires knowledge of the 213 // graph it is part of. 214 Roots []string `json:",omitempty"` 215 216 // Packages is the full set of packages in the graph. 217 // The packages are not connected into a graph. 218 // The Imports if populated will be stubs that only have their ID set. 219 // Imports will be connected and then type and syntax information added in a 220 // later pass (see refine). 221 Packages []*Package 222} 223 224// Load loads and returns the Go packages named by the given patterns. 225// 226// Config specifies loading options; 227// nil behaves the same as an empty Config. 228// 229// Load returns an error if any of the patterns was invalid 230// as defined by the underlying build system. 231// It may return an empty list of packages without an error, 232// for instance for an empty expansion of a valid wildcard. 233// Errors associated with a particular package are recorded in the 234// corresponding Package's Errors list, and do not cause Load to 235// return an error. Clients may need to handle such errors before 236// proceeding with further analysis. The PrintErrors function is 237// provided for convenient display of all errors. 238func Load(cfg *Config, patterns ...string) ([]*Package, error) { 239 l := newLoader(cfg) 240 response, err := defaultDriver(&l.Config, patterns...) 241 if err != nil { 242 return nil, err 243 } 244 l.sizes = response.Sizes 245 return l.refine(response.Roots, response.Packages...) 246} 247 248// defaultDriver is a driver that implements go/packages' fallback behavior. 249// It will try to request to an external driver, if one exists. If there's 250// no external driver, or the driver returns a response with NotHandled set, 251// defaultDriver will fall back to the go list driver. 252func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) { 253 driver := findExternalDriver(cfg) 254 if driver == nil { 255 driver = goListDriver 256 } 257 response, err := driver(cfg, patterns...) 258 if err != nil { 259 return response, err 260 } else if response.NotHandled { 261 return goListDriver(cfg, patterns...) 262 } 263 return response, nil 264} 265 266// A Package describes a loaded Go package. 267type Package struct { 268 // ID is a unique identifier for a package, 269 // in a syntax provided by the underlying build system. 270 // 271 // Because the syntax varies based on the build system, 272 // clients should treat IDs as opaque and not attempt to 273 // interpret them. 274 ID string 275 276 // Name is the package name as it appears in the package source code. 277 Name string 278 279 // PkgPath is the package path as used by the go/types package. 280 PkgPath string 281 282 // Errors contains any errors encountered querying the metadata 283 // of the package, or while parsing or type-checking its files. 284 Errors []Error 285 286 // GoFiles lists the absolute file paths of the package's Go source files. 287 GoFiles []string 288 289 // CompiledGoFiles lists the absolute file paths of the package's source 290 // files that are suitable for type checking. 291 // This may differ from GoFiles if files are processed before compilation. 292 CompiledGoFiles []string 293 294 // OtherFiles lists the absolute file paths of the package's non-Go source files, 295 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on. 296 OtherFiles []string 297 298 // IgnoredFiles lists source files that are not part of the package 299 // using the current build configuration but that might be part of 300 // the package using other build configurations. 301 IgnoredFiles []string 302 303 // ExportFile is the absolute path to a file containing type 304 // information for the package as provided by the build system. 305 ExportFile string 306 307 // Imports maps import paths appearing in the package's Go source files 308 // to corresponding loaded Packages. 309 Imports map[string]*Package 310 311 // Types provides type information for the package. 312 // The NeedTypes LoadMode bit sets this field for packages matching the 313 // patterns; type information for dependencies may be missing or incomplete, 314 // unless NeedDeps and NeedImports are also set. 315 Types *types.Package 316 317 // Fset provides position information for Types, TypesInfo, and Syntax. 318 // It is set only when Types is set. 319 Fset *token.FileSet 320 321 // IllTyped indicates whether the package or any dependency contains errors. 322 // It is set only when Types is set. 323 IllTyped bool 324 325 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles. 326 // 327 // The NeedSyntax LoadMode bit populates this field for packages matching the patterns. 328 // If NeedDeps and NeedImports are also set, this field will also be populated 329 // for dependencies. 330 Syntax []*ast.File 331 332 // TypesInfo provides type information about the package's syntax trees. 333 // It is set only when Syntax is set. 334 TypesInfo *types.Info 335 336 // TypesSizes provides the effective size function for types in TypesInfo. 337 TypesSizes types.Sizes 338 339 // forTest is the package under test, if any. 340 forTest string 341 342 // module is the module information for the package if it exists. 343 Module *Module 344} 345 346// Module provides module information for a package. 347type Module struct { 348 Path string // module path 349 Version string // module version 350 Replace *Module // replaced by this module 351 Time *time.Time // time version was created 352 Main bool // is this the main module? 353 Indirect bool // is this module only an indirect dependency of main module? 354 Dir string // directory holding files for this module, if any 355 GoMod string // path to go.mod file used when loading this module, if any 356 GoVersion string // go version used in module 357 Error *ModuleError // error loading module 358} 359 360// ModuleError holds errors loading a module. 361type ModuleError struct { 362 Err string // the error itself 363} 364 365func init() { 366 packagesinternal.GetForTest = func(p interface{}) string { 367 return p.(*Package).forTest 368 } 369 packagesinternal.GetGoCmdRunner = func(config interface{}) *gocommand.Runner { 370 return config.(*Config).gocmdRunner 371 } 372 packagesinternal.SetGoCmdRunner = func(config interface{}, runner *gocommand.Runner) { 373 config.(*Config).gocmdRunner = runner 374 } 375 packagesinternal.SetModFile = func(config interface{}, value string) { 376 config.(*Config).modFile = value 377 } 378 packagesinternal.SetModFlag = func(config interface{}, value string) { 379 config.(*Config).modFlag = value 380 } 381 packagesinternal.TypecheckCgo = int(typecheckCgo) 382} 383 384// An Error describes a problem with a package's metadata, syntax, or types. 385type Error struct { 386 Pos string // "file:line:col" or "file:line" or "" or "-" 387 Msg string 388 Kind ErrorKind 389} 390 391// ErrorKind describes the source of the error, allowing the user to 392// differentiate between errors generated by the driver, the parser, or the 393// type-checker. 394type ErrorKind int 395 396const ( 397 UnknownError ErrorKind = iota 398 ListError 399 ParseError 400 TypeError 401) 402 403func (err Error) Error() string { 404 pos := err.Pos 405 if pos == "" { 406 pos = "-" // like token.Position{}.String() 407 } 408 return pos + ": " + err.Msg 409} 410 411// flatPackage is the JSON form of Package 412// It drops all the type and syntax fields, and transforms the Imports 413// 414// TODO(adonovan): identify this struct with Package, effectively 415// publishing the JSON protocol. 416type flatPackage struct { 417 ID string 418 Name string `json:",omitempty"` 419 PkgPath string `json:",omitempty"` 420 Errors []Error `json:",omitempty"` 421 GoFiles []string `json:",omitempty"` 422 CompiledGoFiles []string `json:",omitempty"` 423 OtherFiles []string `json:",omitempty"` 424 IgnoredFiles []string `json:",omitempty"` 425 ExportFile string `json:",omitempty"` 426 Imports map[string]string `json:",omitempty"` 427} 428 429// MarshalJSON returns the Package in its JSON form. 430// For the most part, the structure fields are written out unmodified, and 431// the type and syntax fields are skipped. 432// The imports are written out as just a map of path to package id. 433// The errors are written using a custom type that tries to preserve the 434// structure of error types we know about. 435// 436// This method exists to enable support for additional build systems. It is 437// not intended for use by clients of the API and we may change the format. 438func (p *Package) MarshalJSON() ([]byte, error) { 439 flat := &flatPackage{ 440 ID: p.ID, 441 Name: p.Name, 442 PkgPath: p.PkgPath, 443 Errors: p.Errors, 444 GoFiles: p.GoFiles, 445 CompiledGoFiles: p.CompiledGoFiles, 446 OtherFiles: p.OtherFiles, 447 IgnoredFiles: p.IgnoredFiles, 448 ExportFile: p.ExportFile, 449 } 450 if len(p.Imports) > 0 { 451 flat.Imports = make(map[string]string, len(p.Imports)) 452 for path, ipkg := range p.Imports { 453 flat.Imports[path] = ipkg.ID 454 } 455 } 456 return json.Marshal(flat) 457} 458 459// UnmarshalJSON reads in a Package from its JSON format. 460// See MarshalJSON for details about the format accepted. 461func (p *Package) UnmarshalJSON(b []byte) error { 462 flat := &flatPackage{} 463 if err := json.Unmarshal(b, &flat); err != nil { 464 return err 465 } 466 *p = Package{ 467 ID: flat.ID, 468 Name: flat.Name, 469 PkgPath: flat.PkgPath, 470 Errors: flat.Errors, 471 GoFiles: flat.GoFiles, 472 CompiledGoFiles: flat.CompiledGoFiles, 473 OtherFiles: flat.OtherFiles, 474 ExportFile: flat.ExportFile, 475 } 476 if len(flat.Imports) > 0 { 477 p.Imports = make(map[string]*Package, len(flat.Imports)) 478 for path, id := range flat.Imports { 479 p.Imports[path] = &Package{ID: id} 480 } 481 } 482 return nil 483} 484 485func (p *Package) String() string { return p.ID } 486 487// loaderPackage augments Package with state used during the loading phase 488type loaderPackage struct { 489 *Package 490 importErrors map[string]error // maps each bad import to its error 491 loadOnce sync.Once 492 color uint8 // for cycle detection 493 needsrc bool // load from source (Mode >= LoadTypes) 494 needtypes bool // type information is either requested or depended on 495 initial bool // package was matched by a pattern 496} 497 498// loader holds the working state of a single call to load. 499type loader struct { 500 pkgs map[string]*loaderPackage 501 Config 502 sizes types.Sizes 503 parseCache map[string]*parseValue 504 parseCacheMu sync.Mutex 505 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations 506 507 // Config.Mode contains the implied mode (see impliedLoadMode). 508 // Implied mode contains all the fields we need the data for. 509 // In requestedMode there are the actually requested fields. 510 // We'll zero them out before returning packages to the user. 511 // This makes it easier for us to get the conditions where 512 // we need certain modes right. 513 requestedMode LoadMode 514} 515 516type parseValue struct { 517 f *ast.File 518 err error 519 ready chan struct{} 520} 521 522func newLoader(cfg *Config) *loader { 523 ld := &loader{ 524 parseCache: map[string]*parseValue{}, 525 } 526 if cfg != nil { 527 ld.Config = *cfg 528 // If the user has provided a logger, use it. 529 ld.Config.Logf = cfg.Logf 530 } 531 if ld.Config.Logf == nil { 532 // If the GOPACKAGESDEBUG environment variable is set to true, 533 // but the user has not provided a logger, default to log.Printf. 534 if debug { 535 ld.Config.Logf = log.Printf 536 } else { 537 ld.Config.Logf = func(format string, args ...interface{}) {} 538 } 539 } 540 if ld.Config.Mode == 0 { 541 ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility. 542 } 543 if ld.Config.Env == nil { 544 ld.Config.Env = os.Environ() 545 } 546 if ld.Config.gocmdRunner == nil { 547 ld.Config.gocmdRunner = &gocommand.Runner{} 548 } 549 if ld.Context == nil { 550 ld.Context = context.Background() 551 } 552 if ld.Dir == "" { 553 if dir, err := os.Getwd(); err == nil { 554 ld.Dir = dir 555 } 556 } 557 558 // Save the actually requested fields. We'll zero them out before returning packages to the user. 559 ld.requestedMode = ld.Mode 560 ld.Mode = impliedLoadMode(ld.Mode) 561 562 if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 { 563 if ld.Fset == nil { 564 ld.Fset = token.NewFileSet() 565 } 566 567 // ParseFile is required even in LoadTypes mode 568 // because we load source if export data is missing. 569 if ld.ParseFile == nil { 570 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) { 571 const mode = parser.AllErrors | parser.ParseComments 572 return parser.ParseFile(fset, filename, src, mode) 573 } 574 } 575 } 576 577 return ld 578} 579 580// refine connects the supplied packages into a graph and then adds type and 581// and syntax information as requested by the LoadMode. 582func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) { 583 rootMap := make(map[string]int, len(roots)) 584 for i, root := range roots { 585 rootMap[root] = i 586 } 587 ld.pkgs = make(map[string]*loaderPackage) 588 // first pass, fixup and build the map and roots 589 var initial = make([]*loaderPackage, len(roots)) 590 for _, pkg := range list { 591 rootIndex := -1 592 if i, found := rootMap[pkg.ID]; found { 593 rootIndex = i 594 } 595 596 // Overlays can invalidate export data. 597 // TODO(matloob): make this check fine-grained based on dependencies on overlaid files 598 exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe" 599 // This package needs type information if the caller requested types and the package is 600 // either a root, or it's a non-root and the user requested dependencies ... 601 needtypes := (ld.Mode&NeedTypes|NeedTypesInfo != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) 602 // This package needs source if the call requested source (or types info, which implies source) 603 // and the package is either a root, or itas a non- root and the user requested dependencies... 604 needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) || 605 // ... or if we need types and the exportData is invalid. We fall back to (incompletely) 606 // typechecking packages from source if they fail to compile. 607 (ld.Mode&NeedTypes|NeedTypesInfo != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe" 608 lpkg := &loaderPackage{ 609 Package: pkg, 610 needtypes: needtypes, 611 needsrc: needsrc, 612 } 613 ld.pkgs[lpkg.ID] = lpkg 614 if rootIndex >= 0 { 615 initial[rootIndex] = lpkg 616 lpkg.initial = true 617 } 618 } 619 for i, root := range roots { 620 if initial[i] == nil { 621 return nil, fmt.Errorf("root package %v is missing", root) 622 } 623 } 624 625 // Materialize the import graph. 626 627 const ( 628 white = 0 // new 629 grey = 1 // in progress 630 black = 2 // complete 631 ) 632 633 // visit traverses the import graph, depth-first, 634 // and materializes the graph as Packages.Imports. 635 // 636 // Valid imports are saved in the Packages.Import map. 637 // Invalid imports (cycles and missing nodes) are saved in the importErrors map. 638 // Thus, even in the presence of both kinds of errors, the Import graph remains a DAG. 639 // 640 // visit returns whether the package needs src or has a transitive 641 // dependency on a package that does. These are the only packages 642 // for which we load source code. 643 var stack []*loaderPackage 644 var visit func(lpkg *loaderPackage) bool 645 var srcPkgs []*loaderPackage 646 visit = func(lpkg *loaderPackage) bool { 647 switch lpkg.color { 648 case black: 649 return lpkg.needsrc 650 case grey: 651 panic("internal error: grey node") 652 } 653 lpkg.color = grey 654 stack = append(stack, lpkg) // push 655 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports 656 // If NeedImports isn't set, the imports fields will all be zeroed out. 657 if ld.Mode&NeedImports != 0 { 658 lpkg.Imports = make(map[string]*Package, len(stubs)) 659 for importPath, ipkg := range stubs { 660 var importErr error 661 imp := ld.pkgs[ipkg.ID] 662 if imp == nil { 663 // (includes package "C" when DisableCgo) 664 importErr = fmt.Errorf("missing package: %q", ipkg.ID) 665 } else if imp.color == grey { 666 importErr = fmt.Errorf("import cycle: %s", stack) 667 } 668 if importErr != nil { 669 if lpkg.importErrors == nil { 670 lpkg.importErrors = make(map[string]error) 671 } 672 lpkg.importErrors[importPath] = importErr 673 continue 674 } 675 676 if visit(imp) { 677 lpkg.needsrc = true 678 } 679 lpkg.Imports[importPath] = imp.Package 680 } 681 } 682 if lpkg.needsrc { 683 srcPkgs = append(srcPkgs, lpkg) 684 } 685 if ld.Mode&NeedTypesSizes != 0 { 686 lpkg.TypesSizes = ld.sizes 687 } 688 stack = stack[:len(stack)-1] // pop 689 lpkg.color = black 690 691 return lpkg.needsrc 692 } 693 694 if ld.Mode&NeedImports == 0 { 695 // We do this to drop the stub import packages that we are not even going to try to resolve. 696 for _, lpkg := range initial { 697 lpkg.Imports = nil 698 } 699 } else { 700 // For each initial package, create its import DAG. 701 for _, lpkg := range initial { 702 visit(lpkg) 703 } 704 } 705 if ld.Mode&NeedImports != 0 && ld.Mode&NeedTypes != 0 { 706 for _, lpkg := range srcPkgs { 707 // Complete type information is required for the 708 // immediate dependencies of each source package. 709 for _, ipkg := range lpkg.Imports { 710 imp := ld.pkgs[ipkg.ID] 711 imp.needtypes = true 712 } 713 } 714 } 715 // Load type data and syntax if needed, starting at 716 // the initial packages (roots of the import DAG). 717 if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 { 718 var wg sync.WaitGroup 719 for _, lpkg := range initial { 720 wg.Add(1) 721 go func(lpkg *loaderPackage) { 722 ld.loadRecursive(lpkg) 723 wg.Done() 724 }(lpkg) 725 } 726 wg.Wait() 727 } 728 729 result := make([]*Package, len(initial)) 730 for i, lpkg := range initial { 731 result[i] = lpkg.Package 732 } 733 for i := range ld.pkgs { 734 // Clear all unrequested fields, 735 // to catch programs that use more than they request. 736 if ld.requestedMode&NeedName == 0 { 737 ld.pkgs[i].Name = "" 738 ld.pkgs[i].PkgPath = "" 739 } 740 if ld.requestedMode&NeedFiles == 0 { 741 ld.pkgs[i].GoFiles = nil 742 ld.pkgs[i].OtherFiles = nil 743 ld.pkgs[i].IgnoredFiles = nil 744 } 745 if ld.requestedMode&NeedCompiledGoFiles == 0 { 746 ld.pkgs[i].CompiledGoFiles = nil 747 } 748 if ld.requestedMode&NeedImports == 0 { 749 ld.pkgs[i].Imports = nil 750 } 751 if ld.requestedMode&NeedExportsFile == 0 { 752 ld.pkgs[i].ExportFile = "" 753 } 754 if ld.requestedMode&NeedTypes == 0 { 755 ld.pkgs[i].Types = nil 756 ld.pkgs[i].Fset = nil 757 ld.pkgs[i].IllTyped = false 758 } 759 if ld.requestedMode&NeedSyntax == 0 { 760 ld.pkgs[i].Syntax = nil 761 } 762 if ld.requestedMode&NeedTypesInfo == 0 { 763 ld.pkgs[i].TypesInfo = nil 764 } 765 if ld.requestedMode&NeedTypesSizes == 0 { 766 ld.pkgs[i].TypesSizes = nil 767 } 768 if ld.requestedMode&NeedModule == 0 { 769 ld.pkgs[i].Module = nil 770 } 771 } 772 773 return result, nil 774} 775 776// loadRecursive loads the specified package and its dependencies, 777// recursively, in parallel, in topological order. 778// It is atomic and idempotent. 779// Precondition: ld.Mode&NeedTypes. 780func (ld *loader) loadRecursive(lpkg *loaderPackage) { 781 lpkg.loadOnce.Do(func() { 782 // Load the direct dependencies, in parallel. 783 var wg sync.WaitGroup 784 for _, ipkg := range lpkg.Imports { 785 imp := ld.pkgs[ipkg.ID] 786 wg.Add(1) 787 go func(imp *loaderPackage) { 788 ld.loadRecursive(imp) 789 wg.Done() 790 }(imp) 791 } 792 wg.Wait() 793 ld.loadPackage(lpkg) 794 }) 795} 796 797// loadPackage loads the specified package. 798// It must be called only once per Package, 799// after immediate dependencies are loaded. 800// Precondition: ld.Mode & NeedTypes. 801func (ld *loader) loadPackage(lpkg *loaderPackage) { 802 if lpkg.PkgPath == "unsafe" { 803 // Fill in the blanks to avoid surprises. 804 lpkg.Types = types.Unsafe 805 lpkg.Fset = ld.Fset 806 lpkg.Syntax = []*ast.File{} 807 lpkg.TypesInfo = new(types.Info) 808 lpkg.TypesSizes = ld.sizes 809 return 810 } 811 812 // Call NewPackage directly with explicit name. 813 // This avoids skew between golist and go/types when the files' 814 // package declarations are inconsistent. 815 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name) 816 lpkg.Fset = ld.Fset 817 818 // Subtle: we populate all Types fields with an empty Package 819 // before loading export data so that export data processing 820 // never has to create a types.Package for an indirect dependency, 821 // which would then require that such created packages be explicitly 822 // inserted back into the Import graph as a final step after export data loading. 823 // The Diamond test exercises this case. 824 if !lpkg.needtypes && !lpkg.needsrc { 825 return 826 } 827 if !lpkg.needsrc { 828 ld.loadFromExportData(lpkg) 829 return // not a source package, don't get syntax trees 830 } 831 832 appendError := func(err error) { 833 // Convert various error types into the one true Error. 834 var errs []Error 835 switch err := err.(type) { 836 case Error: 837 // from driver 838 errs = append(errs, err) 839 840 case *os.PathError: 841 // from parser 842 errs = append(errs, Error{ 843 Pos: err.Path + ":1", 844 Msg: err.Err.Error(), 845 Kind: ParseError, 846 }) 847 848 case scanner.ErrorList: 849 // from parser 850 for _, err := range err { 851 errs = append(errs, Error{ 852 Pos: err.Pos.String(), 853 Msg: err.Msg, 854 Kind: ParseError, 855 }) 856 } 857 858 case types.Error: 859 // from type checker 860 errs = append(errs, Error{ 861 Pos: err.Fset.Position(err.Pos).String(), 862 Msg: err.Msg, 863 Kind: TypeError, 864 }) 865 866 default: 867 // unexpected impoverished error from parser? 868 errs = append(errs, Error{ 869 Pos: "-", 870 Msg: err.Error(), 871 Kind: UnknownError, 872 }) 873 874 // If you see this error message, please file a bug. 875 log.Printf("internal error: error %q (%T) without position", err, err) 876 } 877 878 lpkg.Errors = append(lpkg.Errors, errs...) 879 } 880 881 if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" { 882 // The config requested loading sources and types, but sources are missing. 883 // Add an error to the package and fall back to loading from export data. 884 appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError}) 885 ld.loadFromExportData(lpkg) 886 return // can't get syntax trees for this package 887 } 888 889 files, errs := ld.parseFiles(lpkg.CompiledGoFiles) 890 for _, err := range errs { 891 appendError(err) 892 } 893 894 lpkg.Syntax = files 895 if ld.Config.Mode&NeedTypes == 0 { 896 return 897 } 898 899 lpkg.TypesInfo = &types.Info{ 900 Types: make(map[ast.Expr]types.TypeAndValue), 901 Defs: make(map[*ast.Ident]types.Object), 902 Uses: make(map[*ast.Ident]types.Object), 903 Implicits: make(map[ast.Node]types.Object), 904 Scopes: make(map[ast.Node]*types.Scope), 905 Selections: make(map[*ast.SelectorExpr]*types.Selection), 906 } 907 lpkg.TypesSizes = ld.sizes 908 909 importer := importerFunc(func(path string) (*types.Package, error) { 910 if path == "unsafe" { 911 return types.Unsafe, nil 912 } 913 914 // The imports map is keyed by import path. 915 ipkg := lpkg.Imports[path] 916 if ipkg == nil { 917 if err := lpkg.importErrors[path]; err != nil { 918 return nil, err 919 } 920 // There was skew between the metadata and the 921 // import declarations, likely due to an edit 922 // race, or because the ParseFile feature was 923 // used to supply alternative file contents. 924 return nil, fmt.Errorf("no metadata for %s", path) 925 } 926 927 if ipkg.Types != nil && ipkg.Types.Complete() { 928 return ipkg.Types, nil 929 } 930 log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg) 931 panic("unreachable") 932 }) 933 934 // type-check 935 tc := &types.Config{ 936 Importer: importer, 937 938 // Type-check bodies of functions only in non-initial packages. 939 // Example: for import graph A->B->C and initial packages {A,C}, 940 // we can ignore function bodies in B. 941 IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial, 942 943 Error: appendError, 944 Sizes: ld.sizes, 945 } 946 if (ld.Mode & typecheckCgo) != 0 { 947 if !typesinternal.SetUsesCgo(tc) { 948 appendError(Error{ 949 Msg: "typecheckCgo requires Go 1.15+", 950 Kind: ListError, 951 }) 952 return 953 } 954 } 955 types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax) 956 957 lpkg.importErrors = nil // no longer needed 958 959 // If !Cgo, the type-checker uses FakeImportC mode, so 960 // it doesn't invoke the importer for import "C", 961 // nor report an error for the import, 962 // or for any undefined C.f reference. 963 // We must detect this explicitly and correctly 964 // mark the package as IllTyped (by reporting an error). 965 // TODO(adonovan): if these errors are annoying, 966 // we could just set IllTyped quietly. 967 if tc.FakeImportC { 968 outer: 969 for _, f := range lpkg.Syntax { 970 for _, imp := range f.Imports { 971 if imp.Path.Value == `"C"` { 972 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`} 973 appendError(err) 974 break outer 975 } 976 } 977 } 978 } 979 980 // Record accumulated errors. 981 illTyped := len(lpkg.Errors) > 0 982 if !illTyped { 983 for _, imp := range lpkg.Imports { 984 if imp.IllTyped { 985 illTyped = true 986 break 987 } 988 } 989 } 990 lpkg.IllTyped = illTyped 991} 992 993// An importFunc is an implementation of the single-method 994// types.Importer interface based on a function value. 995type importerFunc func(path string) (*types.Package, error) 996 997func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } 998 999// We use a counting semaphore to limit 1000// the number of parallel I/O calls per process. 1001var ioLimit = make(chan bool, 20) 1002 1003func (ld *loader) parseFile(filename string) (*ast.File, error) { 1004 ld.parseCacheMu.Lock() 1005 v, ok := ld.parseCache[filename] 1006 if ok { 1007 // cache hit 1008 ld.parseCacheMu.Unlock() 1009 <-v.ready 1010 } else { 1011 // cache miss 1012 v = &parseValue{ready: make(chan struct{})} 1013 ld.parseCache[filename] = v 1014 ld.parseCacheMu.Unlock() 1015 1016 var src []byte 1017 for f, contents := range ld.Config.Overlay { 1018 if sameFile(f, filename) { 1019 src = contents 1020 } 1021 } 1022 var err error 1023 if src == nil { 1024 ioLimit <- true // wait 1025 src, err = ioutil.ReadFile(filename) 1026 <-ioLimit // signal 1027 } 1028 if err != nil { 1029 v.err = err 1030 } else { 1031 v.f, v.err = ld.ParseFile(ld.Fset, filename, src) 1032 } 1033 1034 close(v.ready) 1035 } 1036 return v.f, v.err 1037} 1038 1039// parseFiles reads and parses the Go source files and returns the ASTs 1040// of the ones that could be at least partially parsed, along with a 1041// list of I/O and parse errors encountered. 1042// 1043// Because files are scanned in parallel, the token.Pos 1044// positions of the resulting ast.Files are not ordered. 1045// 1046func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { 1047 var wg sync.WaitGroup 1048 n := len(filenames) 1049 parsed := make([]*ast.File, n) 1050 errors := make([]error, n) 1051 for i, file := range filenames { 1052 if ld.Config.Context.Err() != nil { 1053 parsed[i] = nil 1054 errors[i] = ld.Config.Context.Err() 1055 continue 1056 } 1057 wg.Add(1) 1058 go func(i int, filename string) { 1059 parsed[i], errors[i] = ld.parseFile(filename) 1060 wg.Done() 1061 }(i, file) 1062 } 1063 wg.Wait() 1064 1065 // Eliminate nils, preserving order. 1066 var o int 1067 for _, f := range parsed { 1068 if f != nil { 1069 parsed[o] = f 1070 o++ 1071 } 1072 } 1073 parsed = parsed[:o] 1074 1075 o = 0 1076 for _, err := range errors { 1077 if err != nil { 1078 errors[o] = err 1079 o++ 1080 } 1081 } 1082 errors = errors[:o] 1083 1084 return parsed, errors 1085} 1086 1087// sameFile returns true if x and y have the same basename and denote 1088// the same file. 1089// 1090func sameFile(x, y string) bool { 1091 if x == y { 1092 // It could be the case that y doesn't exist. 1093 // For instance, it may be an overlay file that 1094 // hasn't been written to disk. To handle that case 1095 // let x == y through. (We added the exact absolute path 1096 // string to the CompiledGoFiles list, so the unwritten 1097 // overlay case implies x==y.) 1098 return true 1099 } 1100 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation) 1101 if xi, err := os.Stat(x); err == nil { 1102 if yi, err := os.Stat(y); err == nil { 1103 return os.SameFile(xi, yi) 1104 } 1105 } 1106 } 1107 return false 1108} 1109 1110// loadFromExportData returns type information for the specified 1111// package, loading it from an export data file on the first request. 1112func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) { 1113 if lpkg.PkgPath == "" { 1114 log.Fatalf("internal error: Package %s has no PkgPath", lpkg) 1115 } 1116 1117 // Because gcexportdata.Read has the potential to create or 1118 // modify the types.Package for each node in the transitive 1119 // closure of dependencies of lpkg, all exportdata operations 1120 // must be sequential. (Finer-grained locking would require 1121 // changes to the gcexportdata API.) 1122 // 1123 // The exportMu lock guards the Package.Pkg field and the 1124 // types.Package it points to, for each Package in the graph. 1125 // 1126 // Not all accesses to Package.Pkg need to be protected by exportMu: 1127 // graph ordering ensures that direct dependencies of source 1128 // packages are fully loaded before the importer reads their Pkg field. 1129 ld.exportMu.Lock() 1130 defer ld.exportMu.Unlock() 1131 1132 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() { 1133 return tpkg, nil // cache hit 1134 } 1135 1136 lpkg.IllTyped = true // fail safe 1137 1138 if lpkg.ExportFile == "" { 1139 // Errors while building export data will have been printed to stderr. 1140 return nil, fmt.Errorf("no export data file") 1141 } 1142 f, err := os.Open(lpkg.ExportFile) 1143 if err != nil { 1144 return nil, err 1145 } 1146 defer f.Close() 1147 1148 // Read gc export data. 1149 // 1150 // We don't currently support gccgo export data because all 1151 // underlying workspaces use the gc toolchain. (Even build 1152 // systems that support gccgo don't use it for workspace 1153 // queries.) 1154 r, err := gcexportdata.NewReader(f) 1155 if err != nil { 1156 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 1157 } 1158 1159 // Build the view. 1160 // 1161 // The gcexportdata machinery has no concept of package ID. 1162 // It identifies packages by their PkgPath, which although not 1163 // globally unique is unique within the scope of one invocation 1164 // of the linker, type-checker, or gcexportdata. 1165 // 1166 // So, we must build a PkgPath-keyed view of the global 1167 // (conceptually ID-keyed) cache of packages and pass it to 1168 // gcexportdata. The view must contain every existing 1169 // package that might possibly be mentioned by the 1170 // current package---its transitive closure. 1171 // 1172 // In loadPackage, we unconditionally create a types.Package for 1173 // each dependency so that export data loading does not 1174 // create new ones. 1175 // 1176 // TODO(adonovan): it would be simpler and more efficient 1177 // if the export data machinery invoked a callback to 1178 // get-or-create a package instead of a map. 1179 // 1180 view := make(map[string]*types.Package) // view seen by gcexportdata 1181 seen := make(map[*loaderPackage]bool) // all visited packages 1182 var visit func(pkgs map[string]*Package) 1183 visit = func(pkgs map[string]*Package) { 1184 for _, p := range pkgs { 1185 lpkg := ld.pkgs[p.ID] 1186 if !seen[lpkg] { 1187 seen[lpkg] = true 1188 view[lpkg.PkgPath] = lpkg.Types 1189 visit(lpkg.Imports) 1190 } 1191 } 1192 } 1193 visit(lpkg.Imports) 1194 1195 viewLen := len(view) + 1 // adding the self package 1196 // Parse the export data. 1197 // (May modify incomplete packages in view but not create new ones.) 1198 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath) 1199 if err != nil { 1200 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 1201 } 1202 if viewLen != len(view) { 1203 log.Fatalf("Unexpected package creation during export data loading") 1204 } 1205 1206 lpkg.Types = tpkg 1207 lpkg.IllTyped = false 1208 1209 return tpkg, nil 1210} 1211 1212// impliedLoadMode returns loadMode with its dependencies. 1213func impliedLoadMode(loadMode LoadMode) LoadMode { 1214 if loadMode&NeedTypesInfo != 0 && loadMode&NeedImports == 0 { 1215 // If NeedTypesInfo, go/packages needs to do typechecking itself so it can 1216 // associate type info with the AST. To do so, we need the export data 1217 // for dependencies, which means we need to ask for the direct dependencies. 1218 // NeedImports is used to ask for the direct dependencies. 1219 loadMode |= NeedImports 1220 } 1221 1222 if loadMode&NeedDeps != 0 && loadMode&NeedImports == 0 { 1223 // With NeedDeps we need to load at least direct dependencies. 1224 // NeedImports is used to ask for the direct dependencies. 1225 loadMode |= NeedImports 1226 } 1227 1228 return loadMode 1229} 1230 1231func usesExportData(cfg *Config) bool { 1232 return cfg.Mode&NeedExportsFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0 1233} 1234