1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package packages 6 7// See doc.go for package documentation and implementation notes. 8 9import ( 10 "context" 11 "encoding/json" 12 "fmt" 13 "go/ast" 14 "go/parser" 15 "go/scanner" 16 "go/token" 17 "go/types" 18 "io/ioutil" 19 "log" 20 "os" 21 "path/filepath" 22 "strings" 23 "sync" 24 25 "golang.org/x/tools/go/gcexportdata" 26) 27 28// A LoadMode controls the amount of detail to return when loading. 29// The bits below can be combined to specify which fields should be 30// filled in the result packages. 31// The zero value is a special case, equivalent to combining 32// the NeedName, NeedFiles, and NeedCompiledGoFiles bits. 33// ID and Errors (if present) will always be filled. 34// Load may return more information than requested. 35type LoadMode int 36 37const ( 38 // NeedName adds Name and PkgPath. 39 NeedName LoadMode = 1 << iota 40 41 // NeedFiles adds GoFiles and OtherFiles. 42 NeedFiles 43 44 // NeedCompiledGoFiles adds CompiledGoFiles. 45 NeedCompiledGoFiles 46 47 // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain 48 // "placeholder" Packages with only the ID set. 49 NeedImports 50 51 // NeedDeps adds the fields requested by the LoadMode in the packages in Imports. 52 NeedDeps 53 54 // NeedExportsFile adds ExportsFile. 55 NeedExportsFile 56 57 // NeedTypes adds Types, Fset, and IllTyped. 58 NeedTypes 59 60 // NeedSyntax adds Syntax. 61 NeedSyntax 62 63 // NeedTypesInfo adds TypesInfo. 64 NeedTypesInfo 65 66 // NeedTypesSizes adds TypesSizes. 67 NeedTypesSizes 68) 69 70const ( 71 // Deprecated: LoadFiles exists for historical compatibility 72 // and should not be used. Please directly specify the needed fields using the Need values. 73 LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles 74 75 // Deprecated: LoadImports exists for historical compatibility 76 // and should not be used. Please directly specify the needed fields using the Need values. 77 LoadImports = LoadFiles | NeedImports 78 79 // Deprecated: LoadTypes exists for historical compatibility 80 // and should not be used. Please directly specify the needed fields using the Need values. 81 LoadTypes = LoadImports | NeedTypes | NeedTypesSizes 82 83 // Deprecated: LoadSyntax exists for historical compatibility 84 // and should not be used. Please directly specify the needed fields using the Need values. 85 LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo 86 87 // Deprecated: LoadAllSyntax exists for historical compatibility 88 // and should not be used. Please directly specify the needed fields using the Need values. 89 LoadAllSyntax = LoadSyntax | NeedDeps 90) 91 92// A Config specifies details about how packages should be loaded. 93// The zero value is a valid configuration. 94// Calls to Load do not modify this struct. 95type Config struct { 96 // Mode controls the level of information returned for each package. 97 Mode LoadMode 98 99 // Context specifies the context for the load operation. 100 // If the context is cancelled, the loader may stop early 101 // and return an ErrCancelled error. 102 // If Context is nil, the load cannot be cancelled. 103 Context context.Context 104 105 // Logf is the logger for the config. 106 // If the user provides a logger, debug logging is enabled. 107 // If the GOPACKAGESDEBUG environment variable is set to true, 108 // but the logger is nil, default to log.Printf. 109 Logf func(format string, args ...interface{}) 110 111 // Dir is the directory in which to run the build system's query tool 112 // that provides information about the packages. 113 // If Dir is empty, the tool is run in the current directory. 114 Dir string 115 116 // Env is the environment to use when invoking the build system's query tool. 117 // If Env is nil, the current environment is used. 118 // As in os/exec's Cmd, only the last value in the slice for 119 // each environment key is used. To specify the setting of only 120 // a few variables, append to the current environment, as in: 121 // 122 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386") 123 // 124 Env []string 125 126 // BuildFlags is a list of command-line flags to be passed through to 127 // the build system's query tool. 128 BuildFlags []string 129 130 // Fset provides source position information for syntax trees and types. 131 // If Fset is nil, Load will use a new fileset, but preserve Fset's value. 132 Fset *token.FileSet 133 134 // ParseFile is called to read and parse each file 135 // when preparing a package's type-checked syntax tree. 136 // It must be safe to call ParseFile simultaneously from multiple goroutines. 137 // If ParseFile is nil, the loader will uses parser.ParseFile. 138 // 139 // ParseFile should parse the source from src and use filename only for 140 // recording position information. 141 // 142 // An application may supply a custom implementation of ParseFile 143 // to change the effective file contents or the behavior of the parser, 144 // or to modify the syntax tree. For example, selectively eliminating 145 // unwanted function bodies can significantly accelerate type checking. 146 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) 147 148 // If Tests is set, the loader includes not just the packages 149 // matching a particular pattern but also any related test packages, 150 // including test-only variants of the package and the test executable. 151 // 152 // For example, when using the go command, loading "fmt" with Tests=true 153 // returns four packages, with IDs "fmt" (the standard package), 154 // "fmt [fmt.test]" (the package as compiled for the test), 155 // "fmt_test" (the test functions from source files in package fmt_test), 156 // and "fmt.test" (the test binary). 157 // 158 // In build systems with explicit names for tests, 159 // setting Tests may have no effect. 160 Tests bool 161 162 // Overlay provides a mapping of absolute file paths to file contents. 163 // If the file with the given path already exists, the parser will use the 164 // alternative file contents provided by the map. 165 // 166 // Overlays provide incomplete support for when a given file doesn't 167 // already exist on disk. See the package doc above for more details. 168 Overlay map[string][]byte 169} 170 171// driver is the type for functions that query the build system for the 172// packages named by the patterns. 173type driver func(cfg *Config, patterns ...string) (*driverResponse, error) 174 175// driverResponse contains the results for a driver query. 176type driverResponse struct { 177 // Sizes, if not nil, is the types.Sizes to use when type checking. 178 Sizes *types.StdSizes 179 180 // Roots is the set of package IDs that make up the root packages. 181 // We have to encode this separately because when we encode a single package 182 // we cannot know if it is one of the roots as that requires knowledge of the 183 // graph it is part of. 184 Roots []string `json:",omitempty"` 185 186 // Packages is the full set of packages in the graph. 187 // The packages are not connected into a graph. 188 // The Imports if populated will be stubs that only have their ID set. 189 // Imports will be connected and then type and syntax information added in a 190 // later pass (see refine). 191 Packages []*Package 192} 193 194// Load loads and returns the Go packages named by the given patterns. 195// 196// Config specifies loading options; 197// nil behaves the same as an empty Config. 198// 199// Load returns an error if any of the patterns was invalid 200// as defined by the underlying build system. 201// It may return an empty list of packages without an error, 202// for instance for an empty expansion of a valid wildcard. 203// Errors associated with a particular package are recorded in the 204// corresponding Package's Errors list, and do not cause Load to 205// return an error. Clients may need to handle such errors before 206// proceeding with further analysis. The PrintErrors function is 207// provided for convenient display of all errors. 208func Load(cfg *Config, patterns ...string) ([]*Package, error) { 209 l := newLoader(cfg) 210 response, err := defaultDriver(&l.Config, patterns...) 211 if err != nil { 212 return nil, err 213 } 214 l.sizes = response.Sizes 215 return l.refine(response.Roots, response.Packages...) 216} 217 218// defaultDriver is a driver that looks for an external driver binary, and if 219// it does not find it falls back to the built in go list driver. 220func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) { 221 driver := findExternalDriver(cfg) 222 if driver == nil { 223 driver = goListDriver 224 } 225 return driver(cfg, patterns...) 226} 227 228// A Package describes a loaded Go package. 229type Package struct { 230 // ID is a unique identifier for a package, 231 // in a syntax provided by the underlying build system. 232 // 233 // Because the syntax varies based on the build system, 234 // clients should treat IDs as opaque and not attempt to 235 // interpret them. 236 ID string 237 238 // Name is the package name as it appears in the package source code. 239 Name string 240 241 // PkgPath is the package path as used by the go/types package. 242 PkgPath string 243 244 // Errors contains any errors encountered querying the metadata 245 // of the package, or while parsing or type-checking its files. 246 Errors []Error 247 248 // GoFiles lists the absolute file paths of the package's Go source files. 249 GoFiles []string 250 251 // CompiledGoFiles lists the absolute file paths of the package's source 252 // files that were presented to the compiler. 253 // This may differ from GoFiles if files are processed before compilation. 254 CompiledGoFiles []string 255 256 // OtherFiles lists the absolute file paths of the package's non-Go source files, 257 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on. 258 OtherFiles []string 259 260 // ExportFile is the absolute path to a file containing type 261 // information for the package as provided by the build system. 262 ExportFile string 263 264 // Imports maps import paths appearing in the package's Go source files 265 // to corresponding loaded Packages. 266 Imports map[string]*Package 267 268 // Types provides type information for the package. 269 // The NeedTypes LoadMode bit sets this field for packages matching the 270 // patterns; type information for dependencies may be missing or incomplete, 271 // unless NeedDeps and NeedImports are also set. 272 Types *types.Package 273 274 // Fset provides position information for Types, TypesInfo, and Syntax. 275 // It is set only when Types is set. 276 Fset *token.FileSet 277 278 // IllTyped indicates whether the package or any dependency contains errors. 279 // It is set only when Types is set. 280 IllTyped bool 281 282 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles. 283 // 284 // The NeedSyntax LoadMode bit populates this field for packages matching the patterns. 285 // If NeedDeps and NeedImports are also set, this field will also be populated 286 // for dependencies. 287 Syntax []*ast.File 288 289 // TypesInfo provides type information about the package's syntax trees. 290 // It is set only when Syntax is set. 291 TypesInfo *types.Info 292 293 // TypesSizes provides the effective size function for types in TypesInfo. 294 TypesSizes types.Sizes 295} 296 297// An Error describes a problem with a package's metadata, syntax, or types. 298type Error struct { 299 Pos string // "file:line:col" or "file:line" or "" or "-" 300 Msg string 301 Kind ErrorKind 302} 303 304// ErrorKind describes the source of the error, allowing the user to 305// differentiate between errors generated by the driver, the parser, or the 306// type-checker. 307type ErrorKind int 308 309const ( 310 UnknownError ErrorKind = iota 311 ListError 312 ParseError 313 TypeError 314) 315 316func (err Error) Error() string { 317 pos := err.Pos 318 if pos == "" { 319 pos = "-" // like token.Position{}.String() 320 } 321 return pos + ": " + err.Msg 322} 323 324// flatPackage is the JSON form of Package 325// It drops all the type and syntax fields, and transforms the Imports 326// 327// TODO(adonovan): identify this struct with Package, effectively 328// publishing the JSON protocol. 329type flatPackage struct { 330 ID string 331 Name string `json:",omitempty"` 332 PkgPath string `json:",omitempty"` 333 Errors []Error `json:",omitempty"` 334 GoFiles []string `json:",omitempty"` 335 CompiledGoFiles []string `json:",omitempty"` 336 OtherFiles []string `json:",omitempty"` 337 ExportFile string `json:",omitempty"` 338 Imports map[string]string `json:",omitempty"` 339} 340 341// MarshalJSON returns the Package in its JSON form. 342// For the most part, the structure fields are written out unmodified, and 343// the type and syntax fields are skipped. 344// The imports are written out as just a map of path to package id. 345// The errors are written using a custom type that tries to preserve the 346// structure of error types we know about. 347// 348// This method exists to enable support for additional build systems. It is 349// not intended for use by clients of the API and we may change the format. 350func (p *Package) MarshalJSON() ([]byte, error) { 351 flat := &flatPackage{ 352 ID: p.ID, 353 Name: p.Name, 354 PkgPath: p.PkgPath, 355 Errors: p.Errors, 356 GoFiles: p.GoFiles, 357 CompiledGoFiles: p.CompiledGoFiles, 358 OtherFiles: p.OtherFiles, 359 ExportFile: p.ExportFile, 360 } 361 if len(p.Imports) > 0 { 362 flat.Imports = make(map[string]string, len(p.Imports)) 363 for path, ipkg := range p.Imports { 364 flat.Imports[path] = ipkg.ID 365 } 366 } 367 return json.Marshal(flat) 368} 369 370// UnmarshalJSON reads in a Package from its JSON format. 371// See MarshalJSON for details about the format accepted. 372func (p *Package) UnmarshalJSON(b []byte) error { 373 flat := &flatPackage{} 374 if err := json.Unmarshal(b, &flat); err != nil { 375 return err 376 } 377 *p = Package{ 378 ID: flat.ID, 379 Name: flat.Name, 380 PkgPath: flat.PkgPath, 381 Errors: flat.Errors, 382 GoFiles: flat.GoFiles, 383 CompiledGoFiles: flat.CompiledGoFiles, 384 OtherFiles: flat.OtherFiles, 385 ExportFile: flat.ExportFile, 386 } 387 if len(flat.Imports) > 0 { 388 p.Imports = make(map[string]*Package, len(flat.Imports)) 389 for path, id := range flat.Imports { 390 p.Imports[path] = &Package{ID: id} 391 } 392 } 393 return nil 394} 395 396func (p *Package) String() string { return p.ID } 397 398// loaderPackage augments Package with state used during the loading phase 399type loaderPackage struct { 400 *Package 401 importErrors map[string]error // maps each bad import to its error 402 loadOnce sync.Once 403 color uint8 // for cycle detection 404 needsrc bool // load from source (Mode >= LoadTypes) 405 needtypes bool // type information is either requested or depended on 406 initial bool // package was matched by a pattern 407} 408 409// loader holds the working state of a single call to load. 410type loader struct { 411 pkgs map[string]*loaderPackage 412 Config 413 sizes types.Sizes 414 parseCache map[string]*parseValue 415 parseCacheMu sync.Mutex 416 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations 417 418 // Config.Mode contains the implied mode (see impliedLoadMode). 419 // Implied mode contains all the fields we need the data for. 420 // In requestedMode there are the actually requested fields. 421 // We'll zero them out before returning packages to the user. 422 // This makes it easier for us to get the conditions where 423 // we need certain modes right. 424 requestedMode LoadMode 425} 426 427type parseValue struct { 428 f *ast.File 429 err error 430 ready chan struct{} 431} 432 433func newLoader(cfg *Config) *loader { 434 ld := &loader{ 435 parseCache: map[string]*parseValue{}, 436 } 437 if cfg != nil { 438 ld.Config = *cfg 439 // If the user has provided a logger, use it. 440 ld.Config.Logf = cfg.Logf 441 } 442 if ld.Config.Logf == nil { 443 // If the GOPACKAGESDEBUG environment variable is set to true, 444 // but the user has not provided a logger, default to log.Printf. 445 if debug { 446 ld.Config.Logf = log.Printf 447 } else { 448 ld.Config.Logf = func(format string, args ...interface{}) {} 449 } 450 } 451 if ld.Config.Mode == 0 { 452 ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility. 453 } 454 if ld.Config.Env == nil { 455 ld.Config.Env = os.Environ() 456 } 457 if ld.Context == nil { 458 ld.Context = context.Background() 459 } 460 if ld.Dir == "" { 461 if dir, err := os.Getwd(); err == nil { 462 ld.Dir = dir 463 } 464 } 465 466 // Save the actually requested fields. We'll zero them out before returning packages to the user. 467 ld.requestedMode = ld.Mode 468 ld.Mode = impliedLoadMode(ld.Mode) 469 470 if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 { 471 if ld.Fset == nil { 472 ld.Fset = token.NewFileSet() 473 } 474 475 // ParseFile is required even in LoadTypes mode 476 // because we load source if export data is missing. 477 if ld.ParseFile == nil { 478 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) { 479 const mode = parser.AllErrors | parser.ParseComments 480 return parser.ParseFile(fset, filename, src, mode) 481 } 482 } 483 } 484 485 return ld 486} 487 488// refine connects the supplied packages into a graph and then adds type and 489// and syntax information as requested by the LoadMode. 490func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) { 491 rootMap := make(map[string]int, len(roots)) 492 for i, root := range roots { 493 rootMap[root] = i 494 } 495 ld.pkgs = make(map[string]*loaderPackage) 496 // first pass, fixup and build the map and roots 497 var initial = make([]*loaderPackage, len(roots)) 498 for _, pkg := range list { 499 rootIndex := -1 500 if i, found := rootMap[pkg.ID]; found { 501 rootIndex = i 502 } 503 lpkg := &loaderPackage{ 504 Package: pkg, 505 needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0, 506 needsrc: (ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0 || 507 len(ld.Overlay) > 0 || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files 508 pkg.ExportFile == "" && pkg.PkgPath != "unsafe", 509 } 510 ld.pkgs[lpkg.ID] = lpkg 511 if rootIndex >= 0 { 512 initial[rootIndex] = lpkg 513 lpkg.initial = true 514 } 515 } 516 for i, root := range roots { 517 if initial[i] == nil { 518 return nil, fmt.Errorf("root package %v is missing", root) 519 } 520 } 521 522 // Materialize the import graph. 523 524 const ( 525 white = 0 // new 526 grey = 1 // in progress 527 black = 2 // complete 528 ) 529 530 // visit traverses the import graph, depth-first, 531 // and materializes the graph as Packages.Imports. 532 // 533 // Valid imports are saved in the Packages.Import map. 534 // Invalid imports (cycles and missing nodes) are saved in the importErrors map. 535 // Thus, even in the presence of both kinds of errors, the Import graph remains a DAG. 536 // 537 // visit returns whether the package needs src or has a transitive 538 // dependency on a package that does. These are the only packages 539 // for which we load source code. 540 var stack []*loaderPackage 541 var visit func(lpkg *loaderPackage) bool 542 var srcPkgs []*loaderPackage 543 visit = func(lpkg *loaderPackage) bool { 544 switch lpkg.color { 545 case black: 546 return lpkg.needsrc 547 case grey: 548 panic("internal error: grey node") 549 } 550 lpkg.color = grey 551 stack = append(stack, lpkg) // push 552 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports 553 // If NeedImports isn't set, the imports fields will all be zeroed out. 554 if ld.Mode&NeedImports != 0 { 555 lpkg.Imports = make(map[string]*Package, len(stubs)) 556 for importPath, ipkg := range stubs { 557 var importErr error 558 imp := ld.pkgs[ipkg.ID] 559 if imp == nil { 560 // (includes package "C" when DisableCgo) 561 importErr = fmt.Errorf("missing package: %q", ipkg.ID) 562 } else if imp.color == grey { 563 importErr = fmt.Errorf("import cycle: %s", stack) 564 } 565 if importErr != nil { 566 if lpkg.importErrors == nil { 567 lpkg.importErrors = make(map[string]error) 568 } 569 lpkg.importErrors[importPath] = importErr 570 continue 571 } 572 573 if visit(imp) { 574 lpkg.needsrc = true 575 } 576 lpkg.Imports[importPath] = imp.Package 577 } 578 } 579 if lpkg.needsrc { 580 srcPkgs = append(srcPkgs, lpkg) 581 } 582 if ld.Mode&NeedTypesSizes != 0 { 583 lpkg.TypesSizes = ld.sizes 584 } 585 stack = stack[:len(stack)-1] // pop 586 lpkg.color = black 587 588 return lpkg.needsrc 589 } 590 591 if ld.Mode&NeedImports == 0 { 592 // We do this to drop the stub import packages that we are not even going to try to resolve. 593 for _, lpkg := range initial { 594 lpkg.Imports = nil 595 } 596 } else { 597 // For each initial package, create its import DAG. 598 for _, lpkg := range initial { 599 visit(lpkg) 600 } 601 } 602 if ld.Mode&NeedImports != 0 && ld.Mode&NeedTypes != 0 { 603 for _, lpkg := range srcPkgs { 604 // Complete type information is required for the 605 // immediate dependencies of each source package. 606 for _, ipkg := range lpkg.Imports { 607 imp := ld.pkgs[ipkg.ID] 608 imp.needtypes = true 609 } 610 } 611 } 612 // Load type data and syntax if needed, starting at 613 // the initial packages (roots of the import DAG). 614 if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 { 615 var wg sync.WaitGroup 616 for _, lpkg := range initial { 617 wg.Add(1) 618 go func(lpkg *loaderPackage) { 619 ld.loadRecursive(lpkg) 620 wg.Done() 621 }(lpkg) 622 } 623 wg.Wait() 624 } 625 626 result := make([]*Package, len(initial)) 627 for i, lpkg := range initial { 628 result[i] = lpkg.Package 629 } 630 for i := range ld.pkgs { 631 // Clear all unrequested fields, for extra de-Hyrum-ization. 632 if ld.requestedMode&NeedName == 0 { 633 ld.pkgs[i].Name = "" 634 ld.pkgs[i].PkgPath = "" 635 } 636 if ld.requestedMode&NeedFiles == 0 { 637 ld.pkgs[i].GoFiles = nil 638 ld.pkgs[i].OtherFiles = nil 639 } 640 if ld.requestedMode&NeedCompiledGoFiles == 0 { 641 ld.pkgs[i].CompiledGoFiles = nil 642 } 643 if ld.requestedMode&NeedImports == 0 { 644 ld.pkgs[i].Imports = nil 645 } 646 if ld.requestedMode&NeedExportsFile == 0 { 647 ld.pkgs[i].ExportFile = "" 648 } 649 if ld.requestedMode&NeedTypes == 0 { 650 ld.pkgs[i].Types = nil 651 ld.pkgs[i].Fset = nil 652 ld.pkgs[i].IllTyped = false 653 } 654 if ld.requestedMode&NeedSyntax == 0 { 655 ld.pkgs[i].Syntax = nil 656 } 657 if ld.requestedMode&NeedTypesInfo == 0 { 658 ld.pkgs[i].TypesInfo = nil 659 } 660 if ld.requestedMode&NeedTypesSizes == 0 { 661 ld.pkgs[i].TypesSizes = nil 662 } 663 } 664 665 return result, nil 666} 667 668// loadRecursive loads the specified package and its dependencies, 669// recursively, in parallel, in topological order. 670// It is atomic and idempotent. 671// Precondition: ld.Mode&NeedTypes. 672func (ld *loader) loadRecursive(lpkg *loaderPackage) { 673 lpkg.loadOnce.Do(func() { 674 // Load the direct dependencies, in parallel. 675 var wg sync.WaitGroup 676 for _, ipkg := range lpkg.Imports { 677 imp := ld.pkgs[ipkg.ID] 678 wg.Add(1) 679 go func(imp *loaderPackage) { 680 ld.loadRecursive(imp) 681 wg.Done() 682 }(imp) 683 } 684 wg.Wait() 685 ld.loadPackage(lpkg) 686 }) 687} 688 689// loadPackage loads the specified package. 690// It must be called only once per Package, 691// after immediate dependencies are loaded. 692// Precondition: ld.Mode & NeedTypes. 693func (ld *loader) loadPackage(lpkg *loaderPackage) { 694 if lpkg.PkgPath == "unsafe" { 695 // Fill in the blanks to avoid surprises. 696 lpkg.Types = types.Unsafe 697 lpkg.Fset = ld.Fset 698 lpkg.Syntax = []*ast.File{} 699 lpkg.TypesInfo = new(types.Info) 700 lpkg.TypesSizes = ld.sizes 701 return 702 } 703 704 // Call NewPackage directly with explicit name. 705 // This avoids skew between golist and go/types when the files' 706 // package declarations are inconsistent. 707 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name) 708 lpkg.Fset = ld.Fset 709 710 // Subtle: we populate all Types fields with an empty Package 711 // before loading export data so that export data processing 712 // never has to create a types.Package for an indirect dependency, 713 // which would then require that such created packages be explicitly 714 // inserted back into the Import graph as a final step after export data loading. 715 // The Diamond test exercises this case. 716 if !lpkg.needtypes { 717 return 718 } 719 if !lpkg.needsrc { 720 ld.loadFromExportData(lpkg) 721 return // not a source package, don't get syntax trees 722 } 723 724 appendError := func(err error) { 725 // Convert various error types into the one true Error. 726 var errs []Error 727 switch err := err.(type) { 728 case Error: 729 // from driver 730 errs = append(errs, err) 731 732 case *os.PathError: 733 // from parser 734 errs = append(errs, Error{ 735 Pos: err.Path + ":1", 736 Msg: err.Err.Error(), 737 Kind: ParseError, 738 }) 739 740 case scanner.ErrorList: 741 // from parser 742 for _, err := range err { 743 errs = append(errs, Error{ 744 Pos: err.Pos.String(), 745 Msg: err.Msg, 746 Kind: ParseError, 747 }) 748 } 749 750 case types.Error: 751 // from type checker 752 errs = append(errs, Error{ 753 Pos: err.Fset.Position(err.Pos).String(), 754 Msg: err.Msg, 755 Kind: TypeError, 756 }) 757 758 default: 759 // unexpected impoverished error from parser? 760 errs = append(errs, Error{ 761 Pos: "-", 762 Msg: err.Error(), 763 Kind: UnknownError, 764 }) 765 766 // If you see this error message, please file a bug. 767 log.Printf("internal error: error %q (%T) without position", err, err) 768 } 769 770 lpkg.Errors = append(lpkg.Errors, errs...) 771 } 772 773 if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" { 774 // The config requested loading sources and types, but sources are missing. 775 // Add an error to the package and fall back to loading from export data. 776 appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError}) 777 ld.loadFromExportData(lpkg) 778 return // can't get syntax trees for this package 779 } 780 781 files, errs := ld.parseFiles(lpkg.CompiledGoFiles) 782 for _, err := range errs { 783 appendError(err) 784 } 785 786 lpkg.Syntax = files 787 if ld.Config.Mode&NeedTypes == 0 { 788 return 789 } 790 791 lpkg.TypesInfo = &types.Info{ 792 Types: make(map[ast.Expr]types.TypeAndValue), 793 Defs: make(map[*ast.Ident]types.Object), 794 Uses: make(map[*ast.Ident]types.Object), 795 Implicits: make(map[ast.Node]types.Object), 796 Scopes: make(map[ast.Node]*types.Scope), 797 Selections: make(map[*ast.SelectorExpr]*types.Selection), 798 } 799 lpkg.TypesSizes = ld.sizes 800 801 importer := importerFunc(func(path string) (*types.Package, error) { 802 if path == "unsafe" { 803 return types.Unsafe, nil 804 } 805 806 // The imports map is keyed by import path. 807 ipkg := lpkg.Imports[path] 808 if ipkg == nil { 809 if err := lpkg.importErrors[path]; err != nil { 810 return nil, err 811 } 812 // There was skew between the metadata and the 813 // import declarations, likely due to an edit 814 // race, or because the ParseFile feature was 815 // used to supply alternative file contents. 816 return nil, fmt.Errorf("no metadata for %s", path) 817 } 818 819 if ipkg.Types != nil && ipkg.Types.Complete() { 820 return ipkg.Types, nil 821 } 822 log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg) 823 panic("unreachable") 824 }) 825 826 // type-check 827 tc := &types.Config{ 828 Importer: importer, 829 830 // Type-check bodies of functions only in non-initial packages. 831 // Example: for import graph A->B->C and initial packages {A,C}, 832 // we can ignore function bodies in B. 833 IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial, 834 835 Error: appendError, 836 Sizes: ld.sizes, 837 } 838 types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax) 839 840 lpkg.importErrors = nil // no longer needed 841 842 // If !Cgo, the type-checker uses FakeImportC mode, so 843 // it doesn't invoke the importer for import "C", 844 // nor report an error for the import, 845 // or for any undefined C.f reference. 846 // We must detect this explicitly and correctly 847 // mark the package as IllTyped (by reporting an error). 848 // TODO(adonovan): if these errors are annoying, 849 // we could just set IllTyped quietly. 850 if tc.FakeImportC { 851 outer: 852 for _, f := range lpkg.Syntax { 853 for _, imp := range f.Imports { 854 if imp.Path.Value == `"C"` { 855 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`} 856 appendError(err) 857 break outer 858 } 859 } 860 } 861 } 862 863 // Record accumulated errors. 864 illTyped := len(lpkg.Errors) > 0 865 if !illTyped { 866 for _, imp := range lpkg.Imports { 867 if imp.IllTyped { 868 illTyped = true 869 break 870 } 871 } 872 } 873 lpkg.IllTyped = illTyped 874} 875 876// An importFunc is an implementation of the single-method 877// types.Importer interface based on a function value. 878type importerFunc func(path string) (*types.Package, error) 879 880func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } 881 882// We use a counting semaphore to limit 883// the number of parallel I/O calls per process. 884var ioLimit = make(chan bool, 20) 885 886func (ld *loader) parseFile(filename string) (*ast.File, error) { 887 ld.parseCacheMu.Lock() 888 v, ok := ld.parseCache[filename] 889 if ok { 890 // cache hit 891 ld.parseCacheMu.Unlock() 892 <-v.ready 893 } else { 894 // cache miss 895 v = &parseValue{ready: make(chan struct{})} 896 ld.parseCache[filename] = v 897 ld.parseCacheMu.Unlock() 898 899 var src []byte 900 for f, contents := range ld.Config.Overlay { 901 if sameFile(f, filename) { 902 src = contents 903 } 904 } 905 var err error 906 if src == nil { 907 ioLimit <- true // wait 908 src, err = ioutil.ReadFile(filename) 909 <-ioLimit // signal 910 } 911 if err != nil { 912 v.err = err 913 } else { 914 v.f, v.err = ld.ParseFile(ld.Fset, filename, src) 915 } 916 917 close(v.ready) 918 } 919 return v.f, v.err 920} 921 922// parseFiles reads and parses the Go source files and returns the ASTs 923// of the ones that could be at least partially parsed, along with a 924// list of I/O and parse errors encountered. 925// 926// Because files are scanned in parallel, the token.Pos 927// positions of the resulting ast.Files are not ordered. 928// 929func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) { 930 var wg sync.WaitGroup 931 n := len(filenames) 932 parsed := make([]*ast.File, n) 933 errors := make([]error, n) 934 for i, file := range filenames { 935 if ld.Config.Context.Err() != nil { 936 parsed[i] = nil 937 errors[i] = ld.Config.Context.Err() 938 continue 939 } 940 wg.Add(1) 941 go func(i int, filename string) { 942 parsed[i], errors[i] = ld.parseFile(filename) 943 wg.Done() 944 }(i, file) 945 } 946 wg.Wait() 947 948 // Eliminate nils, preserving order. 949 var o int 950 for _, f := range parsed { 951 if f != nil { 952 parsed[o] = f 953 o++ 954 } 955 } 956 parsed = parsed[:o] 957 958 o = 0 959 for _, err := range errors { 960 if err != nil { 961 errors[o] = err 962 o++ 963 } 964 } 965 errors = errors[:o] 966 967 return parsed, errors 968} 969 970// sameFile returns true if x and y have the same basename and denote 971// the same file. 972// 973func sameFile(x, y string) bool { 974 if x == y { 975 // It could be the case that y doesn't exist. 976 // For instance, it may be an overlay file that 977 // hasn't been written to disk. To handle that case 978 // let x == y through. (We added the exact absolute path 979 // string to the CompiledGoFiles list, so the unwritten 980 // overlay case implies x==y.) 981 return true 982 } 983 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation) 984 if xi, err := os.Stat(x); err == nil { 985 if yi, err := os.Stat(y); err == nil { 986 return os.SameFile(xi, yi) 987 } 988 } 989 } 990 return false 991} 992 993// loadFromExportData returns type information for the specified 994// package, loading it from an export data file on the first request. 995func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) { 996 if lpkg.PkgPath == "" { 997 log.Fatalf("internal error: Package %s has no PkgPath", lpkg) 998 } 999 1000 // Because gcexportdata.Read has the potential to create or 1001 // modify the types.Package for each node in the transitive 1002 // closure of dependencies of lpkg, all exportdata operations 1003 // must be sequential. (Finer-grained locking would require 1004 // changes to the gcexportdata API.) 1005 // 1006 // The exportMu lock guards the Package.Pkg field and the 1007 // types.Package it points to, for each Package in the graph. 1008 // 1009 // Not all accesses to Package.Pkg need to be protected by exportMu: 1010 // graph ordering ensures that direct dependencies of source 1011 // packages are fully loaded before the importer reads their Pkg field. 1012 ld.exportMu.Lock() 1013 defer ld.exportMu.Unlock() 1014 1015 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() { 1016 return tpkg, nil // cache hit 1017 } 1018 1019 lpkg.IllTyped = true // fail safe 1020 1021 if lpkg.ExportFile == "" { 1022 // Errors while building export data will have been printed to stderr. 1023 return nil, fmt.Errorf("no export data file") 1024 } 1025 f, err := os.Open(lpkg.ExportFile) 1026 if err != nil { 1027 return nil, err 1028 } 1029 defer f.Close() 1030 1031 // Read gc export data. 1032 // 1033 // We don't currently support gccgo export data because all 1034 // underlying workspaces use the gc toolchain. (Even build 1035 // systems that support gccgo don't use it for workspace 1036 // queries.) 1037 r, err := gcexportdata.NewReader(f) 1038 if err != nil { 1039 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 1040 } 1041 1042 // Build the view. 1043 // 1044 // The gcexportdata machinery has no concept of package ID. 1045 // It identifies packages by their PkgPath, which although not 1046 // globally unique is unique within the scope of one invocation 1047 // of the linker, type-checker, or gcexportdata. 1048 // 1049 // So, we must build a PkgPath-keyed view of the global 1050 // (conceptually ID-keyed) cache of packages and pass it to 1051 // gcexportdata. The view must contain every existing 1052 // package that might possibly be mentioned by the 1053 // current package---its transitive closure. 1054 // 1055 // In loadPackage, we unconditionally create a types.Package for 1056 // each dependency so that export data loading does not 1057 // create new ones. 1058 // 1059 // TODO(adonovan): it would be simpler and more efficient 1060 // if the export data machinery invoked a callback to 1061 // get-or-create a package instead of a map. 1062 // 1063 view := make(map[string]*types.Package) // view seen by gcexportdata 1064 seen := make(map[*loaderPackage]bool) // all visited packages 1065 var visit func(pkgs map[string]*Package) 1066 visit = func(pkgs map[string]*Package) { 1067 for _, p := range pkgs { 1068 lpkg := ld.pkgs[p.ID] 1069 if !seen[lpkg] { 1070 seen[lpkg] = true 1071 view[lpkg.PkgPath] = lpkg.Types 1072 visit(lpkg.Imports) 1073 } 1074 } 1075 } 1076 visit(lpkg.Imports) 1077 1078 viewLen := len(view) + 1 // adding the self package 1079 // Parse the export data. 1080 // (May modify incomplete packages in view but not create new ones.) 1081 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath) 1082 if err != nil { 1083 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err) 1084 } 1085 if viewLen != len(view) { 1086 log.Fatalf("Unexpected package creation during export data loading") 1087 } 1088 1089 lpkg.Types = tpkg 1090 lpkg.IllTyped = false 1091 1092 return tpkg, nil 1093} 1094 1095// impliedLoadMode returns loadMode with its dependencies. 1096func impliedLoadMode(loadMode LoadMode) LoadMode { 1097 if loadMode&NeedTypesInfo != 0 && loadMode&NeedImports == 0 { 1098 // If NeedTypesInfo, go/packages needs to do typechecking itself so it can 1099 // associate type info with the AST. To do so, we need the export data 1100 // for dependencies, which means we need to ask for the direct dependencies. 1101 // NeedImports is used to ask for the direct dependencies. 1102 loadMode |= NeedImports 1103 } 1104 1105 if loadMode&NeedDeps != 0 && loadMode&NeedImports == 0 { 1106 // With NeedDeps we need to load at least direct dependencies. 1107 // NeedImports is used to ask for the direct dependencies. 1108 loadMode |= NeedImports 1109 } 1110 1111 return loadMode 1112} 1113 1114func usesExportData(cfg *Config) bool { 1115 return cfg.Mode&NeedExportsFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0 1116} 1117