1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "bytes" 9 "context" 10 "fmt" 11 "go/ast" 12 "go/token" 13 "go/types" 14 "io" 15 "io/ioutil" 16 "os" 17 "path/filepath" 18 "regexp" 19 "sort" 20 "strconv" 21 "strings" 22 "sync" 23 24 "golang.org/x/mod/modfile" 25 "golang.org/x/mod/module" 26 "golang.org/x/mod/semver" 27 "golang.org/x/tools/go/analysis" 28 "golang.org/x/tools/go/packages" 29 "golang.org/x/tools/internal/event" 30 "golang.org/x/tools/internal/gocommand" 31 "golang.org/x/tools/internal/lsp/debug/log" 32 "golang.org/x/tools/internal/lsp/debug/tag" 33 "golang.org/x/tools/internal/lsp/source" 34 "golang.org/x/tools/internal/memoize" 35 "golang.org/x/tools/internal/packagesinternal" 36 "golang.org/x/tools/internal/span" 37 "golang.org/x/tools/internal/typesinternal" 38 errors "golang.org/x/xerrors" 39) 40 41type snapshot struct { 42 memoize.Arg // allow as a memoize.Function arg 43 44 id uint64 45 view *View 46 47 cancel func() 48 backgroundCtx context.Context 49 50 // the cache generation that contains the data for this snapshot. 51 generation *memoize.Generation 52 53 // The snapshot's initialization state is controlled by the fields below. 54 // 55 // initializeOnce guards snapshot initialization. Each snapshot is 56 // initialized at most once: reinitialization is triggered on later snapshots 57 // by invalidating this field. 58 initializeOnce *sync.Once 59 // initializedErr holds the last error resulting from initialization. If 60 // initialization fails, we only retry when the the workspace modules change, 61 // to avoid too many go/packages calls. 62 initializedErr *source.CriticalError 63 64 // mu guards all of the maps in the snapshot, as well as the builtin URI. 65 mu sync.Mutex 66 67 // builtin pins the AST and package for builtin.go in memory. 68 builtin span.URI 69 70 // ids maps file URIs to package IDs. 71 // It may be invalidated on calls to go/packages. 72 ids map[span.URI][]packageID 73 74 // metadata maps file IDs to their associated metadata. 75 // It may invalidated on calls to go/packages. 76 metadata map[packageID]*knownMetadata 77 78 // importedBy maps package IDs to the list of packages that import them. 79 importedBy map[packageID][]packageID 80 81 // files maps file URIs to their corresponding FileHandles. 82 // It may invalidated when a file's content changes. 83 files map[span.URI]source.VersionedFileHandle 84 85 // goFiles maps a parseKey to its parseGoHandle. 86 goFiles map[parseKey]*parseGoHandle 87 88 // packages maps a packageKey to a set of packageHandles to which that file belongs. 89 // It may be invalidated when a file's content changes. 90 packages map[packageKey]*packageHandle 91 92 // actions maps an actionkey to its actionHandle. 93 actions map[actionKey]*actionHandle 94 95 // workspacePackages contains the workspace's packages, which are loaded 96 // when the view is created. 97 workspacePackages map[packageID]packagePath 98 99 // unloadableFiles keeps track of files that we've failed to load. 100 unloadableFiles map[span.URI]struct{} 101 102 // parseModHandles keeps track of any ParseModHandles for the snapshot. 103 // The handles need not refer to only the view's go.mod file. 104 parseModHandles map[span.URI]*parseModHandle 105 106 // Preserve go.mod-related handles to avoid garbage-collecting the results 107 // of various calls to the go command. The handles need not refer to only 108 // the view's go.mod file. 109 modTidyHandles map[span.URI]*modTidyHandle 110 modWhyHandles map[span.URI]*modWhyHandle 111 112 workspace *workspace 113 workspaceDirHandle *memoize.Handle 114} 115 116type packageKey struct { 117 mode source.ParseMode 118 id packageID 119} 120 121type actionKey struct { 122 pkg packageKey 123 analyzer *analysis.Analyzer 124} 125 126// knownMetadata is a wrapper around metadata that tracks its validity. 127type knownMetadata struct { 128 *metadata 129 130 // valid is true if the given metadata is valid. 131 // Invalid metadata can still be used if a metadata reload fails. 132 valid bool 133} 134 135func (s *snapshot) ID() uint64 { 136 return s.id 137} 138 139func (s *snapshot) View() source.View { 140 return s.view 141} 142 143func (s *snapshot) BackgroundContext() context.Context { 144 return s.backgroundCtx 145} 146 147func (s *snapshot) FileSet() *token.FileSet { 148 return s.view.session.cache.fset 149} 150 151func (s *snapshot) ModFiles() []span.URI { 152 var uris []span.URI 153 for modURI := range s.workspace.getActiveModFiles() { 154 uris = append(uris, modURI) 155 } 156 return uris 157} 158 159func (s *snapshot) Templates() map[span.URI]source.VersionedFileHandle { 160 if !s.view.options.ExperimentalTemplateSupport { 161 return nil 162 } 163 ans := map[span.URI]source.VersionedFileHandle{} 164 for k, x := range s.files { 165 if strings.HasSuffix(filepath.Ext(k.Filename()), "tmpl") { 166 ans[k] = x 167 } 168 } 169 return ans 170} 171 172func (s *snapshot) ValidBuildConfiguration() bool { 173 return validBuildConfiguration(s.view.rootURI, &s.view.workspaceInformation, s.workspace.getActiveModFiles()) 174} 175 176// workspaceMode describes the way in which the snapshot's workspace should 177// be loaded. 178func (s *snapshot) workspaceMode() workspaceMode { 179 var mode workspaceMode 180 181 // If the view has an invalid configuration, don't build the workspace 182 // module. 183 validBuildConfiguration := s.ValidBuildConfiguration() 184 if !validBuildConfiguration { 185 return mode 186 } 187 // If the view is not in a module and contains no modules, but still has a 188 // valid workspace configuration, do not create the workspace module. 189 // It could be using GOPATH or a different build system entirely. 190 if len(s.workspace.getActiveModFiles()) == 0 && validBuildConfiguration { 191 return mode 192 } 193 mode |= moduleMode 194 options := s.view.Options() 195 // The -modfile flag is available for Go versions >= 1.14. 196 if options.TempModfile && s.view.workspaceInformation.goversion >= 14 { 197 mode |= tempModfile 198 } 199 // If the user is intentionally limiting their workspace scope, don't 200 // enable multi-module workspace mode. 201 // TODO(rstambler): This should only change the calculation of the root, 202 // not the mode. 203 if !options.ExpandWorkspaceToModule { 204 return mode 205 } 206 // The workspace module has been disabled by the user. 207 if !options.ExperimentalWorkspaceModule { 208 return mode 209 } 210 mode |= usesWorkspaceModule 211 return mode 212} 213 214// config returns the configuration used for the snapshot's interaction with 215// the go/packages API. It uses the given working directory. 216// 217// TODO(rstambler): go/packages requires that we do not provide overlays for 218// multiple modules in on config, so buildOverlay needs to filter overlays by 219// module. 220func (s *snapshot) config(ctx context.Context, inv *gocommand.Invocation) *packages.Config { 221 s.view.optionsMu.Lock() 222 verboseOutput := s.view.options.VerboseOutput 223 s.view.optionsMu.Unlock() 224 225 cfg := &packages.Config{ 226 Context: ctx, 227 Dir: inv.WorkingDir, 228 Env: inv.Env, 229 BuildFlags: inv.BuildFlags, 230 Mode: packages.NeedName | 231 packages.NeedFiles | 232 packages.NeedCompiledGoFiles | 233 packages.NeedImports | 234 packages.NeedDeps | 235 packages.NeedTypesSizes | 236 packages.NeedModule, 237 Fset: s.view.session.cache.fset, 238 Overlay: s.buildOverlay(), 239 ParseFile: func(*token.FileSet, string, []byte) (*ast.File, error) { 240 panic("go/packages must not be used to parse files") 241 }, 242 Logf: func(format string, args ...interface{}) { 243 if verboseOutput { 244 event.Log(ctx, fmt.Sprintf(format, args...)) 245 } 246 }, 247 Tests: true, 248 } 249 packagesinternal.SetModFile(cfg, inv.ModFile) 250 packagesinternal.SetModFlag(cfg, inv.ModFlag) 251 // We want to type check cgo code if go/types supports it. 252 if typesinternal.SetUsesCgo(&types.Config{}) { 253 cfg.Mode |= packages.LoadMode(packagesinternal.TypecheckCgo) 254 } 255 packagesinternal.SetGoCmdRunner(cfg, s.view.session.gocmdRunner) 256 return cfg 257} 258 259func (s *snapshot) RunGoCommandDirect(ctx context.Context, mode source.InvocationFlags, inv *gocommand.Invocation) (*bytes.Buffer, error) { 260 _, inv, cleanup, err := s.goCommandInvocation(ctx, mode, inv) 261 if err != nil { 262 return nil, err 263 } 264 defer cleanup() 265 266 return s.view.session.gocmdRunner.Run(ctx, *inv) 267} 268 269func (s *snapshot) RunGoCommandPiped(ctx context.Context, mode source.InvocationFlags, inv *gocommand.Invocation, stdout, stderr io.Writer) error { 270 _, inv, cleanup, err := s.goCommandInvocation(ctx, mode, inv) 271 if err != nil { 272 return err 273 } 274 defer cleanup() 275 return s.view.session.gocmdRunner.RunPiped(ctx, *inv, stdout, stderr) 276} 277 278func (s *snapshot) RunGoCommands(ctx context.Context, allowNetwork bool, wd string, run func(invoke func(...string) (*bytes.Buffer, error)) error) (bool, []byte, []byte, error) { 279 var flags source.InvocationFlags 280 if s.workspaceMode()&tempModfile != 0 { 281 flags = source.WriteTemporaryModFile 282 } else { 283 flags = source.Normal 284 } 285 if allowNetwork { 286 flags |= source.AllowNetwork 287 } 288 tmpURI, inv, cleanup, err := s.goCommandInvocation(ctx, flags, &gocommand.Invocation{WorkingDir: wd}) 289 if err != nil { 290 return false, nil, nil, err 291 } 292 defer cleanup() 293 invoke := func(args ...string) (*bytes.Buffer, error) { 294 inv.Verb = args[0] 295 inv.Args = args[1:] 296 return s.view.session.gocmdRunner.Run(ctx, *inv) 297 } 298 if err := run(invoke); err != nil { 299 return false, nil, nil, err 300 } 301 if flags.Mode() != source.WriteTemporaryModFile { 302 return false, nil, nil, nil 303 } 304 var modBytes, sumBytes []byte 305 modBytes, err = ioutil.ReadFile(tmpURI.Filename()) 306 if err != nil && !os.IsNotExist(err) { 307 return false, nil, nil, err 308 } 309 sumBytes, err = ioutil.ReadFile(strings.TrimSuffix(tmpURI.Filename(), ".mod") + ".sum") 310 if err != nil && !os.IsNotExist(err) { 311 return false, nil, nil, err 312 } 313 return true, modBytes, sumBytes, nil 314} 315 316func (s *snapshot) goCommandInvocation(ctx context.Context, flags source.InvocationFlags, inv *gocommand.Invocation) (tmpURI span.URI, updatedInv *gocommand.Invocation, cleanup func(), err error) { 317 s.view.optionsMu.Lock() 318 allowModfileModificationOption := s.view.options.AllowModfileModifications 319 allowNetworkOption := s.view.options.AllowImplicitNetworkAccess 320 inv.Env = append(append(append(os.Environ(), s.view.options.EnvSlice()...), inv.Env...), "GO111MODULE="+s.view.effectiveGo111Module) 321 inv.BuildFlags = append([]string{}, s.view.options.BuildFlags...) 322 s.view.optionsMu.Unlock() 323 cleanup = func() {} // fallback 324 325 // All logic below is for module mode. 326 if s.workspaceMode()&moduleMode == 0 { 327 return "", inv, cleanup, nil 328 } 329 330 mode, allowNetwork := flags.Mode(), flags.AllowNetwork() 331 if !allowNetwork && !allowNetworkOption { 332 inv.Env = append(inv.Env, "GOPROXY=off") 333 } 334 335 var modURI span.URI 336 // Select the module context to use. 337 // If we're type checking, we need to use the workspace context, meaning 338 // the main (workspace) module. Otherwise, we should use the module for 339 // the passed-in working dir. 340 if mode == source.LoadWorkspace { 341 if s.workspaceMode()&usesWorkspaceModule == 0 { 342 for m := range s.workspace.getActiveModFiles() { // range to access the only element 343 modURI = m 344 } 345 } else { 346 var tmpDir span.URI 347 var err error 348 tmpDir, err = s.getWorkspaceDir(ctx) 349 if err != nil { 350 return "", nil, cleanup, err 351 } 352 inv.WorkingDir = tmpDir.Filename() 353 modURI = span.URIFromPath(filepath.Join(tmpDir.Filename(), "go.mod")) 354 } 355 } else { 356 modURI = s.GoModForFile(span.URIFromPath(inv.WorkingDir)) 357 } 358 359 var modContent []byte 360 if modURI != "" { 361 modFH, err := s.GetFile(ctx, modURI) 362 if err != nil { 363 return "", nil, cleanup, err 364 } 365 modContent, err = modFH.Read() 366 if err != nil { 367 return "", nil, cleanup, err 368 } 369 } 370 371 vendorEnabled, err := s.vendorEnabled(ctx, modURI, modContent) 372 if err != nil { 373 return "", nil, cleanup, err 374 } 375 376 // If the mod flag isn't set, populate it based on the mode and workspace. 377 if inv.ModFlag == "" { 378 mutableModFlag := "" 379 if s.view.goversion >= 16 { 380 mutableModFlag = "mod" 381 } 382 383 switch mode { 384 case source.LoadWorkspace, source.Normal: 385 if vendorEnabled { 386 inv.ModFlag = "vendor" 387 } else if !allowModfileModificationOption { 388 inv.ModFlag = "readonly" 389 } else { 390 inv.ModFlag = mutableModFlag 391 } 392 case source.UpdateUserModFile, source.WriteTemporaryModFile: 393 inv.ModFlag = mutableModFlag 394 } 395 } 396 397 wantTempMod := mode != source.UpdateUserModFile 398 needTempMod := mode == source.WriteTemporaryModFile 399 tempMod := wantTempMod && s.workspaceMode()&tempModfile != 0 400 if needTempMod && !tempMod { 401 return "", nil, cleanup, source.ErrTmpModfileUnsupported 402 } 403 404 if tempMod { 405 if modURI == "" { 406 return "", nil, cleanup, fmt.Errorf("no go.mod file found in %s", inv.WorkingDir) 407 } 408 modFH, err := s.GetFile(ctx, modURI) 409 if err != nil { 410 return "", nil, cleanup, err 411 } 412 // Use the go.sum if it happens to be available. 413 gosum := s.goSum(ctx, modURI) 414 tmpURI, cleanup, err = tempModFile(modFH, gosum) 415 if err != nil { 416 return "", nil, cleanup, err 417 } 418 inv.ModFile = tmpURI.Filename() 419 } 420 421 return tmpURI, inv, cleanup, nil 422} 423 424func (s *snapshot) buildOverlay() map[string][]byte { 425 s.mu.Lock() 426 defer s.mu.Unlock() 427 428 overlays := make(map[string][]byte) 429 for uri, fh := range s.files { 430 overlay, ok := fh.(*overlay) 431 if !ok { 432 continue 433 } 434 if overlay.saved { 435 continue 436 } 437 // TODO(rstambler): Make sure not to send overlays outside of the current view. 438 overlays[uri.Filename()] = overlay.text 439 } 440 return overlays 441} 442 443func hashUnsavedOverlays(files map[span.URI]source.VersionedFileHandle) string { 444 var unsaved []string 445 for uri, fh := range files { 446 if overlay, ok := fh.(*overlay); ok && !overlay.saved { 447 unsaved = append(unsaved, uri.Filename()) 448 } 449 } 450 sort.Strings(unsaved) 451 return hashContents([]byte(strings.Join(unsaved, ""))) 452} 453 454func (s *snapshot) PackagesForFile(ctx context.Context, uri span.URI, mode source.TypecheckMode) ([]source.Package, error) { 455 ctx = event.Label(ctx, tag.URI.Of(uri)) 456 457 phs, err := s.packageHandlesForFile(ctx, uri, mode) 458 if err != nil { 459 return nil, err 460 } 461 var pkgs []source.Package 462 for _, ph := range phs { 463 pkg, err := ph.check(ctx, s) 464 if err != nil { 465 return nil, err 466 } 467 pkgs = append(pkgs, pkg) 468 } 469 return pkgs, nil 470} 471 472func (s *snapshot) PackageForFile(ctx context.Context, uri span.URI, mode source.TypecheckMode, pkgPolicy source.PackageFilter) (source.Package, error) { 473 ctx = event.Label(ctx, tag.URI.Of(uri)) 474 475 phs, err := s.packageHandlesForFile(ctx, uri, mode) 476 if err != nil { 477 return nil, err 478 } 479 480 if len(phs) < 1 { 481 return nil, errors.Errorf("no packages") 482 } 483 484 ph := phs[0] 485 for _, handle := range phs[1:] { 486 switch pkgPolicy { 487 case source.WidestPackage: 488 if ph == nil || len(handle.CompiledGoFiles()) > len(ph.CompiledGoFiles()) { 489 ph = handle 490 } 491 case source.NarrowestPackage: 492 if ph == nil || len(handle.CompiledGoFiles()) < len(ph.CompiledGoFiles()) { 493 ph = handle 494 } 495 } 496 } 497 if ph == nil { 498 return nil, errors.Errorf("no packages in input") 499 } 500 501 return ph.check(ctx, s) 502} 503 504func (s *snapshot) packageHandlesForFile(ctx context.Context, uri span.URI, mode source.TypecheckMode) ([]*packageHandle, error) { 505 // Check if we should reload metadata for the file. We don't invalidate IDs 506 // (though we should), so the IDs will be a better source of truth than the 507 // metadata. If there are no IDs for the file, then we should also reload. 508 fh, err := s.GetFile(ctx, uri) 509 if err != nil { 510 return nil, err 511 } 512 if fh.Kind() != source.Go { 513 return nil, fmt.Errorf("no packages for non-Go file %s", uri) 514 } 515 knownIDs := s.getIDsForURI(uri) 516 reload := len(knownIDs) == 0 517 for _, id := range knownIDs { 518 // Reload package metadata if any of the metadata has missing 519 // dependencies, in case something has changed since the last time we 520 // reloaded it. 521 if s.noValidMetadataForID(id) { 522 reload = true 523 break 524 } 525 // TODO(golang/go#36918): Previously, we would reload any package with 526 // missing dependencies. This is expensive and results in too many 527 // calls to packages.Load. Determine what we should do instead. 528 } 529 if reload { 530 err = s.load(ctx, false, fileURI(uri)) 531 532 if !s.useInvalidMetadata() && err != nil { 533 return nil, err 534 } 535 // We've tried to reload and there are still no known IDs for the URI. 536 // Return the load error, if there was one. 537 knownIDs = s.getIDsForURI(uri) 538 if len(knownIDs) == 0 { 539 return nil, err 540 } 541 } 542 543 var phs []*packageHandle 544 for _, id := range knownIDs { 545 var parseModes []source.ParseMode 546 switch mode { 547 case source.TypecheckAll: 548 if s.workspaceParseMode(id) == source.ParseFull { 549 parseModes = []source.ParseMode{source.ParseFull} 550 } else { 551 parseModes = []source.ParseMode{source.ParseExported, source.ParseFull} 552 } 553 case source.TypecheckFull: 554 parseModes = []source.ParseMode{source.ParseFull} 555 case source.TypecheckWorkspace: 556 parseModes = []source.ParseMode{s.workspaceParseMode(id)} 557 } 558 559 for _, parseMode := range parseModes { 560 ph, err := s.buildPackageHandle(ctx, id, parseMode) 561 if err != nil { 562 return nil, err 563 } 564 phs = append(phs, ph) 565 } 566 } 567 return phs, nil 568} 569 570// Only use invalid metadata for Go versions >= 1.13. Go 1.12 and below has 571// issues with overlays that will cause confusing error messages if we reuse 572// old metadata. 573func (s *snapshot) useInvalidMetadata() bool { 574 return s.view.goversion >= 13 && s.view.Options().ExperimentalUseInvalidMetadata 575} 576 577func (s *snapshot) GetReverseDependencies(ctx context.Context, id string) ([]source.Package, error) { 578 if err := s.awaitLoaded(ctx); err != nil { 579 return nil, err 580 } 581 ids := make(map[packageID]struct{}) 582 s.transitiveReverseDependencies(packageID(id), ids) 583 584 // Make sure to delete the original package ID from the map. 585 delete(ids, packageID(id)) 586 587 var pkgs []source.Package 588 for id := range ids { 589 pkg, err := s.checkedPackage(ctx, id, s.workspaceParseMode(id)) 590 if err != nil { 591 return nil, err 592 } 593 pkgs = append(pkgs, pkg) 594 } 595 return pkgs, nil 596} 597 598func (s *snapshot) checkedPackage(ctx context.Context, id packageID, mode source.ParseMode) (*pkg, error) { 599 ph, err := s.buildPackageHandle(ctx, id, mode) 600 if err != nil { 601 return nil, err 602 } 603 return ph.check(ctx, s) 604} 605 606// transitiveReverseDependencies populates the ids map with package IDs 607// belonging to the provided package and its transitive reverse dependencies. 608func (s *snapshot) transitiveReverseDependencies(id packageID, ids map[packageID]struct{}) { 609 if _, ok := ids[id]; ok { 610 return 611 } 612 m := s.getMetadata(id) 613 // Only use invalid metadata if we support it. 614 if m == nil || !(m.valid || s.useInvalidMetadata()) { 615 return 616 } 617 ids[id] = struct{}{} 618 importedBy := s.getImportedBy(id) 619 for _, parentID := range importedBy { 620 s.transitiveReverseDependencies(parentID, ids) 621 } 622} 623 624func (s *snapshot) getGoFile(key parseKey) *parseGoHandle { 625 s.mu.Lock() 626 defer s.mu.Unlock() 627 return s.goFiles[key] 628} 629 630func (s *snapshot) addGoFile(key parseKey, pgh *parseGoHandle) *parseGoHandle { 631 s.mu.Lock() 632 defer s.mu.Unlock() 633 if existing, ok := s.goFiles[key]; ok { 634 return existing 635 } 636 s.goFiles[key] = pgh 637 return pgh 638} 639 640func (s *snapshot) getParseModHandle(uri span.URI) *parseModHandle { 641 s.mu.Lock() 642 defer s.mu.Unlock() 643 return s.parseModHandles[uri] 644} 645 646func (s *snapshot) getModWhyHandle(uri span.URI) *modWhyHandle { 647 s.mu.Lock() 648 defer s.mu.Unlock() 649 return s.modWhyHandles[uri] 650} 651 652func (s *snapshot) getModTidyHandle(uri span.URI) *modTidyHandle { 653 s.mu.Lock() 654 defer s.mu.Unlock() 655 return s.modTidyHandles[uri] 656} 657 658func (s *snapshot) getImportedBy(id packageID) []packageID { 659 s.mu.Lock() 660 defer s.mu.Unlock() 661 return s.getImportedByLocked(id) 662} 663 664func (s *snapshot) getImportedByLocked(id packageID) []packageID { 665 // If we haven't rebuilt the import graph since creating the snapshot. 666 if len(s.importedBy) == 0 { 667 s.rebuildImportGraph() 668 } 669 return s.importedBy[id] 670} 671 672func (s *snapshot) clearAndRebuildImportGraph() { 673 s.mu.Lock() 674 defer s.mu.Unlock() 675 676 // Completely invalidate the original map. 677 s.importedBy = make(map[packageID][]packageID) 678 s.rebuildImportGraph() 679} 680 681func (s *snapshot) rebuildImportGraph() { 682 for id, m := range s.metadata { 683 for _, importID := range m.deps { 684 s.importedBy[importID] = append(s.importedBy[importID], id) 685 } 686 } 687} 688 689func (s *snapshot) addPackageHandle(ph *packageHandle) *packageHandle { 690 s.mu.Lock() 691 defer s.mu.Unlock() 692 693 // If the package handle has already been cached, 694 // return the cached handle instead of overriding it. 695 if ph, ok := s.packages[ph.packageKey()]; ok { 696 return ph 697 } 698 s.packages[ph.packageKey()] = ph 699 return ph 700} 701 702func (s *snapshot) workspacePackageIDs() (ids []packageID) { 703 s.mu.Lock() 704 defer s.mu.Unlock() 705 706 for id := range s.workspacePackages { 707 ids = append(ids, id) 708 } 709 return ids 710} 711 712func (s *snapshot) fileWatchingGlobPatterns(ctx context.Context) map[string]struct{} { 713 // Work-around microsoft/vscode#100870 by making sure that we are, 714 // at least, watching the user's entire workspace. This will still be 715 // applied to every folder in the workspace. 716 patterns := map[string]struct{}{ 717 "**/*.{go,mod,sum}": {}, 718 "**/*.*tmpl": {}, 719 } 720 dirs := s.workspace.dirs(ctx, s) 721 for _, dir := range dirs { 722 dirName := dir.Filename() 723 724 // If the directory is within the view's folder, we're already watching 725 // it with the pattern above. 726 if source.InDir(s.view.folder.Filename(), dirName) { 727 continue 728 } 729 // TODO(rstambler): If microsoft/vscode#3025 is resolved before 730 // microsoft/vscode#101042, we will need a work-around for Windows 731 // drive letter casing. 732 patterns[fmt.Sprintf("%s/**/*.{go,mod,sum,tmpl}", dirName)] = struct{}{} 733 } 734 735 // Some clients do not send notifications for changes to directories that 736 // contain Go code (golang/go#42348). To handle this, explicitly watch all 737 // of the directories in the workspace. We find them by adding the 738 // directories of every file in the snapshot's workspace directories. 739 var dirNames []string 740 for uri := range s.allKnownSubdirs(ctx) { 741 dirNames = append(dirNames, uri.Filename()) 742 } 743 sort.Strings(dirNames) 744 if len(dirNames) > 0 { 745 patterns[fmt.Sprintf("{%s}", strings.Join(dirNames, ","))] = struct{}{} 746 } 747 return patterns 748} 749 750// allKnownSubdirs returns all of the subdirectories within the snapshot's 751// workspace directories. None of the workspace directories are included. 752func (s *snapshot) allKnownSubdirs(ctx context.Context) map[span.URI]struct{} { 753 dirs := s.workspace.dirs(ctx, s) 754 755 s.mu.Lock() 756 defer s.mu.Unlock() 757 seen := make(map[span.URI]struct{}) 758 for uri := range s.files { 759 dir := filepath.Dir(uri.Filename()) 760 var matched span.URI 761 for _, wsDir := range dirs { 762 if source.InDir(wsDir.Filename(), dir) { 763 matched = wsDir 764 break 765 } 766 } 767 // Don't watch any directory outside of the workspace directories. 768 if matched == "" { 769 continue 770 } 771 for { 772 if dir == "" || dir == matched.Filename() { 773 break 774 } 775 uri := span.URIFromPath(dir) 776 if _, ok := seen[uri]; ok { 777 break 778 } 779 seen[uri] = struct{}{} 780 dir = filepath.Dir(dir) 781 } 782 } 783 return seen 784} 785 786// knownFilesInDir returns the files known to the given snapshot that are in 787// the given directory. It does not respect symlinks. 788func (s *snapshot) knownFilesInDir(ctx context.Context, dir span.URI) []span.URI { 789 var files []span.URI 790 s.mu.Lock() 791 defer s.mu.Unlock() 792 793 for uri := range s.files { 794 if source.InDir(dir.Filename(), uri.Filename()) { 795 files = append(files, uri) 796 } 797 } 798 return files 799} 800 801func (s *snapshot) WorkspacePackages(ctx context.Context) ([]source.Package, error) { 802 phs, err := s.workspacePackageHandles(ctx) 803 if err != nil { 804 return nil, err 805 } 806 var pkgs []source.Package 807 for _, ph := range phs { 808 pkg, err := ph.check(ctx, s) 809 if err != nil { 810 return nil, err 811 } 812 pkgs = append(pkgs, pkg) 813 } 814 return pkgs, nil 815} 816 817func (s *snapshot) workspacePackageHandles(ctx context.Context) ([]*packageHandle, error) { 818 if err := s.awaitLoaded(ctx); err != nil { 819 return nil, err 820 } 821 var phs []*packageHandle 822 for _, pkgID := range s.workspacePackageIDs() { 823 ph, err := s.buildPackageHandle(ctx, pkgID, s.workspaceParseMode(pkgID)) 824 if err != nil { 825 return nil, err 826 } 827 phs = append(phs, ph) 828 } 829 return phs, nil 830} 831 832func (s *snapshot) KnownPackages(ctx context.Context) ([]source.Package, error) { 833 if err := s.awaitLoaded(ctx); err != nil { 834 return nil, err 835 } 836 837 // The WorkspaceSymbols implementation relies on this function returning 838 // workspace packages first. 839 ids := s.workspacePackageIDs() 840 s.mu.Lock() 841 for id := range s.metadata { 842 if _, ok := s.workspacePackages[id]; ok { 843 continue 844 } 845 ids = append(ids, id) 846 } 847 s.mu.Unlock() 848 849 var pkgs []source.Package 850 for _, id := range ids { 851 pkg, err := s.checkedPackage(ctx, id, s.workspaceParseMode(id)) 852 if err != nil { 853 return nil, err 854 } 855 pkgs = append(pkgs, pkg) 856 } 857 return pkgs, nil 858} 859 860func (s *snapshot) CachedImportPaths(ctx context.Context) (map[string]source.Package, error) { 861 // Don't reload workspace package metadata. 862 // This function is meant to only return currently cached information. 863 s.AwaitInitialized(ctx) 864 865 s.mu.Lock() 866 defer s.mu.Unlock() 867 868 results := map[string]source.Package{} 869 for _, ph := range s.packages { 870 cachedPkg, err := ph.cached(s.generation) 871 if err != nil { 872 continue 873 } 874 for importPath, newPkg := range cachedPkg.imports { 875 if oldPkg, ok := results[string(importPath)]; ok { 876 // Using the same trick as NarrowestPackage, prefer non-variants. 877 if len(newPkg.compiledGoFiles) < len(oldPkg.(*pkg).compiledGoFiles) { 878 results[string(importPath)] = newPkg 879 } 880 } else { 881 results[string(importPath)] = newPkg 882 } 883 } 884 } 885 return results, nil 886} 887 888func (s *snapshot) GoModForFile(uri span.URI) span.URI { 889 return moduleForURI(s.workspace.activeModFiles, uri) 890} 891 892func moduleForURI(modFiles map[span.URI]struct{}, uri span.URI) span.URI { 893 var match span.URI 894 for modURI := range modFiles { 895 if !source.InDir(dirURI(modURI).Filename(), uri.Filename()) { 896 continue 897 } 898 if len(modURI) > len(match) { 899 match = modURI 900 } 901 } 902 return match 903} 904 905func (s *snapshot) getPackage(id packageID, mode source.ParseMode) *packageHandle { 906 s.mu.Lock() 907 defer s.mu.Unlock() 908 909 key := packageKey{ 910 id: id, 911 mode: mode, 912 } 913 return s.packages[key] 914} 915 916func (s *snapshot) getActionHandle(id packageID, m source.ParseMode, a *analysis.Analyzer) *actionHandle { 917 s.mu.Lock() 918 defer s.mu.Unlock() 919 920 key := actionKey{ 921 pkg: packageKey{ 922 id: id, 923 mode: m, 924 }, 925 analyzer: a, 926 } 927 return s.actions[key] 928} 929 930func (s *snapshot) addActionHandle(ah *actionHandle) *actionHandle { 931 s.mu.Lock() 932 defer s.mu.Unlock() 933 934 key := actionKey{ 935 analyzer: ah.analyzer, 936 pkg: packageKey{ 937 id: ah.pkg.m.id, 938 mode: ah.pkg.mode, 939 }, 940 } 941 if ah, ok := s.actions[key]; ok { 942 return ah 943 } 944 s.actions[key] = ah 945 return ah 946} 947 948func (s *snapshot) getIDsForURI(uri span.URI) []packageID { 949 s.mu.Lock() 950 defer s.mu.Unlock() 951 952 return s.ids[uri] 953} 954 955func (s *snapshot) getMetadata(id packageID) *knownMetadata { 956 s.mu.Lock() 957 defer s.mu.Unlock() 958 959 return s.metadata[id] 960} 961 962// noValidMetadataForURILocked reports whether there is any valid metadata for 963// the given URI. 964func (s *snapshot) noValidMetadataForURILocked(uri span.URI) bool { 965 ids, ok := s.ids[uri] 966 if !ok { 967 return true 968 } 969 for _, id := range ids { 970 if m, ok := s.metadata[id]; ok && m.valid { 971 return false 972 } 973 } 974 return true 975} 976 977// noValidMetadataForID reports whether there is no valid metadata for the 978// given ID. 979func (s *snapshot) noValidMetadataForID(id packageID) bool { 980 m := s.getMetadata(id) 981 return m == nil || !m.valid 982} 983 984// addID adds the given ID to the set of known IDs for the given URI. 985// Any existing invalid IDs are not preserved, and IDs that are not 986// "command-line-arguments" are preferred. 987func (s *snapshot) addID(uri span.URI, id packageID) { 988 s.mu.Lock() 989 defer s.mu.Unlock() 990 991 var newIDs []packageID 992 for i, existingID := range s.ids[uri] { 993 if existingID == id { 994 return 995 } 996 // If the package previously only had a command-line-arguments ID, 997 // we should just replace it. 998 if source.IsCommandLineArguments(string(existingID)) { 999 s.ids[uri][i] = id 1000 // Delete command-line-arguments if it was a workspace package. 1001 delete(s.workspacePackages, existingID) 1002 return 1003 } 1004 if m, ok := s.metadata[existingID]; !ok || !m.valid { 1005 continue 1006 } 1007 newIDs = append(newIDs, existingID) 1008 } 1009 sort.Slice(newIDs, func(i, j int) bool { 1010 return newIDs[i] < newIDs[j] 1011 }) 1012 s.ids[uri] = append(newIDs, id) 1013} 1014 1015func (s *snapshot) isWorkspacePackage(id packageID) bool { 1016 s.mu.Lock() 1017 defer s.mu.Unlock() 1018 1019 _, ok := s.workspacePackages[id] 1020 return ok 1021} 1022 1023func (s *snapshot) FindFile(uri span.URI) source.VersionedFileHandle { 1024 f := s.view.getFile(uri) 1025 1026 s.mu.Lock() 1027 defer s.mu.Unlock() 1028 1029 return s.files[f.URI()] 1030} 1031 1032// GetVersionedFile returns a File for the given URI. If the file is unknown it 1033// is added to the managed set. 1034// 1035// GetVersionedFile succeeds even if the file does not exist. A non-nil error return 1036// indicates some type of internal error, for example if ctx is cancelled. 1037func (s *snapshot) GetVersionedFile(ctx context.Context, uri span.URI) (source.VersionedFileHandle, error) { 1038 f := s.view.getFile(uri) 1039 1040 s.mu.Lock() 1041 defer s.mu.Unlock() 1042 return s.getFileLocked(ctx, f) 1043} 1044 1045// GetFile implements the fileSource interface by wrapping GetVersionedFile. 1046func (s *snapshot) GetFile(ctx context.Context, uri span.URI) (source.FileHandle, error) { 1047 return s.GetVersionedFile(ctx, uri) 1048} 1049 1050func (s *snapshot) getFileLocked(ctx context.Context, f *fileBase) (source.VersionedFileHandle, error) { 1051 if fh, ok := s.files[f.URI()]; ok { 1052 return fh, nil 1053 } 1054 1055 fh, err := s.view.session.cache.getFile(ctx, f.URI()) 1056 if err != nil { 1057 return nil, err 1058 } 1059 closed := &closedFile{fh} 1060 s.files[f.URI()] = closed 1061 return closed, nil 1062} 1063 1064func (s *snapshot) IsOpen(uri span.URI) bool { 1065 s.mu.Lock() 1066 defer s.mu.Unlock() 1067 return s.isOpenLocked(uri) 1068 1069} 1070 1071func (s *snapshot) openFiles() []source.VersionedFileHandle { 1072 s.mu.Lock() 1073 defer s.mu.Unlock() 1074 1075 var open []source.VersionedFileHandle 1076 for _, fh := range s.files { 1077 if s.isOpenLocked(fh.URI()) { 1078 open = append(open, fh) 1079 } 1080 } 1081 return open 1082} 1083 1084func (s *snapshot) isOpenLocked(uri span.URI) bool { 1085 _, open := s.files[uri].(*overlay) 1086 return open 1087} 1088 1089func (s *snapshot) awaitLoaded(ctx context.Context) error { 1090 loadErr := s.awaitLoadedAllErrors(ctx) 1091 1092 // If we still have absolutely no metadata, check if the view failed to 1093 // initialize and return any errors. 1094 // TODO(rstambler): Should we clear the error after we return it? 1095 useInvalidMetadata := s.useInvalidMetadata() 1096 1097 s.mu.Lock() 1098 defer s.mu.Unlock() 1099 1100 // Only return a load error if we have no valid metadata. 1101 if useInvalidMetadata && len(s.metadata) > 0 { 1102 return nil 1103 } 1104 for _, m := range s.metadata { 1105 if m.valid { 1106 return nil 1107 } 1108 } 1109 if loadErr != nil { 1110 return loadErr.MainError 1111 } 1112 return nil 1113} 1114 1115func (s *snapshot) GetCriticalError(ctx context.Context) *source.CriticalError { 1116 loadErr := s.awaitLoadedAllErrors(ctx) 1117 if loadErr != nil && errors.Is(loadErr.MainError, context.Canceled) { 1118 return nil 1119 } 1120 1121 // Even if packages didn't fail to load, we still may want to show 1122 // additional warnings. 1123 if loadErr == nil { 1124 wsPkgs, _ := s.WorkspacePackages(ctx) 1125 if msg := shouldShowAdHocPackagesWarning(s, wsPkgs); msg != "" { 1126 return &source.CriticalError{ 1127 MainError: errors.New(msg), 1128 } 1129 } 1130 // Even if workspace packages were returned, there still may be an error 1131 // with the user's workspace layout. Workspace packages that only have the 1132 // ID "command-line-arguments" are usually a symptom of a bad workspace 1133 // configuration. 1134 if containsCommandLineArguments(wsPkgs) { 1135 return s.workspaceLayoutError(ctx) 1136 } 1137 return nil 1138 } 1139 1140 if errMsg := loadErr.MainError.Error(); strings.Contains(errMsg, "cannot find main module") || strings.Contains(errMsg, "go.mod file not found") { 1141 return s.workspaceLayoutError(ctx) 1142 } 1143 return loadErr 1144} 1145 1146const adHocPackagesWarning = `You are outside of a module and outside of $GOPATH/src. 1147If you are using modules, please open your editor to a directory in your module. 1148If you believe this warning is incorrect, please file an issue: https://github.com/golang/go/issues/new.` 1149 1150func shouldShowAdHocPackagesWarning(snapshot source.Snapshot, pkgs []source.Package) string { 1151 if snapshot.ValidBuildConfiguration() { 1152 return "" 1153 } 1154 for _, pkg := range pkgs { 1155 if len(pkg.MissingDependencies()) > 0 { 1156 return adHocPackagesWarning 1157 } 1158 } 1159 return "" 1160} 1161 1162func containsCommandLineArguments(pkgs []source.Package) bool { 1163 for _, pkg := range pkgs { 1164 if source.IsCommandLineArguments(pkg.ID()) { 1165 return true 1166 } 1167 } 1168 return false 1169} 1170 1171func (s *snapshot) awaitLoadedAllErrors(ctx context.Context) *source.CriticalError { 1172 // Do not return results until the snapshot's view has been initialized. 1173 s.AwaitInitialized(ctx) 1174 1175 // TODO(rstambler): Should we be more careful about returning the 1176 // initialization error? Is it possible for the initialization error to be 1177 // corrected without a successful reinitialization? 1178 s.mu.Lock() 1179 initializedErr := s.initializedErr 1180 s.mu.Unlock() 1181 if initializedErr != nil { 1182 return initializedErr 1183 } 1184 1185 if ctx.Err() != nil { 1186 return &source.CriticalError{MainError: ctx.Err()} 1187 } 1188 1189 if err := s.reloadWorkspace(ctx); err != nil { 1190 diags, _ := s.extractGoCommandErrors(ctx, err.Error()) 1191 return &source.CriticalError{ 1192 MainError: err, 1193 DiagList: diags, 1194 } 1195 } 1196 if err := s.reloadOrphanedFiles(ctx); err != nil { 1197 diags, _ := s.extractGoCommandErrors(ctx, err.Error()) 1198 return &source.CriticalError{ 1199 MainError: err, 1200 DiagList: diags, 1201 } 1202 } 1203 return nil 1204} 1205 1206func (s *snapshot) AwaitInitialized(ctx context.Context) { 1207 select { 1208 case <-ctx.Done(): 1209 return 1210 case <-s.view.initialWorkspaceLoad: 1211 } 1212 // We typically prefer to run something as intensive as the IWL without 1213 // blocking. I'm not sure if there is a way to do that here. 1214 s.initialize(ctx, false) 1215} 1216 1217// reloadWorkspace reloads the metadata for all invalidated workspace packages. 1218func (s *snapshot) reloadWorkspace(ctx context.Context) error { 1219 useInvalidMetadata := s.useInvalidMetadata() 1220 1221 // See which of the workspace packages are missing metadata. 1222 s.mu.Lock() 1223 missingMetadata := len(s.workspacePackages) == 0 || len(s.metadata) == 0 1224 pkgPathSet := map[packagePath]struct{}{} 1225 for id, pkgPath := range s.workspacePackages { 1226 if m, ok := s.metadata[id]; ok && (m.valid || useInvalidMetadata) { 1227 continue 1228 } 1229 missingMetadata = true 1230 1231 // Don't try to reload "command-line-arguments" directly. 1232 if source.IsCommandLineArguments(string(pkgPath)) { 1233 continue 1234 } 1235 pkgPathSet[pkgPath] = struct{}{} 1236 } 1237 s.mu.Unlock() 1238 1239 // If the view's build configuration is invalid, we cannot reload by 1240 // package path. Just reload the directory instead. 1241 if missingMetadata && !s.ValidBuildConfiguration() { 1242 return s.load(ctx, false, viewLoadScope("LOAD_INVALID_VIEW")) 1243 } 1244 1245 if len(pkgPathSet) == 0 { 1246 return nil 1247 } 1248 1249 var pkgPaths []interface{} 1250 for pkgPath := range pkgPathSet { 1251 pkgPaths = append(pkgPaths, pkgPath) 1252 } 1253 return s.load(ctx, false, pkgPaths...) 1254} 1255 1256func (s *snapshot) reloadOrphanedFiles(ctx context.Context) error { 1257 // When we load ./... or a package path directly, we may not get packages 1258 // that exist only in overlays. As a workaround, we search all of the files 1259 // available in the snapshot and reload their metadata individually using a 1260 // file= query if the metadata is unavailable. 1261 files := s.orphanedFiles() 1262 1263 // Files without a valid package declaration can't be loaded. Don't try. 1264 var scopes []interface{} 1265 for _, file := range files { 1266 pgf, err := s.ParseGo(ctx, file, source.ParseHeader) 1267 if err != nil { 1268 continue 1269 } 1270 if !pgf.File.Package.IsValid() { 1271 continue 1272 } 1273 scopes = append(scopes, fileURI(file.URI())) 1274 } 1275 1276 if len(scopes) == 0 { 1277 return nil 1278 } 1279 1280 // The regtests match this exact log message, keep them in sync. 1281 event.Log(ctx, "reloadOrphanedFiles reloading", tag.Query.Of(scopes)) 1282 err := s.load(ctx, false, scopes...) 1283 1284 // If we failed to load some files, i.e. they have no metadata, 1285 // mark the failures so we don't bother retrying until the file's 1286 // content changes. 1287 // 1288 // TODO(rstambler): This may be an overestimate if the load stopped 1289 // early for an unrelated errors. Add a fallback? 1290 // 1291 // Check for context cancellation so that we don't incorrectly mark files 1292 // as unloadable, but don't return before setting all workspace packages. 1293 if ctx.Err() == nil && err != nil { 1294 event.Error(ctx, "reloadOrphanedFiles: failed to load", err, tag.Query.Of(scopes)) 1295 s.mu.Lock() 1296 for _, scope := range scopes { 1297 uri := span.URI(scope.(fileURI)) 1298 if s.noValidMetadataForURILocked(uri) { 1299 s.unloadableFiles[uri] = struct{}{} 1300 } 1301 } 1302 s.mu.Unlock() 1303 } 1304 return nil 1305} 1306 1307func (s *snapshot) orphanedFiles() []source.VersionedFileHandle { 1308 s.mu.Lock() 1309 defer s.mu.Unlock() 1310 1311 var files []source.VersionedFileHandle 1312 for uri, fh := range s.files { 1313 // Don't try to reload metadata for go.mod files. 1314 if fh.Kind() != source.Go { 1315 continue 1316 } 1317 // If the URI doesn't belong to this view, then it's not in a workspace 1318 // package and should not be reloaded directly. 1319 if !contains(s.view.session.viewsOf(uri), s.view) { 1320 continue 1321 } 1322 // If the file is not open and is in a vendor directory, don't treat it 1323 // like a workspace package. 1324 if _, ok := fh.(*overlay); !ok && inVendor(uri) { 1325 continue 1326 } 1327 // Don't reload metadata for files we've already deemed unloadable. 1328 if _, ok := s.unloadableFiles[uri]; ok { 1329 continue 1330 } 1331 if s.noValidMetadataForURILocked(uri) { 1332 files = append(files, fh) 1333 } 1334 } 1335 return files 1336} 1337 1338func contains(views []*View, view *View) bool { 1339 for _, v := range views { 1340 if v == view { 1341 return true 1342 } 1343 } 1344 return false 1345} 1346 1347func inVendor(uri span.URI) bool { 1348 if !strings.Contains(string(uri), "/vendor/") { 1349 return false 1350 } 1351 // Only packages in _subdirectories_ of /vendor/ are considered vendored 1352 // (/vendor/a/foo.go is vendored, /vendor/foo.go is not). 1353 split := strings.Split(string(uri), "/vendor/") 1354 if len(split) < 2 { 1355 return false 1356 } 1357 return strings.Contains(split[1], "/") 1358} 1359 1360func generationName(v *View, snapshotID uint64) string { 1361 return fmt.Sprintf("v%v/%v", v.id, snapshotID) 1362} 1363 1364// checkSnapshotLocked verifies that some invariants are preserved on the 1365// snapshot. 1366func checkSnapshotLocked(ctx context.Context, s *snapshot) { 1367 // Check that every go file for a workspace package is identified as 1368 // belonging to that workspace package. 1369 for wsID := range s.workspacePackages { 1370 if m, ok := s.metadata[wsID]; ok { 1371 for _, uri := range m.goFiles { 1372 found := false 1373 for _, id := range s.ids[uri] { 1374 if id == wsID { 1375 found = true 1376 break 1377 } 1378 } 1379 if !found { 1380 log.Error.Logf(ctx, "workspace package %v not associated with %v", wsID, uri) 1381 } 1382 } 1383 } 1384 } 1385} 1386 1387func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (*snapshot, bool) { 1388 var vendorChanged bool 1389 newWorkspace, workspaceChanged, workspaceReload := s.workspace.invalidate(ctx, changes) 1390 1391 s.mu.Lock() 1392 defer s.mu.Unlock() 1393 1394 checkSnapshotLocked(ctx, s) 1395 1396 newGen := s.view.session.cache.store.Generation(generationName(s.view, s.id+1)) 1397 bgCtx, cancel := context.WithCancel(bgCtx) 1398 result := &snapshot{ 1399 id: s.id + 1, 1400 generation: newGen, 1401 view: s.view, 1402 backgroundCtx: bgCtx, 1403 cancel: cancel, 1404 builtin: s.builtin, 1405 initializeOnce: s.initializeOnce, 1406 initializedErr: s.initializedErr, 1407 ids: make(map[span.URI][]packageID, len(s.ids)), 1408 importedBy: make(map[packageID][]packageID, len(s.importedBy)), 1409 metadata: make(map[packageID]*knownMetadata, len(s.metadata)), 1410 packages: make(map[packageKey]*packageHandle, len(s.packages)), 1411 actions: make(map[actionKey]*actionHandle, len(s.actions)), 1412 files: make(map[span.URI]source.VersionedFileHandle, len(s.files)), 1413 goFiles: make(map[parseKey]*parseGoHandle, len(s.goFiles)), 1414 workspacePackages: make(map[packageID]packagePath, len(s.workspacePackages)), 1415 unloadableFiles: make(map[span.URI]struct{}, len(s.unloadableFiles)), 1416 parseModHandles: make(map[span.URI]*parseModHandle, len(s.parseModHandles)), 1417 modTidyHandles: make(map[span.URI]*modTidyHandle, len(s.modTidyHandles)), 1418 modWhyHandles: make(map[span.URI]*modWhyHandle, len(s.modWhyHandles)), 1419 workspace: newWorkspace, 1420 } 1421 1422 if !workspaceChanged && s.workspaceDirHandle != nil { 1423 result.workspaceDirHandle = s.workspaceDirHandle 1424 newGen.Inherit(s.workspaceDirHandle) 1425 } 1426 1427 // Copy all of the FileHandles. 1428 for k, v := range s.files { 1429 result.files[k] = v 1430 } 1431 1432 // Copy the set of unloadable files. 1433 for k, v := range s.unloadableFiles { 1434 result.unloadableFiles[k] = v 1435 } 1436 // Copy all of the modHandles. 1437 for k, v := range s.parseModHandles { 1438 result.parseModHandles[k] = v 1439 } 1440 1441 for k, v := range s.goFiles { 1442 if _, ok := changes[k.file.URI]; ok { 1443 continue 1444 } 1445 newGen.Inherit(v.handle) 1446 result.goFiles[k] = v 1447 } 1448 1449 // Copy all of the go.mod-related handles. They may be invalidated later, 1450 // so we inherit them at the end of the function. 1451 for k, v := range s.modTidyHandles { 1452 if _, ok := changes[k]; ok { 1453 continue 1454 } 1455 result.modTidyHandles[k] = v 1456 } 1457 for k, v := range s.modWhyHandles { 1458 if _, ok := changes[k]; ok { 1459 continue 1460 } 1461 result.modWhyHandles[k] = v 1462 } 1463 1464 // directIDs keeps track of package IDs that have directly changed. 1465 // It maps id->invalidateMetadata. 1466 directIDs := map[packageID]bool{} 1467 // Invalidate all package metadata if the workspace module has changed. 1468 if workspaceReload { 1469 for k := range s.metadata { 1470 directIDs[k] = true 1471 } 1472 } 1473 1474 changedPkgNames := map[packageID]struct{}{} 1475 for uri, change := range changes { 1476 // Maybe reinitialize the view if we see a change in the vendor 1477 // directory. 1478 if inVendor(uri) { 1479 vendorChanged = true 1480 } 1481 1482 // The original FileHandle for this URI is cached on the snapshot. 1483 originalFH := s.files[uri] 1484 1485 // Check if the file's package name or imports have changed, 1486 // and if so, invalidate this file's packages' metadata. 1487 shouldInvalidateMetadata, pkgNameChanged := s.shouldInvalidateMetadata(ctx, result, originalFH, change.fileHandle) 1488 invalidateMetadata := forceReloadMetadata || workspaceReload || shouldInvalidateMetadata 1489 1490 // Mark all of the package IDs containing the given file. 1491 // TODO: if the file has moved into a new package, we should invalidate that too. 1492 filePackageIDs := guessPackageIDsForURI(uri, s.ids) 1493 if pkgNameChanged { 1494 for _, id := range filePackageIDs { 1495 changedPkgNames[id] = struct{}{} 1496 } 1497 } 1498 for _, id := range filePackageIDs { 1499 directIDs[id] = directIDs[id] || invalidateMetadata 1500 } 1501 1502 // Invalidate the previous modTidyHandle if any of the files have been 1503 // saved or if any of the metadata has been invalidated. 1504 if invalidateMetadata || fileWasSaved(originalFH, change.fileHandle) { 1505 // TODO(rstambler): Only delete mod handles for which the 1506 // withoutURI is relevant. 1507 for k := range s.modTidyHandles { 1508 delete(result.modTidyHandles, k) 1509 } 1510 for k := range s.modWhyHandles { 1511 delete(result.modWhyHandles, k) 1512 } 1513 } 1514 if isGoMod(uri) { 1515 delete(result.parseModHandles, uri) 1516 } 1517 // Handle the invalidated file; it may have new contents or not exist. 1518 if !change.exists { 1519 delete(result.files, uri) 1520 } else { 1521 result.files[uri] = change.fileHandle 1522 } 1523 1524 // Make sure to remove the changed file from the unloadable set. 1525 delete(result.unloadableFiles, uri) 1526 } 1527 1528 // Invalidate reverse dependencies too. 1529 // TODO(heschi): figure out the locking model and use transitiveReverseDeps? 1530 // transitiveIDs keeps track of transitive reverse dependencies. 1531 // If an ID is present in the map, invalidate its types. 1532 // If an ID's value is true, invalidate its metadata too. 1533 idsToInvalidate := make(map[packageID]bool) 1534 var addRevDeps func(packageID, bool) 1535 addRevDeps = func(id packageID, invalidateMetadata bool) { 1536 current, seen := idsToInvalidate[id] 1537 newInvalidateMetadata := current || invalidateMetadata 1538 1539 // If we've already seen this ID, and the value of invalidate 1540 // metadata has not changed, we can return early. 1541 if seen && current == newInvalidateMetadata { 1542 return 1543 } 1544 idsToInvalidate[id] = newInvalidateMetadata 1545 for _, rid := range s.getImportedByLocked(id) { 1546 addRevDeps(rid, invalidateMetadata) 1547 } 1548 } 1549 for id, invalidateMetadata := range directIDs { 1550 addRevDeps(id, invalidateMetadata) 1551 } 1552 1553 // Copy the package type information. 1554 for k, v := range s.packages { 1555 if _, ok := idsToInvalidate[k.id]; ok { 1556 continue 1557 } 1558 newGen.Inherit(v.handle) 1559 result.packages[k] = v 1560 } 1561 // Copy the package analysis information. 1562 for k, v := range s.actions { 1563 if _, ok := idsToInvalidate[k.pkg.id]; ok { 1564 continue 1565 } 1566 newGen.Inherit(v.handle) 1567 result.actions[k] = v 1568 } 1569 1570 // If the workspace mode has changed or a file has been deleted, we *must* 1571 // delete all metadata, as it is unusable and may produce confusing or 1572 // incorrect diagnostics. 1573 workspaceModeChanged := s.workspaceMode() != result.workspaceMode() 1574 deletedFiles := map[span.URI]struct{}{} 1575 for _, c := range changes { 1576 if c.exists { 1577 continue 1578 } 1579 deletedFiles[c.fileHandle.URI()] = struct{}{} 1580 } 1581 skipID := map[packageID]struct{}{} 1582 for uri, ids := range s.ids { 1583 if _, ok := deletedFiles[uri]; ok { 1584 for _, id := range ids { 1585 skipID[id] = struct{}{} 1586 } 1587 } 1588 } 1589 1590 // Copy the URI to package ID mappings, skipping only those URIs whose 1591 // metadata will be reloaded in future calls to load. 1592 deleteInvalidMetadata := forceReloadMetadata || workspaceModeChanged 1593 idsInSnapshot := map[packageID]bool{} // track all known IDs 1594 for uri, ids := range s.ids { 1595 for _, id := range ids { 1596 invalidateMetadata := idsToInvalidate[id] 1597 if _, ok := skipID[id]; ok || (invalidateMetadata && deleteInvalidMetadata) { 1598 continue 1599 } 1600 idsInSnapshot[id] = true 1601 result.ids[uri] = append(result.ids[uri], id) 1602 } 1603 } 1604 1605 // Copy the package metadata. We only need to invalidate packages directly 1606 // containing the affected file, and only if it changed in a relevant way. 1607 for k, v := range s.metadata { 1608 if !idsInSnapshot[k] { 1609 // Delete metadata for IDs that are no longer reachable from files 1610 // in the snapshot. 1611 continue 1612 } 1613 invalidateMetadata := idsToInvalidate[k] 1614 // Mark invalidated metadata rather than deleting it outright. 1615 result.metadata[k] = &knownMetadata{ 1616 metadata: v.metadata, 1617 valid: v.valid && !invalidateMetadata, 1618 } 1619 } 1620 // Copy the URI to package ID mappings, skipping only those URIs whose 1621 // metadata will be reloaded in future calls to load. 1622 for k, ids := range s.ids { 1623 var newIDs []packageID 1624 for _, id := range ids { 1625 if invalidateMetadata, ok := idsToInvalidate[id]; invalidateMetadata && ok { 1626 continue 1627 } 1628 newIDs = append(newIDs, id) 1629 } 1630 if len(newIDs) != 0 { 1631 result.ids[k] = newIDs 1632 } 1633 } 1634 1635 // Copy the set of initially loaded packages. 1636 for id, pkgPath := range s.workspacePackages { 1637 // Packages with the id "command-line-arguments" are generated by the 1638 // go command when the user is outside of GOPATH and outside of a 1639 // module. Do not cache them as workspace packages for longer than 1640 // necessary. 1641 if source.IsCommandLineArguments(string(id)) { 1642 if invalidateMetadata, ok := idsToInvalidate[id]; invalidateMetadata && ok { 1643 continue 1644 } 1645 } 1646 1647 // If all the files we know about in a package have been deleted, 1648 // the package is gone and we should no longer try to load it. 1649 if m := s.metadata[id]; m != nil { 1650 hasFiles := false 1651 for _, uri := range s.metadata[id].goFiles { 1652 // For internal tests, we need _test files, not just the normal 1653 // ones. External tests only have _test files, but we can check 1654 // them anyway. 1655 if m.forTest != "" && !strings.HasSuffix(string(uri), "_test.go") { 1656 continue 1657 } 1658 if _, ok := result.files[uri]; ok { 1659 hasFiles = true 1660 break 1661 } 1662 } 1663 if !hasFiles { 1664 continue 1665 } 1666 } 1667 1668 // If the package name of a file in the package has changed, it's 1669 // possible that the package ID may no longer exist. Delete it from 1670 // the set of workspace packages, on the assumption that we will add it 1671 // back when the relevant files are reloaded. 1672 if _, ok := changedPkgNames[id]; ok { 1673 continue 1674 } 1675 1676 result.workspacePackages[id] = pkgPath 1677 } 1678 1679 // Inherit all of the go.mod-related handles. 1680 for _, v := range result.modTidyHandles { 1681 newGen.Inherit(v.handle) 1682 } 1683 for _, v := range result.modWhyHandles { 1684 newGen.Inherit(v.handle) 1685 } 1686 for _, v := range result.parseModHandles { 1687 newGen.Inherit(v.handle) 1688 } 1689 // Don't bother copying the importedBy graph, 1690 // as it changes each time we update metadata. 1691 1692 // If the snapshot's workspace mode has changed, the packages loaded using 1693 // the previous mode are no longer relevant, so clear them out. 1694 if workspaceModeChanged { 1695 result.workspacePackages = map[packageID]packagePath{} 1696 } 1697 1698 // The snapshot may need to be reinitialized. 1699 if workspaceReload || vendorChanged { 1700 if workspaceChanged || result.initializedErr != nil { 1701 result.initializeOnce = &sync.Once{} 1702 } 1703 } 1704 return result, workspaceChanged 1705} 1706 1707// guessPackageIDsForURI returns all packages related to uri. If we haven't 1708// seen this URI before, we guess based on files in the same directory. This 1709// is of course incorrect in build systems where packages are not organized by 1710// directory. 1711func guessPackageIDsForURI(uri span.URI, known map[span.URI][]packageID) []packageID { 1712 packages := known[uri] 1713 if len(packages) > 0 { 1714 // We've seen this file before. 1715 return packages 1716 } 1717 // This is a file we don't yet know about. Guess relevant packages by 1718 // considering files in the same directory. 1719 1720 // Cache of FileInfo to avoid unnecessary stats for multiple files in the 1721 // same directory. 1722 stats := make(map[string]struct { 1723 os.FileInfo 1724 error 1725 }) 1726 getInfo := func(dir string) (os.FileInfo, error) { 1727 if res, ok := stats[dir]; ok { 1728 return res.FileInfo, res.error 1729 } 1730 fi, err := os.Stat(dir) 1731 stats[dir] = struct { 1732 os.FileInfo 1733 error 1734 }{fi, err} 1735 return fi, err 1736 } 1737 dir := filepath.Dir(uri.Filename()) 1738 fi, err := getInfo(dir) 1739 if err != nil { 1740 return nil 1741 } 1742 1743 // Aggregate all possibly relevant package IDs. 1744 var found []packageID 1745 for knownURI, ids := range known { 1746 knownDir := filepath.Dir(knownURI.Filename()) 1747 knownFI, err := getInfo(knownDir) 1748 if err != nil { 1749 continue 1750 } 1751 if os.SameFile(fi, knownFI) { 1752 found = append(found, ids...) 1753 } 1754 } 1755 return found 1756} 1757 1758// fileWasSaved reports whether the FileHandle passed in has been saved. It 1759// accomplishes this by checking to see if the original and current FileHandles 1760// are both overlays, and if the current FileHandle is saved while the original 1761// FileHandle was not saved. 1762func fileWasSaved(originalFH, currentFH source.FileHandle) bool { 1763 c, ok := currentFH.(*overlay) 1764 if !ok || c == nil { 1765 return true 1766 } 1767 o, ok := originalFH.(*overlay) 1768 if !ok || o == nil { 1769 return c.saved 1770 } 1771 return !o.saved && c.saved 1772} 1773 1774// shouldInvalidateMetadata reparses a file's package and import declarations to 1775// determine if the file requires a metadata reload. 1776func (s *snapshot) shouldInvalidateMetadata(ctx context.Context, newSnapshot *snapshot, originalFH, currentFH source.FileHandle) (invalidate, pkgNameChanged bool) { 1777 if originalFH == nil { 1778 return true, false 1779 } 1780 // If the file hasn't changed, there's no need to reload. 1781 if originalFH.FileIdentity() == currentFH.FileIdentity() { 1782 return false, false 1783 } 1784 // Get the original and current parsed files in order to check package name 1785 // and imports. Use the new snapshot to parse to avoid modifying the 1786 // current snapshot. 1787 original, originalErr := newSnapshot.ParseGo(ctx, originalFH, source.ParseHeader) 1788 current, currentErr := newSnapshot.ParseGo(ctx, currentFH, source.ParseHeader) 1789 if originalErr != nil || currentErr != nil { 1790 return (originalErr == nil) != (currentErr == nil), false 1791 } 1792 // Check if the package's metadata has changed. The cases handled are: 1793 // 1. A package's name has changed 1794 // 2. A file's imports have changed 1795 if original.File.Name.Name != current.File.Name.Name { 1796 return true, true 1797 } 1798 importSet := make(map[string]struct{}) 1799 for _, importSpec := range original.File.Imports { 1800 importSet[importSpec.Path.Value] = struct{}{} 1801 } 1802 // If any of the current imports were not in the original imports. 1803 for _, importSpec := range current.File.Imports { 1804 if _, ok := importSet[importSpec.Path.Value]; ok { 1805 continue 1806 } 1807 // If the import path is obviously not valid, we can skip reloading 1808 // metadata. For now, valid means properly quoted and without a 1809 // terminal slash. 1810 path, err := strconv.Unquote(importSpec.Path.Value) 1811 if err != nil { 1812 continue 1813 } 1814 if path == "" { 1815 continue 1816 } 1817 if path[len(path)-1] == '/' { 1818 continue 1819 } 1820 return true, false 1821 } 1822 1823 // Re-evaluate build constraints and embed patterns. It would be preferable 1824 // to only do this on save, but we don't have the prior versions accessible. 1825 oldComments := extractMagicComments(original.File) 1826 newComments := extractMagicComments(current.File) 1827 if len(oldComments) != len(newComments) { 1828 return true, false 1829 } 1830 for i := range oldComments { 1831 if oldComments[i] != newComments[i] { 1832 return true, false 1833 } 1834 } 1835 1836 return false, false 1837} 1838 1839var buildConstraintOrEmbedRe = regexp.MustCompile(`^//(go:embed|go:build|\s*\+build).*`) 1840 1841// extractMagicComments finds magic comments that affect metadata in f. 1842func extractMagicComments(f *ast.File) []string { 1843 var results []string 1844 for _, cg := range f.Comments { 1845 for _, c := range cg.List { 1846 if buildConstraintOrEmbedRe.MatchString(c.Text) { 1847 results = append(results, c.Text) 1848 } 1849 } 1850 } 1851 return results 1852} 1853 1854func (s *snapshot) BuiltinFile(ctx context.Context) (*source.ParsedGoFile, error) { 1855 s.AwaitInitialized(ctx) 1856 1857 s.mu.Lock() 1858 builtin := s.builtin 1859 s.mu.Unlock() 1860 1861 if builtin == "" { 1862 return nil, errors.Errorf("no builtin package for view %s", s.view.name) 1863 } 1864 1865 fh, err := s.GetFile(ctx, builtin) 1866 if err != nil { 1867 return nil, err 1868 } 1869 return s.ParseGo(ctx, fh, source.ParseFull) 1870} 1871 1872func (s *snapshot) IsBuiltin(ctx context.Context, uri span.URI) bool { 1873 s.mu.Lock() 1874 defer s.mu.Unlock() 1875 // We should always get the builtin URI in a canonical form, so use simple 1876 // string comparison here. span.CompareURI is too expensive. 1877 return uri == s.builtin 1878} 1879 1880func (s *snapshot) setBuiltin(path string) { 1881 s.mu.Lock() 1882 defer s.mu.Unlock() 1883 1884 s.builtin = span.URIFromPath(path) 1885} 1886 1887// BuildGoplsMod generates a go.mod file for all modules in the workspace. It 1888// bypasses any existing gopls.mod. 1889func BuildGoplsMod(ctx context.Context, root span.URI, s source.Snapshot) (*modfile.File, error) { 1890 allModules, err := findModules(root, pathExcludedByFilterFunc(s.View().Options()), 0) 1891 if err != nil { 1892 return nil, err 1893 } 1894 return buildWorkspaceModFile(ctx, allModules, s) 1895} 1896 1897// TODO(rfindley): move this to workspacemodule.go 1898func buildWorkspaceModFile(ctx context.Context, modFiles map[span.URI]struct{}, fs source.FileSource) (*modfile.File, error) { 1899 file := &modfile.File{} 1900 file.AddModuleStmt("gopls-workspace") 1901 // Track the highest Go version, to be set on the workspace module. 1902 // Fall back to 1.12 -- old versions insist on having some version. 1903 goVersion := "1.12" 1904 1905 paths := map[string]span.URI{} 1906 excludes := map[string][]string{} 1907 var sortedModURIs []span.URI 1908 for uri := range modFiles { 1909 sortedModURIs = append(sortedModURIs, uri) 1910 } 1911 sort.Slice(sortedModURIs, func(i, j int) bool { 1912 return sortedModURIs[i] < sortedModURIs[j] 1913 }) 1914 for _, modURI := range sortedModURIs { 1915 fh, err := fs.GetFile(ctx, modURI) 1916 if err != nil { 1917 return nil, err 1918 } 1919 content, err := fh.Read() 1920 if err != nil { 1921 return nil, err 1922 } 1923 parsed, err := modfile.Parse(fh.URI().Filename(), content, nil) 1924 if err != nil { 1925 return nil, err 1926 } 1927 if file == nil || parsed.Module == nil { 1928 return nil, fmt.Errorf("no module declaration for %s", modURI) 1929 } 1930 if parsed.Go != nil && semver.Compare(goVersion, parsed.Go.Version) < 0 { 1931 goVersion = parsed.Go.Version 1932 } 1933 path := parsed.Module.Mod.Path 1934 if _, ok := paths[path]; ok { 1935 return nil, fmt.Errorf("found module %q twice in the workspace", path) 1936 } 1937 paths[path] = modURI 1938 // If the module's path includes a major version, we expect it to have 1939 // a matching major version. 1940 _, majorVersion, _ := module.SplitPathVersion(path) 1941 if majorVersion == "" { 1942 majorVersion = "/v0" 1943 } 1944 majorVersion = strings.TrimLeft(majorVersion, "/.") // handle gopkg.in versions 1945 file.AddNewRequire(path, source.WorkspaceModuleVersion(majorVersion), false) 1946 if err := file.AddReplace(path, "", dirURI(modURI).Filename(), ""); err != nil { 1947 return nil, err 1948 } 1949 for _, exclude := range parsed.Exclude { 1950 excludes[exclude.Mod.Path] = append(excludes[exclude.Mod.Path], exclude.Mod.Version) 1951 } 1952 } 1953 if goVersion != "" { 1954 file.AddGoStmt(goVersion) 1955 } 1956 // Go back through all of the modules to handle any of their replace 1957 // statements. 1958 for _, modURI := range sortedModURIs { 1959 fh, err := fs.GetFile(ctx, modURI) 1960 if err != nil { 1961 return nil, err 1962 } 1963 content, err := fh.Read() 1964 if err != nil { 1965 return nil, err 1966 } 1967 parsed, err := modfile.Parse(fh.URI().Filename(), content, nil) 1968 if err != nil { 1969 return nil, err 1970 } 1971 // If any of the workspace modules have replace directives, they need 1972 // to be reflected in the workspace module. 1973 for _, rep := range parsed.Replace { 1974 // Don't replace any modules that are in our workspace--we should 1975 // always use the version in the workspace. 1976 if _, ok := paths[rep.Old.Path]; ok { 1977 continue 1978 } 1979 newPath := rep.New.Path 1980 newVersion := rep.New.Version 1981 // If a replace points to a module in the workspace, make sure we 1982 // direct it to version of the module in the workspace. 1983 if m, ok := paths[rep.New.Path]; ok { 1984 newPath = dirURI(m).Filename() 1985 newVersion = "" 1986 } else if rep.New.Version == "" && !filepath.IsAbs(rep.New.Path) { 1987 // Make any relative paths absolute. 1988 newPath = filepath.Join(dirURI(modURI).Filename(), rep.New.Path) 1989 } 1990 if err := file.AddReplace(rep.Old.Path, rep.Old.Version, newPath, newVersion); err != nil { 1991 return nil, err 1992 } 1993 } 1994 } 1995 for path, versions := range excludes { 1996 for _, version := range versions { 1997 file.AddExclude(path, version) 1998 } 1999 } 2000 file.SortBlocks() 2001 return file, nil 2002} 2003 2004func buildWorkspaceSumFile(ctx context.Context, modFiles map[span.URI]struct{}, fs source.FileSource) ([]byte, error) { 2005 allSums := map[module.Version][]string{} 2006 for modURI := range modFiles { 2007 // TODO(rfindley): factor out this pattern into a uripath package. 2008 sumURI := span.URIFromPath(filepath.Join(filepath.Dir(modURI.Filename()), "go.sum")) 2009 fh, err := fs.GetFile(ctx, sumURI) 2010 if err != nil { 2011 continue 2012 } 2013 data, err := fh.Read() 2014 if os.IsNotExist(err) { 2015 continue 2016 } 2017 if err != nil { 2018 return nil, errors.Errorf("reading go sum: %w", err) 2019 } 2020 if err := readGoSum(allSums, sumURI.Filename(), data); err != nil { 2021 return nil, err 2022 } 2023 } 2024 // This logic to write go.sum is copied (with minor modifications) from 2025 // https://cs.opensource.google/go/go/+/master:src/cmd/go/internal/modfetch/fetch.go;l=631;drc=762eda346a9f4062feaa8a9fc0d17d72b11586f0 2026 var mods []module.Version 2027 for m := range allSums { 2028 mods = append(mods, m) 2029 } 2030 module.Sort(mods) 2031 2032 var buf bytes.Buffer 2033 for _, m := range mods { 2034 list := allSums[m] 2035 sort.Strings(list) 2036 // Note (rfindley): here we add all sum lines without verification, because 2037 // the assumption is that if they come from a go.sum file, they are 2038 // trusted. 2039 for _, h := range list { 2040 fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) 2041 } 2042 } 2043 return buf.Bytes(), nil 2044} 2045 2046// readGoSum is copied (with minor modifications) from 2047// https://cs.opensource.google/go/go/+/master:src/cmd/go/internal/modfetch/fetch.go;l=398;drc=762eda346a9f4062feaa8a9fc0d17d72b11586f0 2048func readGoSum(dst map[module.Version][]string, file string, data []byte) error { 2049 lineno := 0 2050 for len(data) > 0 { 2051 var line []byte 2052 lineno++ 2053 i := bytes.IndexByte(data, '\n') 2054 if i < 0 { 2055 line, data = data, nil 2056 } else { 2057 line, data = data[:i], data[i+1:] 2058 } 2059 f := strings.Fields(string(line)) 2060 if len(f) == 0 { 2061 // blank line; skip it 2062 continue 2063 } 2064 if len(f) != 3 { 2065 return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f)) 2066 } 2067 mod := module.Version{Path: f[0], Version: f[1]} 2068 dst[mod] = append(dst[mod], f[2]) 2069 } 2070 return nil 2071} 2072