1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "context" 9 "crypto/sha256" 10 "fmt" 11 "go/types" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "sort" 16 "strings" 17 "time" 18 19 "golang.org/x/tools/go/packages" 20 "golang.org/x/tools/internal/event" 21 "golang.org/x/tools/internal/gocommand" 22 "golang.org/x/tools/internal/lsp/debug/tag" 23 "golang.org/x/tools/internal/lsp/protocol" 24 "golang.org/x/tools/internal/lsp/source" 25 "golang.org/x/tools/internal/memoize" 26 "golang.org/x/tools/internal/packagesinternal" 27 "golang.org/x/tools/internal/span" 28 errors "golang.org/x/xerrors" 29) 30 31// metadata holds package metadata extracted from a call to packages.Load. 32type metadata struct { 33 id packageID 34 pkgPath packagePath 35 name packageName 36 goFiles []span.URI 37 compiledGoFiles []span.URI 38 forTest packagePath 39 typesSizes types.Sizes 40 errors []packages.Error 41 deps []packageID 42 missingDeps map[packagePath]struct{} 43 module *packages.Module 44 45 // config is the *packages.Config associated with the loaded package. 46 config *packages.Config 47} 48 49// load calls packages.Load for the given scopes, updating package metadata, 50// import graph, and mapped files with the result. 51func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...interface{}) error { 52 var query []string 53 var containsDir bool // for logging 54 for _, scope := range scopes { 55 switch scope := scope.(type) { 56 case packagePath: 57 if isCommandLineArguments(string(scope)) { 58 panic("attempted to load command-line-arguments") 59 } 60 // The only time we pass package paths is when we're doing a 61 // partial workspace load. In those cases, the paths came back from 62 // go list and should already be GOPATH-vendorized when appropriate. 63 query = append(query, string(scope)) 64 case fileURI: 65 uri := span.URI(scope) 66 // Don't try to load a file that doesn't exist. 67 fh := s.FindFile(uri) 68 if fh == nil || fh.Kind() != source.Go { 69 continue 70 } 71 query = append(query, fmt.Sprintf("file=%s", uri.Filename())) 72 case moduleLoadScope: 73 query = append(query, fmt.Sprintf("%s/...", scope)) 74 case viewLoadScope: 75 // If we are outside of GOPATH, a module, or some other known 76 // build system, don't load subdirectories. 77 if !s.ValidBuildConfiguration() { 78 query = append(query, "./") 79 } else { 80 query = append(query, "./...") 81 } 82 default: 83 panic(fmt.Sprintf("unknown scope type %T", scope)) 84 } 85 switch scope.(type) { 86 case viewLoadScope, moduleLoadScope: 87 containsDir = true 88 } 89 } 90 if len(query) == 0 { 91 return nil 92 } 93 sort.Strings(query) // for determinism 94 95 ctx, done := event.Start(ctx, "cache.view.load", tag.Query.Of(query)) 96 defer done() 97 98 flags := source.LoadWorkspace 99 if allowNetwork { 100 flags |= source.AllowNetwork 101 } 102 _, inv, cleanup, err := s.goCommandInvocation(ctx, flags, &gocommand.Invocation{ 103 WorkingDir: s.view.rootURI.Filename(), 104 }) 105 if err != nil { 106 return err 107 } 108 109 // Set a last resort deadline on packages.Load since it calls the go 110 // command, which may hang indefinitely if it has a bug. golang/go#42132 111 // and golang/go#42255 have more context. 112 ctx, cancel := context.WithTimeout(ctx, 15*time.Minute) 113 defer cancel() 114 115 cfg := s.config(ctx, inv) 116 pkgs, err := packages.Load(cfg, query...) 117 cleanup() 118 119 // If the context was canceled, return early. Otherwise, we might be 120 // type-checking an incomplete result. Check the context directly, 121 // because go/packages adds extra information to the error. 122 if ctx.Err() != nil { 123 return ctx.Err() 124 } 125 if err != nil { 126 event.Error(ctx, "go/packages.Load", err, tag.Snapshot.Of(s.ID()), tag.Directory.Of(cfg.Dir), tag.Query.Of(query), tag.PackageCount.Of(len(pkgs))) 127 } else { 128 event.Log(ctx, "go/packages.Load", tag.Snapshot.Of(s.ID()), tag.Directory.Of(cfg.Dir), tag.Query.Of(query), tag.PackageCount.Of(len(pkgs))) 129 } 130 if len(pkgs) == 0 { 131 if err == nil { 132 err = fmt.Errorf("no packages returned") 133 } 134 return errors.Errorf("%v: %w", err, source.PackagesLoadError) 135 } 136 for _, pkg := range pkgs { 137 if !containsDir || s.view.Options().VerboseOutput { 138 event.Log(ctx, "go/packages.Load", 139 tag.Snapshot.Of(s.ID()), 140 tag.Package.Of(pkg.ID), 141 tag.Files.Of(pkg.CompiledGoFiles)) 142 } 143 // Ignore packages with no sources, since we will never be able to 144 // correctly invalidate that metadata. 145 if len(pkg.GoFiles) == 0 && len(pkg.CompiledGoFiles) == 0 { 146 continue 147 } 148 // Special case for the builtin package, as it has no dependencies. 149 if pkg.PkgPath == "builtin" { 150 if err := s.buildBuiltinPackage(ctx, pkg.GoFiles); err != nil { 151 return err 152 } 153 continue 154 } 155 // Skip test main packages. 156 if isTestMain(pkg, s.view.gocache) { 157 continue 158 } 159 // Skip filtered packages. They may be added anyway if they're 160 // dependencies of non-filtered packages. 161 if s.view.allFilesExcluded(pkg) { 162 continue 163 } 164 // Set the metadata for this package. 165 m, err := s.setMetadata(ctx, packagePath(pkg.PkgPath), pkg, cfg, map[packageID]struct{}{}) 166 if err != nil { 167 return err 168 } 169 if _, err := s.buildPackageHandle(ctx, m.id, s.workspaceParseMode(m.id)); err != nil { 170 return err 171 } 172 } 173 // Rebuild the import graph when the metadata is updated. 174 s.clearAndRebuildImportGraph() 175 176 return nil 177} 178 179// workspaceLayoutErrors returns a diagnostic for every open file, as well as 180// an error message if there are no open files. 181func (s *snapshot) workspaceLayoutError(ctx context.Context) *source.CriticalError { 182 if len(s.workspace.getKnownModFiles()) == 0 { 183 return nil 184 } 185 if s.view.userGo111Module == off { 186 return nil 187 } 188 if s.workspace.moduleSource != legacyWorkspace { 189 return nil 190 } 191 // If the user has one module per view, there is nothing to warn about. 192 if s.ValidBuildConfiguration() && len(s.workspace.getKnownModFiles()) == 1 { 193 return nil 194 } 195 196 // Apply diagnostics about the workspace configuration to relevant open 197 // files. 198 openFiles := s.openFiles() 199 200 // If the snapshot does not have a valid build configuration, it may be 201 // that the user has opened a directory that contains multiple modules. 202 // Check for that an warn about it. 203 if !s.ValidBuildConfiguration() { 204 msg := `gopls requires a module at the root of your workspace. 205You can work with multiple modules by opening each one as a workspace folder. 206Improvements to this workflow will be coming soon (https://github.com/golang/go/issues/32394), 207and you can learn more here: https://github.com/golang/go/issues/36899.` 208 return &source.CriticalError{ 209 MainError: errors.Errorf(msg), 210 ErrorList: s.applyCriticalErrorToFiles(ctx, msg, openFiles), 211 } 212 } 213 214 // If the user has one active go.mod file, they may still be editing files 215 // in nested modules. Check the module of each open file and add warnings 216 // that the nested module must be opened as a workspace folder. 217 if len(s.workspace.getActiveModFiles()) == 1 { 218 // Get the active root go.mod file to compare against. 219 var rootModURI span.URI 220 for uri := range s.workspace.getActiveModFiles() { 221 rootModURI = uri 222 } 223 nestedModules := map[span.URI][]source.VersionedFileHandle{} 224 for _, fh := range openFiles { 225 modURI := moduleForURI(s.workspace.knownModFiles, fh.URI()) 226 if modURI != rootModURI { 227 nestedModules[modURI] = append(nestedModules[modURI], fh) 228 } 229 } 230 // Add a diagnostic to each file in a nested module to mark it as 231 // "orphaned". Don't show a general diagnostic in the progress bar, 232 // because the user may still want to edit a file in a nested module. 233 var srcErrs []*source.Error 234 for modURI, uris := range nestedModules { 235 msg := fmt.Sprintf(`This file is in %s, which is a nested module in the %s module. 236gopls currently requires one module per workspace folder. 237Please open %s as a separate workspace folder. 238You can learn more here: https://github.com/golang/go/issues/36899. 239`, modURI.Filename(), rootModURI.Filename(), modURI.Filename()) 240 srcErrs = append(srcErrs, s.applyCriticalErrorToFiles(ctx, msg, uris)...) 241 } 242 if len(srcErrs) != 0 { 243 return &source.CriticalError{ 244 MainError: errors.Errorf(`You are working in a nested module. Please open it as a separate workspace folder.`), 245 ErrorList: srcErrs, 246 } 247 } 248 } 249 return nil 250} 251 252func (s *snapshot) applyCriticalErrorToFiles(ctx context.Context, msg string, files []source.VersionedFileHandle) []*source.Error { 253 var srcErrs []*source.Error 254 for _, fh := range files { 255 // Place the diagnostics on the package or module declarations. 256 var rng protocol.Range 257 switch fh.Kind() { 258 case source.Go: 259 if pgf, err := s.ParseGo(ctx, fh, source.ParseHeader); err == nil { 260 pkgDecl := span.NewRange(s.FileSet(), pgf.File.Package, pgf.File.Name.End()) 261 if spn, err := pkgDecl.Span(); err == nil { 262 rng, _ = pgf.Mapper.Range(spn) 263 } 264 } 265 case source.Mod: 266 if pmf, err := s.ParseMod(ctx, fh); err == nil { 267 if pmf.File.Module != nil && pmf.File.Module.Syntax != nil { 268 rng, _ = rangeFromPositions(pmf.Mapper, pmf.File.Module.Syntax.Start, pmf.File.Module.Syntax.End) 269 } 270 } 271 } 272 srcErrs = append(srcErrs, &source.Error{ 273 URI: fh.URI(), 274 Range: rng, 275 Kind: source.ListError, 276 Message: msg, 277 }) 278 } 279 return srcErrs 280} 281 282type workspaceDirKey string 283 284type workspaceDirData struct { 285 dir string 286 err error 287} 288 289// getWorkspaceDir gets the URI for the workspace directory associated with 290// this snapshot. The workspace directory is a temp directory containing the 291// go.mod file computed from all active modules. 292func (s *snapshot) getWorkspaceDir(ctx context.Context) (span.URI, error) { 293 s.mu.Lock() 294 h := s.workspaceDirHandle 295 s.mu.Unlock() 296 if h != nil { 297 return getWorkspaceDir(ctx, h, s.generation) 298 } 299 file, err := s.workspace.modFile(ctx, s) 300 if err != nil { 301 return "", err 302 } 303 hash := sha256.New() 304 modContent, err := file.Format() 305 if err != nil { 306 return "", err 307 } 308 sumContent, err := s.workspace.sumFile(ctx, s) 309 if err != nil { 310 return "", err 311 } 312 hash.Write(modContent) 313 hash.Write(sumContent) 314 key := workspaceDirKey(hash.Sum(nil)) 315 s.mu.Lock() 316 h = s.generation.Bind(key, func(context.Context, memoize.Arg) interface{} { 317 tmpdir, err := ioutil.TempDir("", "gopls-workspace-mod") 318 if err != nil { 319 return &workspaceDirData{err: err} 320 } 321 322 for name, content := range map[string][]byte{ 323 "go.mod": modContent, 324 "go.sum": sumContent, 325 } { 326 filename := filepath.Join(tmpdir, name) 327 if err := ioutil.WriteFile(filename, content, 0644); err != nil { 328 os.RemoveAll(tmpdir) 329 return &workspaceDirData{err: err} 330 } 331 } 332 333 return &workspaceDirData{dir: tmpdir} 334 }, func(v interface{}) { 335 d := v.(*workspaceDirData) 336 if d.dir != "" { 337 if err := os.RemoveAll(d.dir); err != nil { 338 event.Error(context.Background(), "cleaning workspace dir", err) 339 } 340 } 341 }) 342 s.workspaceDirHandle = h 343 s.mu.Unlock() 344 return getWorkspaceDir(ctx, h, s.generation) 345} 346 347func getWorkspaceDir(ctx context.Context, h *memoize.Handle, g *memoize.Generation) (span.URI, error) { 348 v, err := h.Get(ctx, g, nil) 349 if err != nil { 350 return "", err 351 } 352 return span.URIFromPath(v.(*workspaceDirData).dir), nil 353} 354 355// setMetadata extracts metadata from pkg and records it in s. It 356// recurses through pkg.Imports to ensure that metadata exists for all 357// dependencies. 358func (s *snapshot) setMetadata(ctx context.Context, pkgPath packagePath, pkg *packages.Package, cfg *packages.Config, seen map[packageID]struct{}) (*metadata, error) { 359 id := packageID(pkg.ID) 360 if _, ok := seen[id]; ok { 361 return nil, errors.Errorf("import cycle detected: %q", id) 362 } 363 // Recreate the metadata rather than reusing it to avoid locking. 364 m := &metadata{ 365 id: id, 366 pkgPath: pkgPath, 367 name: packageName(pkg.Name), 368 forTest: packagePath(packagesinternal.GetForTest(pkg)), 369 typesSizes: pkg.TypesSizes, 370 errors: pkg.Errors, 371 config: cfg, 372 module: pkg.Module, 373 } 374 375 for _, filename := range pkg.CompiledGoFiles { 376 uri := span.URIFromPath(filename) 377 m.compiledGoFiles = append(m.compiledGoFiles, uri) 378 s.addID(uri, m.id) 379 } 380 for _, filename := range pkg.GoFiles { 381 uri := span.URIFromPath(filename) 382 m.goFiles = append(m.goFiles, uri) 383 s.addID(uri, m.id) 384 } 385 386 // TODO(rstambler): is this still necessary? 387 copied := map[packageID]struct{}{ 388 id: {}, 389 } 390 for k, v := range seen { 391 copied[k] = v 392 } 393 for importPath, importPkg := range pkg.Imports { 394 importPkgPath := packagePath(importPath) 395 importID := packageID(importPkg.ID) 396 397 m.deps = append(m.deps, importID) 398 399 // Don't remember any imports with significant errors. 400 if importPkgPath != "unsafe" && len(importPkg.CompiledGoFiles) == 0 { 401 if m.missingDeps == nil { 402 m.missingDeps = make(map[packagePath]struct{}) 403 } 404 m.missingDeps[importPkgPath] = struct{}{} 405 continue 406 } 407 if s.getMetadata(importID) == nil { 408 if _, err := s.setMetadata(ctx, importPkgPath, importPkg, cfg, copied); err != nil { 409 event.Error(ctx, "error in dependency", err) 410 } 411 } 412 } 413 414 // Add the metadata to the cache. 415 s.mu.Lock() 416 defer s.mu.Unlock() 417 418 // TODO: We should make sure not to set duplicate metadata, 419 // and instead panic here. This can be done by making sure not to 420 // reset metadata information for packages we've already seen. 421 if original, ok := s.metadata[m.id]; ok { 422 m = original 423 } else { 424 s.metadata[m.id] = m 425 } 426 427 // Set the workspace packages. If any of the package's files belong to the 428 // view, then the package may be a workspace package. 429 for _, uri := range append(m.compiledGoFiles, m.goFiles...) { 430 if !s.view.contains(uri) { 431 continue 432 } 433 434 // The package's files are in this view. It may be a workspace package. 435 if strings.Contains(string(uri), "/vendor/") { 436 // Vendored packages are not likely to be interesting to the user. 437 continue 438 } 439 440 switch { 441 case m.forTest == "": 442 // A normal package. 443 s.workspacePackages[m.id] = pkgPath 444 case m.forTest == m.pkgPath, m.forTest+"_test" == m.pkgPath: 445 // The test variant of some workspace package or its x_test. 446 // To load it, we need to load the non-test variant with -test. 447 s.workspacePackages[m.id] = m.forTest 448 default: 449 // A test variant of some intermediate package. We don't care about it. 450 } 451 } 452 return m, nil 453} 454 455func isTestMain(pkg *packages.Package, gocache string) bool { 456 // Test mains must have an import path that ends with ".test". 457 if !strings.HasSuffix(pkg.PkgPath, ".test") { 458 return false 459 } 460 // Test main packages are always named "main". 461 if pkg.Name != "main" { 462 return false 463 } 464 // Test mains always have exactly one GoFile that is in the build cache. 465 if len(pkg.GoFiles) > 1 { 466 return false 467 } 468 if !source.InDir(gocache, pkg.GoFiles[0]) { 469 return false 470 } 471 return true 472} 473