1// Copyright 2020 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "context" 9 "fmt" 10 "os" 11 "path/filepath" 12 "sort" 13 "strings" 14 "sync" 15 16 "golang.org/x/mod/modfile" 17 "golang.org/x/tools/internal/event" 18 "golang.org/x/tools/internal/lsp/source" 19 workfile "golang.org/x/tools/internal/mod/modfile" 20 "golang.org/x/tools/internal/span" 21 "golang.org/x/tools/internal/xcontext" 22 errors "golang.org/x/xerrors" 23) 24 25type workspaceSource int 26 27const ( 28 legacyWorkspace = iota 29 goplsModWorkspace 30 goWorkWorkspace 31 fileSystemWorkspace 32) 33 34func (s workspaceSource) String() string { 35 switch s { 36 case legacyWorkspace: 37 return "legacy" 38 case goplsModWorkspace: 39 return "gopls.mod" 40 case goWorkWorkspace: 41 return "go.work" 42 case fileSystemWorkspace: 43 return "file system" 44 default: 45 return "!(unknown module source)" 46 } 47} 48 49// workspace tracks go.mod files in the workspace, along with the 50// gopls.mod file, to provide support for multi-module workspaces. 51// 52// Specifically, it provides: 53// - the set of modules contained within in the workspace root considered to 54// be 'active' 55// - the workspace modfile, to be used for the go command `-modfile` flag 56// - the set of workspace directories 57// 58// This type is immutable (or rather, idempotent), so that it may be shared 59// across multiple snapshots. 60type workspace struct { 61 root span.URI 62 excludePath func(string) bool 63 moduleSource workspaceSource 64 65 // activeModFiles holds the active go.mod files. 66 activeModFiles map[span.URI]struct{} 67 68 // knownModFiles holds the set of all go.mod files in the workspace. 69 // In all modes except for legacy, this is equivalent to modFiles. 70 knownModFiles map[span.URI]struct{} 71 72 // go111moduleOff indicates whether GO111MODULE=off has been configured in 73 // the environment. 74 go111moduleOff bool 75 76 // The workspace module is lazily re-built once after being invalidated. 77 // buildMu+built guards this reconstruction. 78 // 79 // file and wsDirs may be non-nil even if built == false, if they were copied 80 // from the previous workspace module version. In this case, they will be 81 // preserved if building fails. 82 buildMu sync.Mutex 83 built bool 84 buildErr error 85 mod *modfile.File 86 sum []byte 87 wsDirs map[span.URI]struct{} 88} 89 90func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, excludePath func(string) bool, go111moduleOff bool, experimental bool) (*workspace, error) { 91 // In experimental mode, the user may have a gopls.mod file that defines 92 // their workspace. 93 if experimental { 94 ws, err := parseExplicitWorkspaceFile(ctx, root, fs, excludePath) 95 if err == nil { 96 return ws, nil 97 } 98 } 99 // Otherwise, in all other modes, search for all of the go.mod files in the 100 // workspace. 101 knownModFiles, err := findModules(root, excludePath, 0) 102 if err != nil { 103 return nil, err 104 } 105 // When GO111MODULE=off, there are no active go.mod files. 106 if go111moduleOff { 107 return &workspace{ 108 root: root, 109 excludePath: excludePath, 110 moduleSource: legacyWorkspace, 111 knownModFiles: knownModFiles, 112 go111moduleOff: true, 113 }, nil 114 } 115 // In legacy mode, not all known go.mod files will be considered active. 116 if !experimental { 117 activeModFiles, err := getLegacyModules(ctx, root, fs) 118 if err != nil { 119 return nil, err 120 } 121 return &workspace{ 122 root: root, 123 excludePath: excludePath, 124 activeModFiles: activeModFiles, 125 knownModFiles: knownModFiles, 126 moduleSource: legacyWorkspace, 127 }, nil 128 } 129 return &workspace{ 130 root: root, 131 excludePath: excludePath, 132 activeModFiles: knownModFiles, 133 knownModFiles: knownModFiles, 134 moduleSource: fileSystemWorkspace, 135 }, nil 136} 137 138func parseExplicitWorkspaceFile(ctx context.Context, root span.URI, fs source.FileSource, excludePath func(string) bool) (*workspace, error) { 139 for _, src := range []workspaceSource{goWorkWorkspace, goplsModWorkspace} { 140 fh, err := fs.GetFile(ctx, uriForSource(root, src)) 141 if err != nil { 142 return nil, err 143 } 144 contents, err := fh.Read() 145 if err != nil { 146 continue 147 } 148 var file *modfile.File 149 var activeModFiles map[span.URI]struct{} 150 switch src { 151 case goWorkWorkspace: 152 file, activeModFiles, err = parseGoWork(ctx, root, fh.URI(), contents, fs) 153 case goplsModWorkspace: 154 file, activeModFiles, err = parseGoplsMod(root, fh.URI(), contents) 155 } 156 if err != nil { 157 return nil, err 158 } 159 return &workspace{ 160 root: root, 161 excludePath: excludePath, 162 activeModFiles: activeModFiles, 163 knownModFiles: activeModFiles, 164 mod: file, 165 moduleSource: src, 166 }, nil 167 } 168 return nil, noHardcodedWorkspace 169} 170 171var noHardcodedWorkspace = errors.New("no hardcoded workspace") 172 173func (w *workspace) getKnownModFiles() map[span.URI]struct{} { 174 return w.knownModFiles 175} 176 177func (w *workspace) getActiveModFiles() map[span.URI]struct{} { 178 return w.activeModFiles 179} 180 181// modFile gets the workspace modfile associated with this workspace, 182// computing it if it doesn't exist. 183// 184// A fileSource must be passed in to solve a chicken-egg problem: it is not 185// correct to pass in the snapshot file source to newWorkspace when 186// invalidating, because at the time these are called the snapshot is locked. 187// So we must pass it in later on when actually using the modFile. 188func (w *workspace) modFile(ctx context.Context, fs source.FileSource) (*modfile.File, error) { 189 w.build(ctx, fs) 190 return w.mod, w.buildErr 191} 192 193func (w *workspace) sumFile(ctx context.Context, fs source.FileSource) ([]byte, error) { 194 w.build(ctx, fs) 195 return w.sum, w.buildErr 196} 197 198func (w *workspace) build(ctx context.Context, fs source.FileSource) { 199 w.buildMu.Lock() 200 defer w.buildMu.Unlock() 201 202 if w.built { 203 return 204 } 205 // Building should never be cancelled. Since the workspace module is shared 206 // across multiple snapshots, doing so would put us in a bad state, and it 207 // would not be obvious to the user how to recover. 208 ctx = xcontext.Detach(ctx) 209 210 // If our module source is not gopls.mod, try to build the workspace module 211 // from modules. Fall back on the pre-existing mod file if parsing fails. 212 if w.moduleSource != goplsModWorkspace { 213 file, err := buildWorkspaceModFile(ctx, w.activeModFiles, fs) 214 switch { 215 case err == nil: 216 w.mod = file 217 case w.mod != nil: 218 // Parsing failed, but we have a previous file version. 219 event.Error(ctx, "building workspace mod file", err) 220 default: 221 // No file to fall back on. 222 w.buildErr = err 223 } 224 } 225 if w.mod != nil { 226 w.wsDirs = map[span.URI]struct{}{ 227 w.root: {}, 228 } 229 for _, r := range w.mod.Replace { 230 // We may be replacing a module with a different version, not a path 231 // on disk. 232 if r.New.Version != "" { 233 continue 234 } 235 w.wsDirs[span.URIFromPath(r.New.Path)] = struct{}{} 236 } 237 } 238 // Ensure that there is always at least the root dir. 239 if len(w.wsDirs) == 0 { 240 w.wsDirs = map[span.URI]struct{}{ 241 w.root: {}, 242 } 243 } 244 sum, err := buildWorkspaceSumFile(ctx, w.activeModFiles, fs) 245 if err == nil { 246 w.sum = sum 247 } else { 248 event.Error(ctx, "building workspace sum file", err) 249 } 250 w.built = true 251} 252 253// dirs returns the workspace directories for the loaded modules. 254func (w *workspace) dirs(ctx context.Context, fs source.FileSource) []span.URI { 255 w.build(ctx, fs) 256 var dirs []span.URI 257 for d := range w.wsDirs { 258 dirs = append(dirs, d) 259 } 260 sort.Slice(dirs, func(i, j int) bool { 261 return source.CompareURI(dirs[i], dirs[j]) < 0 262 }) 263 return dirs 264} 265 266// invalidate returns a (possibly) new workspace after invalidating the changed 267// files. If w is still valid in the presence of changedURIs, it returns itself 268// unmodified. 269// 270// The returned changed and reload flags control the level of invalidation. 271// Some workspace changes may affect workspace contents without requiring a 272// reload of metadata (for example, unsaved changes to a go.mod or go.sum 273// file). 274func (w *workspace) invalidate(ctx context.Context, changes map[span.URI]*fileChange, fs source.FileSource) (_ *workspace, changed, reload bool) { 275 // Prevent races to w.modFile or w.wsDirs below, if wmhas not yet been built. 276 w.buildMu.Lock() 277 defer w.buildMu.Unlock() 278 279 // Clone the workspace. This may be discarded if nothing changed. 280 result := &workspace{ 281 root: w.root, 282 moduleSource: w.moduleSource, 283 knownModFiles: make(map[span.URI]struct{}), 284 activeModFiles: make(map[span.URI]struct{}), 285 go111moduleOff: w.go111moduleOff, 286 mod: w.mod, 287 sum: w.sum, 288 wsDirs: w.wsDirs, 289 } 290 for k, v := range w.knownModFiles { 291 result.knownModFiles[k] = v 292 } 293 for k, v := range w.activeModFiles { 294 result.activeModFiles[k] = v 295 } 296 297 // First handle changes to the go.work or gopls.mod file. This must be 298 // considered before any changes to go.mod or go.sum files, as these files 299 // determine which modules we care about. In legacy workspace mode we don't 300 // consider the gopls.mod or go.work files. 301 if w.moduleSource != legacyWorkspace { 302 // If go.work/gopls.mod has changed we need to either re-read it if it 303 // exists or walk the filesystem if it has been deleted. 304 // go.work should override the gopls.mod if both exist. 305 for _, src := range []workspaceSource{goplsModWorkspace, goWorkWorkspace} { 306 uri := uriForSource(w.root, src) 307 // File opens/closes are just no-ops. 308 change, ok := changes[uri] 309 if !ok || change.isUnchanged { 310 continue 311 } 312 if change.exists { 313 // Only invalidate if the file if it actually parses. 314 // Otherwise, stick with the current file. 315 var parsedFile *modfile.File 316 var parsedModules map[span.URI]struct{} 317 var err error 318 switch src { 319 case goWorkWorkspace: 320 parsedFile, parsedModules, err = parseGoWork(ctx, w.root, uri, change.content, fs) 321 case goplsModWorkspace: 322 parsedFile, parsedModules, err = parseGoplsMod(w.root, uri, change.content) 323 } 324 if err == nil { 325 changed = true 326 reload = change.fileHandle.Saved() 327 result.mod = parsedFile 328 result.moduleSource = src 329 result.knownModFiles = parsedModules 330 result.activeModFiles = make(map[span.URI]struct{}) 331 for k, v := range parsedModules { 332 result.activeModFiles[k] = v 333 } 334 } else { 335 // An unparseable file should not invalidate the workspace: 336 // nothing good could come from changing the workspace in 337 // this case. 338 event.Error(ctx, fmt.Sprintf("parsing %s", filepath.Base(uri.Filename())), err) 339 } 340 } else { 341 // go.work/gopls.mod is deleted. search for modules again. 342 changed = true 343 reload = true 344 result.moduleSource = fileSystemWorkspace 345 // The parsed file is no longer valid. 346 result.mod = nil 347 knownModFiles, err := findModules(w.root, w.excludePath, 0) 348 if err != nil { 349 result.knownModFiles = nil 350 result.activeModFiles = nil 351 event.Error(ctx, "finding file system modules", err) 352 } else { 353 result.knownModFiles = knownModFiles 354 result.activeModFiles = make(map[span.URI]struct{}) 355 for k, v := range result.knownModFiles { 356 result.activeModFiles[k] = v 357 } 358 } 359 } 360 } 361 } 362 363 // Next, handle go.mod changes that could affect our workspace. If we're 364 // reading our tracked modules from the gopls.mod, there's nothing to do 365 // here. 366 if result.moduleSource != goplsModWorkspace && result.moduleSource != goWorkWorkspace { 367 for uri, change := range changes { 368 if change.isUnchanged || !isGoMod(uri) || !source.InDir(result.root.Filename(), uri.Filename()) { 369 continue 370 } 371 changed = true 372 active := result.moduleSource != legacyWorkspace || source.CompareURI(modURI(w.root), uri) == 0 373 reload = reload || (active && change.fileHandle.Saved()) 374 if change.exists { 375 result.knownModFiles[uri] = struct{}{} 376 if active { 377 result.activeModFiles[uri] = struct{}{} 378 } 379 } else { 380 delete(result.knownModFiles, uri) 381 delete(result.activeModFiles, uri) 382 } 383 } 384 } 385 386 // Finally, process go.sum changes for any modules that are now active. 387 for uri, change := range changes { 388 if !isGoSum(uri) { 389 continue 390 } 391 // TODO(rFindley) factor out this URI mangling. 392 dir := filepath.Dir(uri.Filename()) 393 modURI := span.URIFromPath(filepath.Join(dir, "go.mod")) 394 if _, active := result.activeModFiles[modURI]; !active { 395 continue 396 } 397 // Only changes to active go.sum files actually cause the workspace to 398 // change. 399 changed = true 400 reload = reload || change.fileHandle.Saved() 401 } 402 403 if !changed { 404 return w, false, false 405 } 406 407 return result, changed, reload 408} 409 410// goplsModURI returns the URI for the gopls.mod file contained in root. 411func uriForSource(root span.URI, src workspaceSource) span.URI { 412 var basename string 413 switch src { 414 case goplsModWorkspace: 415 basename = "gopls.mod" 416 case goWorkWorkspace: 417 basename = "go.work" 418 default: 419 return "" 420 } 421 return span.URIFromPath(filepath.Join(root.Filename(), basename)) 422} 423 424// modURI returns the URI for the go.mod file contained in root. 425func modURI(root span.URI) span.URI { 426 return span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) 427} 428 429// isGoMod reports if uri is a go.mod file. 430func isGoMod(uri span.URI) bool { 431 return filepath.Base(uri.Filename()) == "go.mod" 432} 433 434func isGoSum(uri span.URI) bool { 435 return filepath.Base(uri.Filename()) == "go.sum" 436} 437 438// fileExists reports if the file uri exists within source. 439func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) { 440 fh, err := source.GetFile(ctx, uri) 441 if err != nil { 442 return false, err 443 } 444 return fileHandleExists(fh) 445} 446 447// fileHandleExists reports if the file underlying fh actually exits. 448func fileHandleExists(fh source.FileHandle) (bool, error) { 449 _, err := fh.Read() 450 if err == nil { 451 return true, nil 452 } 453 if os.IsNotExist(err) { 454 return false, nil 455 } 456 return false, err 457} 458 459// TODO(rFindley): replace this (and similar) with a uripath package analogous 460// to filepath. 461func dirURI(uri span.URI) span.URI { 462 return span.URIFromPath(filepath.Dir(uri.Filename())) 463} 464 465// getLegacyModules returns a module set containing at most the root module. 466func getLegacyModules(ctx context.Context, root span.URI, fs source.FileSource) (map[span.URI]struct{}, error) { 467 uri := span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) 468 modules := make(map[span.URI]struct{}) 469 exists, err := fileExists(ctx, uri, fs) 470 if err != nil { 471 return nil, err 472 } 473 if exists { 474 modules[uri] = struct{}{} 475 } 476 return modules, nil 477} 478 479func parseGoWork(ctx context.Context, root, uri span.URI, contents []byte, fs source.FileSource) (*modfile.File, map[span.URI]struct{}, error) { 480 workFile, err := workfile.ParseWork(uri.Filename(), contents, nil) 481 if err != nil { 482 return nil, nil, errors.Errorf("parsing go.work: %w", err) 483 } 484 modFiles := make(map[span.URI]struct{}) 485 for _, dir := range workFile.Directory { 486 // The resulting modfile must use absolute paths, so that it can be 487 // written to a temp directory. 488 dir.DiskPath = absolutePath(root, dir.DiskPath) 489 modURI := span.URIFromPath(filepath.Join(dir.DiskPath, "go.mod")) 490 modFiles[modURI] = struct{}{} 491 } 492 modFile, err := buildWorkspaceModFile(ctx, modFiles, fs) 493 if err != nil { 494 return nil, nil, err 495 } 496 if workFile.Go.Version != "" { 497 if err := modFile.AddGoStmt(workFile.Go.Version); err != nil { 498 return nil, nil, err 499 } 500 } 501 502 return modFile, modFiles, nil 503} 504 505func parseGoplsMod(root, uri span.URI, contents []byte) (*modfile.File, map[span.URI]struct{}, error) { 506 modFile, err := modfile.Parse(uri.Filename(), contents, nil) 507 if err != nil { 508 return nil, nil, errors.Errorf("parsing gopls.mod: %w", err) 509 } 510 modFiles := make(map[span.URI]struct{}) 511 for _, replace := range modFile.Replace { 512 if replace.New.Version != "" { 513 return nil, nil, errors.Errorf("gopls.mod: replaced module %q@%q must not have version", replace.New.Path, replace.New.Version) 514 } 515 // The resulting modfile must use absolute paths, so that it can be 516 // written to a temp directory. 517 replace.New.Path = absolutePath(root, replace.New.Path) 518 modURI := span.URIFromPath(filepath.Join(replace.New.Path, "go.mod")) 519 modFiles[modURI] = struct{}{} 520 } 521 return modFile, modFiles, nil 522} 523 524func absolutePath(root span.URI, path string) string { 525 dirFP := filepath.FromSlash(path) 526 if !filepath.IsAbs(dirFP) { 527 dirFP = filepath.Join(root.Filename(), dirFP) 528 } 529 return dirFP 530} 531 532// errExhausted is returned by findModules if the file scan limit is reached. 533var errExhausted = errors.New("exhausted") 534 535// Limit go.mod search to 1 million files. As a point of reference, 536// Kubernetes has 22K files (as of 2020-11-24). 537const fileLimit = 1000000 538 539// findModules recursively walks the root directory looking for go.mod files, 540// returning the set of modules it discovers. If modLimit is non-zero, 541// searching stops once modLimit modules have been found. 542// 543// TODO(rfindley): consider overlays. 544func findModules(root span.URI, excludePath func(string) bool, modLimit int) (map[span.URI]struct{}, error) { 545 // Walk the view's folder to find all modules in the view. 546 modFiles := make(map[span.URI]struct{}) 547 searched := 0 548 errDone := errors.New("done") 549 err := filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { 550 if err != nil { 551 // Probably a permission error. Keep looking. 552 return filepath.SkipDir 553 } 554 // For any path that is not the workspace folder, check if the path 555 // would be ignored by the go command. Vendor directories also do not 556 // contain workspace modules. 557 if info.IsDir() && path != root.Filename() { 558 suffix := strings.TrimPrefix(path, root.Filename()) 559 switch { 560 case checkIgnored(suffix), 561 strings.Contains(filepath.ToSlash(suffix), "/vendor/"), 562 excludePath(suffix): 563 return filepath.SkipDir 564 } 565 } 566 // We're only interested in go.mod files. 567 uri := span.URIFromPath(path) 568 if isGoMod(uri) { 569 modFiles[uri] = struct{}{} 570 } 571 if modLimit > 0 && len(modFiles) >= modLimit { 572 return errDone 573 } 574 searched++ 575 if fileLimit > 0 && searched >= fileLimit { 576 return errExhausted 577 } 578 return nil 579 }) 580 if err == errDone { 581 return modFiles, nil 582 } 583 return modFiles, err 584} 585