1// Copyright 2020 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "context" 9 "os" 10 "path/filepath" 11 "sort" 12 "strings" 13 "sync" 14 15 "golang.org/x/mod/modfile" 16 "golang.org/x/tools/internal/event" 17 "golang.org/x/tools/internal/lsp/source" 18 "golang.org/x/tools/internal/span" 19 "golang.org/x/tools/internal/xcontext" 20 errors "golang.org/x/xerrors" 21) 22 23type workspaceSource int 24 25const ( 26 legacyWorkspace = iota 27 goplsModWorkspace 28 fileSystemWorkspace 29) 30 31func (s workspaceSource) String() string { 32 switch s { 33 case legacyWorkspace: 34 return "legacy" 35 case goplsModWorkspace: 36 return "gopls.mod" 37 case fileSystemWorkspace: 38 return "file system" 39 default: 40 return "!(unknown module source)" 41 } 42} 43 44// workspace tracks go.mod files in the workspace, along with the 45// gopls.mod file, to provide support for multi-module workspaces. 46// 47// Specifically, it provides: 48// - the set of modules contained within in the workspace root considered to 49// be 'active' 50// - the workspace modfile, to be used for the go command `-modfile` flag 51// - the set of workspace directories 52// 53// This type is immutable (or rather, idempotent), so that it may be shared 54// across multiple snapshots. 55type workspace struct { 56 root span.URI 57 moduleSource workspaceSource 58 59 // activeModFiles holds the active go.mod files. 60 activeModFiles map[span.URI]struct{} 61 62 // knownModFiles holds the set of all go.mod files in the workspace. 63 // In all modes except for legacy, this is equivalent to modFiles. 64 knownModFiles map[span.URI]struct{} 65 66 // go111moduleOff indicates whether GO111MODULE=off has been configured in 67 // the environment. 68 go111moduleOff bool 69 70 // The workspace module is lazily re-built once after being invalidated. 71 // buildMu+built guards this reconstruction. 72 // 73 // file and wsDirs may be non-nil even if built == false, if they were copied 74 // from the previous workspace module version. In this case, they will be 75 // preserved if building fails. 76 buildMu sync.Mutex 77 built bool 78 buildErr error 79 file *modfile.File 80 wsDirs map[span.URI]struct{} 81} 82 83func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, go111module string, experimental bool) (*workspace, error) { 84 // In experimental mode, the user may have a gopls.mod file that defines 85 // their workspace. 86 if experimental { 87 goplsModFH, err := fs.GetFile(ctx, goplsModURI(root)) 88 if err != nil { 89 return nil, err 90 } 91 contents, err := goplsModFH.Read() 92 if err == nil { 93 file, activeModFiles, err := parseGoplsMod(root, goplsModFH.URI(), contents) 94 if err != nil { 95 return nil, err 96 } 97 return &workspace{ 98 root: root, 99 activeModFiles: activeModFiles, 100 knownModFiles: activeModFiles, 101 file: file, 102 moduleSource: goplsModWorkspace, 103 }, nil 104 } 105 } 106 // Otherwise, in all other modes, search for all of the go.mod files in the 107 // workspace. 108 knownModFiles, err := findModules(ctx, root, 0) 109 if err != nil { 110 return nil, err 111 } 112 // When GO111MODULE=off, there are no active go.mod files. 113 if go111module == "off" { 114 return &workspace{ 115 root: root, 116 moduleSource: legacyWorkspace, 117 knownModFiles: knownModFiles, 118 go111moduleOff: true, 119 }, nil 120 } 121 // In legacy mode, not all known go.mod files will be considered active. 122 if !experimental { 123 activeModFiles, err := getLegacyModules(ctx, root, fs) 124 if err != nil { 125 return nil, err 126 } 127 return &workspace{ 128 root: root, 129 activeModFiles: activeModFiles, 130 knownModFiles: knownModFiles, 131 moduleSource: legacyWorkspace, 132 }, nil 133 } 134 return &workspace{ 135 root: root, 136 activeModFiles: knownModFiles, 137 knownModFiles: knownModFiles, 138 moduleSource: fileSystemWorkspace, 139 }, nil 140} 141 142func (wm *workspace) getKnownModFiles() map[span.URI]struct{} { 143 return wm.knownModFiles 144} 145 146func (wm *workspace) getActiveModFiles() map[span.URI]struct{} { 147 return wm.activeModFiles 148} 149 150// modFile gets the workspace modfile associated with this workspace, 151// computing it if it doesn't exist. 152// 153// A fileSource must be passed in to solve a chicken-egg problem: it is not 154// correct to pass in the snapshot file source to newWorkspace when 155// invalidating, because at the time these are called the snapshot is locked. 156// So we must pass it in later on when actually using the modFile. 157func (wm *workspace) modFile(ctx context.Context, fs source.FileSource) (*modfile.File, error) { 158 wm.build(ctx, fs) 159 return wm.file, wm.buildErr 160} 161 162func (wm *workspace) build(ctx context.Context, fs source.FileSource) { 163 wm.buildMu.Lock() 164 defer wm.buildMu.Unlock() 165 166 if wm.built { 167 return 168 } 169 // Building should never be cancelled. Since the workspace module is shared 170 // across multiple snapshots, doing so would put us in a bad state, and it 171 // would not be obvious to the user how to recover. 172 ctx = xcontext.Detach(ctx) 173 174 // If our module source is not gopls.mod, try to build the workspace module 175 // from modules. Fall back on the pre-existing mod file if parsing fails. 176 if wm.moduleSource != goplsModWorkspace { 177 file, err := buildWorkspaceModFile(ctx, wm.activeModFiles, fs) 178 switch { 179 case err == nil: 180 wm.file = file 181 case wm.file != nil: 182 // Parsing failed, but we have a previous file version. 183 event.Error(ctx, "building workspace mod file", err) 184 default: 185 // No file to fall back on. 186 wm.buildErr = err 187 } 188 } 189 if wm.file != nil { 190 wm.wsDirs = map[span.URI]struct{}{ 191 wm.root: {}, 192 } 193 for _, r := range wm.file.Replace { 194 // We may be replacing a module with a different version, not a path 195 // on disk. 196 if r.New.Version != "" { 197 continue 198 } 199 wm.wsDirs[span.URIFromPath(r.New.Path)] = struct{}{} 200 } 201 } 202 // Ensure that there is always at least the root dir. 203 if len(wm.wsDirs) == 0 { 204 wm.wsDirs = map[span.URI]struct{}{ 205 wm.root: {}, 206 } 207 } 208 wm.built = true 209} 210 211// dirs returns the workspace directories for the loaded modules. 212func (wm *workspace) dirs(ctx context.Context, fs source.FileSource) []span.URI { 213 wm.build(ctx, fs) 214 var dirs []span.URI 215 for d := range wm.wsDirs { 216 dirs = append(dirs, d) 217 } 218 sort.Slice(dirs, func(i, j int) bool { 219 return source.CompareURI(dirs[i], dirs[j]) < 0 220 }) 221 return dirs 222} 223 224// invalidate returns a (possibly) new workspaceModule after invalidating 225// changedURIs. If wm is still valid in the presence of changedURIs, it returns 226// itself unmodified. 227func (wm *workspace) invalidate(ctx context.Context, changes map[span.URI]*fileChange) (*workspace, bool) { 228 // Prevent races to wm.modFile or wm.wsDirs below, if wm has not yet been 229 // built. 230 wm.buildMu.Lock() 231 defer wm.buildMu.Unlock() 232 233 // Any gopls.mod change is processed first, followed by go.mod changes, as 234 // changes to gopls.mod may affect the set of active go.mod files. 235 var ( 236 // New values. We return a new workspace module if and only if 237 // knownModFiles is non-nil. 238 knownModFiles map[span.URI]struct{} 239 moduleSource = wm.moduleSource 240 modFile = wm.file 241 err error 242 ) 243 if wm.moduleSource == goplsModWorkspace { 244 // If we are currently reading the modfile from gopls.mod, we default to 245 // preserving it even if module metadata changes (which may be the case if 246 // a go.sum file changes). 247 modFile = wm.file 248 } 249 // First handle changes to the gopls.mod file. 250 if wm.moduleSource != legacyWorkspace { 251 // If gopls.mod has changed we need to either re-read it if it exists or 252 // walk the filesystem if it doesn't exist. 253 gmURI := goplsModURI(wm.root) 254 if change, ok := changes[gmURI]; ok { 255 if change.exists { 256 // Only invalidate if the gopls.mod actually parses. Otherwise, stick with the current gopls.mod 257 parsedFile, parsedModules, err := parseGoplsMod(wm.root, gmURI, change.content) 258 if err == nil { 259 modFile = parsedFile 260 moduleSource = goplsModWorkspace 261 knownModFiles = parsedModules 262 } else { 263 // Note that modFile is not invalidated here. 264 event.Error(ctx, "parsing gopls.mod", err) 265 } 266 } else { 267 // gopls.mod is deleted. search for modules again. 268 moduleSource = fileSystemWorkspace 269 knownModFiles, err = findModules(ctx, wm.root, 0) 270 // the modFile is no longer valid. 271 if err != nil { 272 event.Error(ctx, "finding file system modules", err) 273 } 274 modFile = nil 275 } 276 } 277 } 278 279 // Next, handle go.mod changes that could affect our set of tracked modules. 280 // If we're reading our tracked modules from the gopls.mod, there's nothing 281 // to do here. 282 if wm.moduleSource != goplsModWorkspace { 283 for uri, change := range changes { 284 // If a go.mod file has changed, we may need to update the set of active 285 // modules. 286 if !isGoMod(uri) { 287 continue 288 } 289 if !source.InDir(wm.root.Filename(), uri.Filename()) { 290 // Otherwise, the module must be contained within the workspace root. 291 continue 292 } 293 if knownModFiles == nil { 294 knownModFiles = make(map[span.URI]struct{}) 295 for k := range wm.knownModFiles { 296 knownModFiles[k] = struct{}{} 297 } 298 } 299 if change.exists { 300 knownModFiles[uri] = struct{}{} 301 } else { 302 delete(knownModFiles, uri) 303 } 304 } 305 } 306 if knownModFiles != nil { 307 var activeModFiles map[span.URI]struct{} 308 if wm.go111moduleOff { 309 // If GO111MODULE=off, the set of active go.mod files is unchanged. 310 activeModFiles = wm.activeModFiles 311 } else { 312 activeModFiles = make(map[span.URI]struct{}) 313 for uri := range knownModFiles { 314 // Legacy mode only considers a module a workspace root, so don't 315 // update the active go.mod files map. 316 if wm.moduleSource == legacyWorkspace && source.CompareURI(modURI(wm.root), uri) != 0 { 317 continue 318 } 319 activeModFiles[uri] = struct{}{} 320 } 321 } 322 // Any change to modules triggers a new version. 323 return &workspace{ 324 root: wm.root, 325 moduleSource: moduleSource, 326 activeModFiles: activeModFiles, 327 knownModFiles: knownModFiles, 328 file: modFile, 329 wsDirs: wm.wsDirs, 330 }, true 331 } 332 // No change. Just return wm, since it is immutable. 333 return wm, false 334} 335 336// goplsModURI returns the URI for the gopls.mod file contained in root. 337func goplsModURI(root span.URI) span.URI { 338 return span.URIFromPath(filepath.Join(root.Filename(), "gopls.mod")) 339} 340 341// modURI returns the URI for the go.mod file contained in root. 342func modURI(root span.URI) span.URI { 343 return span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) 344} 345 346// isGoMod reports if uri is a go.mod file. 347func isGoMod(uri span.URI) bool { 348 return filepath.Base(uri.Filename()) == "go.mod" 349} 350 351// fileExists reports if the file uri exists within source. 352func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) { 353 fh, err := source.GetFile(ctx, uri) 354 if err != nil { 355 return false, err 356 } 357 return fileHandleExists(fh) 358} 359 360// fileHandleExists reports if the file underlying fh actually exits. 361func fileHandleExists(fh source.FileHandle) (bool, error) { 362 _, err := fh.Read() 363 if err == nil { 364 return true, nil 365 } 366 if os.IsNotExist(err) { 367 return false, nil 368 } 369 return false, err 370} 371 372// TODO(rFindley): replace this (and similar) with a uripath package analogous 373// to filepath. 374func dirURI(uri span.URI) span.URI { 375 return span.URIFromPath(filepath.Dir(uri.Filename())) 376} 377 378// getLegacyModules returns a module set containing at most the root module. 379func getLegacyModules(ctx context.Context, root span.URI, fs source.FileSource) (map[span.URI]struct{}, error) { 380 uri := span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) 381 modules := make(map[span.URI]struct{}) 382 exists, err := fileExists(ctx, uri, fs) 383 if err != nil { 384 return nil, err 385 } 386 if exists { 387 modules[uri] = struct{}{} 388 } 389 return modules, nil 390} 391 392func parseGoplsMod(root, uri span.URI, contents []byte) (*modfile.File, map[span.URI]struct{}, error) { 393 modFile, err := modfile.Parse(uri.Filename(), contents, nil) 394 if err != nil { 395 return nil, nil, errors.Errorf("parsing gopls.mod: %w", err) 396 } 397 modFiles := make(map[span.URI]struct{}) 398 for _, replace := range modFile.Replace { 399 if replace.New.Version != "" { 400 return nil, nil, errors.Errorf("gopls.mod: replaced module %q@%q must not have version", replace.New.Path, replace.New.Version) 401 } 402 dirFP := filepath.FromSlash(replace.New.Path) 403 if !filepath.IsAbs(dirFP) { 404 dirFP = filepath.Join(root.Filename(), dirFP) 405 // The resulting modfile must use absolute paths, so that it can be 406 // written to a temp directory. 407 replace.New.Path = dirFP 408 } 409 modURI := span.URIFromPath(filepath.Join(dirFP, "go.mod")) 410 modFiles[modURI] = struct{}{} 411 } 412 return modFile, modFiles, nil 413} 414 415// errExhausted is returned by findModules if the file scan limit is reached. 416var errExhausted = errors.New("exhausted") 417 418// Limit go.mod search to 1 million files. As a point of reference, 419// Kubernetes has 22K files (as of 2020-11-24). 420const fileLimit = 1000000 421 422// findModules recursively walks the root directory looking for go.mod files, 423// returning the set of modules it discovers. If modLimit is non-zero, 424// searching stops once modLimit modules have been found. 425// 426// TODO(rfindley): consider overlays. 427func findModules(ctx context.Context, root span.URI, modLimit int) (map[span.URI]struct{}, error) { 428 // Walk the view's folder to find all modules in the view. 429 modFiles := make(map[span.URI]struct{}) 430 searched := 0 431 errDone := errors.New("done") 432 err := filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { 433 if err != nil { 434 // Probably a permission error. Keep looking. 435 return filepath.SkipDir 436 } 437 // For any path that is not the workspace folder, check if the path 438 // would be ignored by the go command. Vendor directories also do not 439 // contain workspace modules. 440 if info.IsDir() && path != root.Filename() { 441 suffix := strings.TrimPrefix(path, root.Filename()) 442 switch { 443 case checkIgnored(suffix), 444 strings.Contains(filepath.ToSlash(suffix), "/vendor/"): 445 return filepath.SkipDir 446 } 447 } 448 // We're only interested in go.mod files. 449 uri := span.URIFromPath(path) 450 if isGoMod(uri) { 451 modFiles[uri] = struct{}{} 452 } 453 if modLimit > 0 && len(modFiles) >= modLimit { 454 return errDone 455 } 456 searched++ 457 if fileLimit > 0 && searched >= fileLimit { 458 return errExhausted 459 } 460 return nil 461 }) 462 if err == errDone { 463 return modFiles, nil 464 } 465 return modFiles, err 466} 467