1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "context" 9 "encoding/json" 10 "fmt" 11 "io" 12 "os" 13 "path/filepath" 14 "regexp" 15 "strings" 16 "unicode" 17 18 "golang.org/x/mod/modfile" 19 "golang.org/x/mod/module" 20 "golang.org/x/tools/internal/event" 21 "golang.org/x/tools/internal/gocommand" 22 "golang.org/x/tools/internal/lsp/debug/tag" 23 "golang.org/x/tools/internal/lsp/protocol" 24 "golang.org/x/tools/internal/lsp/source" 25 "golang.org/x/tools/internal/memoize" 26 "golang.org/x/tools/internal/span" 27) 28 29const ( 30 SyntaxError = "syntax" 31 GoCommandError = "go command" 32) 33 34type parseModHandle struct { 35 handle *memoize.Handle 36} 37 38type parseModData struct { 39 parsed *source.ParsedModule 40 41 // err is any error encountered while parsing the file. 42 err error 43} 44 45func (mh *parseModHandle) parse(ctx context.Context, snapshot *snapshot) (*source.ParsedModule, error) { 46 v, err := mh.handle.Get(ctx, snapshot.generation, snapshot) 47 if err != nil { 48 return nil, err 49 } 50 data := v.(*parseModData) 51 return data.parsed, data.err 52} 53 54func (s *snapshot) ParseMod(ctx context.Context, modFH source.FileHandle) (*source.ParsedModule, error) { 55 if handle := s.getParseModHandle(modFH.URI()); handle != nil { 56 return handle.parse(ctx, s) 57 } 58 h := s.generation.Bind(modFH.FileIdentity(), func(ctx context.Context, _ memoize.Arg) interface{} { 59 _, done := event.Start(ctx, "cache.ParseModHandle", tag.URI.Of(modFH.URI())) 60 defer done() 61 62 contents, err := modFH.Read() 63 if err != nil { 64 return &parseModData{err: err} 65 } 66 m := &protocol.ColumnMapper{ 67 URI: modFH.URI(), 68 Converter: span.NewContentConverter(modFH.URI().Filename(), contents), 69 Content: contents, 70 } 71 file, err := modfile.Parse(modFH.URI().Filename(), contents, nil) 72 73 // Attempt to convert the error to a standardized parse error. 74 var parseErrors []*source.Error 75 if err != nil { 76 if parseErr, extractErr := extractErrorWithPosition(ctx, err.Error(), s); extractErr == nil { 77 parseErrors = []*source.Error{parseErr} 78 } 79 } 80 return &parseModData{ 81 parsed: &source.ParsedModule{ 82 URI: modFH.URI(), 83 Mapper: m, 84 File: file, 85 ParseErrors: parseErrors, 86 }, 87 err: err, 88 } 89 }, nil) 90 91 pmh := &parseModHandle{handle: h} 92 s.mu.Lock() 93 s.parseModHandles[modFH.URI()] = pmh 94 s.mu.Unlock() 95 96 return pmh.parse(ctx, s) 97} 98 99// goSum reads the go.sum file for the go.mod file at modURI, if it exists. If 100// it doesn't exist, it returns nil. 101func (s *snapshot) goSum(ctx context.Context, modURI span.URI) []byte { 102 // Get the go.sum file, either from the snapshot or directly from the 103 // cache. Avoid (*snapshot).GetFile here, as we don't want to add 104 // nonexistent file handles to the snapshot if the file does not exist. 105 sumURI := span.URIFromPath(sumFilename(modURI)) 106 var sumFH source.FileHandle = s.FindFile(sumURI) 107 if sumFH == nil { 108 var err error 109 sumFH, err = s.view.session.cache.getFile(ctx, sumURI) 110 if err != nil { 111 return nil 112 } 113 } 114 content, err := sumFH.Read() 115 if err != nil { 116 return nil 117 } 118 return content 119} 120 121func sumFilename(modURI span.URI) string { 122 return strings.TrimSuffix(modURI.Filename(), ".mod") + ".sum" 123} 124 125// modKey is uniquely identifies cached data for `go mod why` or dependencies 126// to upgrade. 127type modKey struct { 128 sessionID, env, view string 129 mod source.FileIdentity 130 verb modAction 131} 132 133type modAction int 134 135const ( 136 why modAction = iota 137 upgrade 138) 139 140type modWhyHandle struct { 141 handle *memoize.Handle 142} 143 144type modWhyData struct { 145 // why keeps track of the `go mod why` results for each require statement 146 // in the go.mod file. 147 why map[string]string 148 149 err error 150} 151 152func (mwh *modWhyHandle) why(ctx context.Context, snapshot *snapshot) (map[string]string, error) { 153 v, err := mwh.handle.Get(ctx, snapshot.generation, snapshot) 154 if err != nil { 155 return nil, err 156 } 157 data := v.(*modWhyData) 158 return data.why, data.err 159} 160 161func (s *snapshot) ModWhy(ctx context.Context, fh source.FileHandle) (map[string]string, error) { 162 if fh.Kind() != source.Mod { 163 return nil, fmt.Errorf("%s is not a go.mod file", fh.URI()) 164 } 165 if handle := s.getModWhyHandle(fh.URI()); handle != nil { 166 return handle.why(ctx, s) 167 } 168 key := modKey{ 169 sessionID: s.view.session.id, 170 env: hashEnv(s), 171 mod: fh.FileIdentity(), 172 view: s.view.rootURI.Filename(), 173 verb: why, 174 } 175 h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} { 176 ctx, done := event.Start(ctx, "cache.ModWhyHandle", tag.URI.Of(fh.URI())) 177 defer done() 178 179 snapshot := arg.(*snapshot) 180 181 pm, err := snapshot.ParseMod(ctx, fh) 182 if err != nil { 183 return &modWhyData{err: err} 184 } 185 // No requires to explain. 186 if len(pm.File.Require) == 0 { 187 return &modWhyData{} 188 } 189 // Run `go mod why` on all the dependencies. 190 inv := &gocommand.Invocation{ 191 Verb: "mod", 192 Args: []string{"why", "-m"}, 193 WorkingDir: filepath.Dir(fh.URI().Filename()), 194 } 195 for _, req := range pm.File.Require { 196 inv.Args = append(inv.Args, req.Mod.Path) 197 } 198 stdout, err := snapshot.RunGoCommandDirect(ctx, source.Normal, inv) 199 if err != nil { 200 return &modWhyData{err: err} 201 } 202 whyList := strings.Split(stdout.String(), "\n\n") 203 if len(whyList) != len(pm.File.Require) { 204 return &modWhyData{ 205 err: fmt.Errorf("mismatched number of results: got %v, want %v", len(whyList), len(pm.File.Require)), 206 } 207 } 208 why := make(map[string]string, len(pm.File.Require)) 209 for i, req := range pm.File.Require { 210 why[req.Mod.Path] = whyList[i] 211 } 212 return &modWhyData{why: why} 213 }, nil) 214 215 mwh := &modWhyHandle{handle: h} 216 s.mu.Lock() 217 s.modWhyHandles[fh.URI()] = mwh 218 s.mu.Unlock() 219 220 return mwh.why(ctx, s) 221} 222 223type modUpgradeHandle struct { 224 handle *memoize.Handle 225} 226 227type modUpgradeData struct { 228 // upgrades maps modules to their latest versions. 229 upgrades map[string]string 230 231 err error 232} 233 234func (muh *modUpgradeHandle) upgrades(ctx context.Context, snapshot *snapshot) (map[string]string, error) { 235 v, err := muh.handle.Get(ctx, snapshot.generation, snapshot) 236 if v == nil { 237 return nil, err 238 } 239 data := v.(*modUpgradeData) 240 return data.upgrades, data.err 241} 242 243// moduleUpgrade describes a module that can be upgraded to a particular 244// version. 245type moduleUpgrade struct { 246 Path string 247 Update struct { 248 Version string 249 } 250} 251 252func (s *snapshot) ModUpgrade(ctx context.Context, fh source.FileHandle) (map[string]string, error) { 253 if fh.Kind() != source.Mod { 254 return nil, fmt.Errorf("%s is not a go.mod file", fh.URI()) 255 } 256 if handle := s.getModUpgradeHandle(fh.URI()); handle != nil { 257 return handle.upgrades(ctx, s) 258 } 259 key := modKey{ 260 sessionID: s.view.session.id, 261 env: hashEnv(s), 262 mod: fh.FileIdentity(), 263 view: s.view.rootURI.Filename(), 264 verb: upgrade, 265 } 266 h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} { 267 ctx, done := event.Start(ctx, "cache.ModUpgradeHandle", tag.URI.Of(fh.URI())) 268 defer done() 269 270 snapshot := arg.(*snapshot) 271 272 pm, err := snapshot.ParseMod(ctx, fh) 273 if err != nil { 274 return &modUpgradeData{err: err} 275 } 276 277 // No requires to upgrade. 278 if len(pm.File.Require) == 0 { 279 return &modUpgradeData{} 280 } 281 // Run "go list -mod readonly -u -m all" to be able to see which deps can be 282 // upgraded without modifying mod file. 283 inv := &gocommand.Invocation{ 284 Verb: "list", 285 Args: []string{"-u", "-m", "-json", "all"}, 286 WorkingDir: filepath.Dir(fh.URI().Filename()), 287 } 288 if s.workspaceMode()&tempModfile == 0 || containsVendor(fh.URI()) { 289 // Use -mod=readonly if the module contains a vendor directory 290 // (see golang/go#38711). 291 inv.ModFlag = "readonly" 292 } 293 stdout, err := snapshot.RunGoCommandDirect(ctx, source.Normal|source.AllowNetwork, inv) 294 if err != nil { 295 return &modUpgradeData{err: err} 296 } 297 var upgradeList []moduleUpgrade 298 dec := json.NewDecoder(stdout) 299 for { 300 var m moduleUpgrade 301 if err := dec.Decode(&m); err == io.EOF { 302 break 303 } else if err != nil { 304 return &modUpgradeData{err: err} 305 } 306 upgradeList = append(upgradeList, m) 307 } 308 if len(upgradeList) <= 1 { 309 return &modUpgradeData{} 310 } 311 upgrades := make(map[string]string) 312 for _, upgrade := range upgradeList[1:] { 313 if upgrade.Update.Version == "" { 314 continue 315 } 316 upgrades[upgrade.Path] = upgrade.Update.Version 317 } 318 return &modUpgradeData{ 319 upgrades: upgrades, 320 } 321 }, nil) 322 muh := &modUpgradeHandle{handle: h} 323 s.mu.Lock() 324 s.modUpgradeHandles[fh.URI()] = muh 325 s.mu.Unlock() 326 327 return muh.upgrades(ctx, s) 328} 329 330// containsVendor reports whether the module has a vendor folder. 331func containsVendor(modURI span.URI) bool { 332 dir := filepath.Dir(modURI.Filename()) 333 f, err := os.Stat(filepath.Join(dir, "vendor")) 334 if err != nil { 335 return false 336 } 337 return f.IsDir() 338} 339 340var moduleAtVersionRe = regexp.MustCompile(`^(?P<module>.*)@(?P<version>.*)$`) 341 342// extractGoCommandError tries to parse errors that come from the go command 343// and shape them into go.mod diagnostics. 344func (s *snapshot) extractGoCommandError(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle, goCmdError string) *source.Error { 345 // If the error message contains a position, use that. Don't pass a file 346 // handle in, as it might not be the file associated with the error. 347 if srcErr, err := extractErrorWithPosition(ctx, goCmdError, s); err == nil { 348 return srcErr 349 } 350 // We try to match module versions in error messages. Some examples: 351 // 352 // example.com@v1.2.2: reading example.com/@v/v1.2.2.mod: no such file or directory 353 // go: github.com/cockroachdb/apd/v2@v2.0.72: reading github.com/cockroachdb/apd/go.mod at revision v2.0.72: unknown revision v2.0.72 354 // go: example.com@v1.2.3 requires\n\trandom.org@v1.2.3: parsing go.mod:\n\tmodule declares its path as: bob.org\n\tbut was required as: random.org 355 // 356 // We split on colons and whitespace, and attempt to match on something 357 // that matches module@version. If we're able to find a match, we try to 358 // find anything that matches it in the go.mod file. 359 var v module.Version 360 fields := strings.FieldsFunc(goCmdError, func(r rune) bool { 361 return unicode.IsSpace(r) || r == ':' 362 }) 363 for _, s := range fields { 364 s = strings.TrimSpace(s) 365 match := moduleAtVersionRe.FindStringSubmatch(s) 366 if match == nil || len(match) < 3 { 367 continue 368 } 369 path, version := match[1], match[2] 370 // Any module versions that come from the workspace module should not 371 // be shown to the user. 372 if source.IsWorkspaceModuleVersion(version) { 373 continue 374 } 375 if err := module.Check(path, version); err != nil { 376 continue 377 } 378 v.Path, v.Version = path, version 379 break 380 } 381 pm, err := snapshot.ParseMod(ctx, fh) 382 if err != nil { 383 return nil 384 } 385 toSourceError := func(line *modfile.Line) *source.Error { 386 rng, err := rangeFromPositions(pm.Mapper, line.Start, line.End) 387 if err != nil { 388 return nil 389 } 390 if v.Path != "" && strings.Contains(goCmdError, "disabled by GOPROXY=off") { 391 args, err := source.MarshalArgs(fh.URI(), false, []string{fmt.Sprintf("%v@%v", v.Path, v.Version)}) 392 if err != nil { 393 return nil 394 } 395 return &source.Error{ 396 Message: fmt.Sprintf("%v@%v has not been downloaded", v.Path, v.Version), 397 Kind: source.ListError, 398 Range: rng, 399 URI: fh.URI(), 400 SuggestedFixes: []source.SuggestedFix{{ 401 Title: fmt.Sprintf("Download %v@%v", v.Path, v.Version), 402 Command: &protocol.Command{ 403 Title: source.CommandAddDependency.Title, 404 Command: source.CommandAddDependency.ID(), 405 Arguments: args, 406 }, 407 }}, 408 } 409 } 410 return &source.Error{ 411 Message: goCmdError, 412 Range: rng, 413 URI: fh.URI(), 414 } 415 } 416 // Check if there are any require, exclude, or replace statements that 417 // match this module version. 418 for _, req := range pm.File.Require { 419 if req.Mod != v { 420 continue 421 } 422 return toSourceError(req.Syntax) 423 } 424 for _, ex := range pm.File.Exclude { 425 if ex.Mod != v { 426 continue 427 } 428 return toSourceError(ex.Syntax) 429 } 430 for _, rep := range pm.File.Replace { 431 if rep.New != v && rep.Old != v { 432 continue 433 } 434 return toSourceError(rep.Syntax) 435 } 436 // No match for the module path was found in the go.mod file. 437 // Show the error on the module declaration, if one exists. 438 if pm.File.Module == nil { 439 return nil 440 } 441 return toSourceError(pm.File.Module.Syntax) 442} 443 444// errorPositionRe matches errors messages of the form <filename>:<line>:<col>, 445// where the <col> is optional. 446var errorPositionRe = regexp.MustCompile(`(?P<pos>.*:([\d]+)(:([\d]+))?): (?P<msg>.+)`) 447 448// extractErrorWithPosition returns a structured error with position 449// information for the given unstructured error. If a file handle is provided, 450// the error position will be on that file. This is useful for parse errors, 451// where we already know the file with the error. 452func extractErrorWithPosition(ctx context.Context, goCmdError string, src source.FileSource) (*source.Error, error) { 453 matches := errorPositionRe.FindStringSubmatch(strings.TrimSpace(goCmdError)) 454 if len(matches) == 0 { 455 return nil, fmt.Errorf("error message doesn't contain a position") 456 } 457 var pos, msg string 458 for i, name := range errorPositionRe.SubexpNames() { 459 if name == "pos" { 460 pos = matches[i] 461 } 462 if name == "msg" { 463 msg = matches[i] 464 } 465 } 466 spn := span.Parse(pos) 467 fh, err := src.GetFile(ctx, spn.URI()) 468 if err != nil { 469 return nil, err 470 } 471 content, err := fh.Read() 472 if err != nil { 473 return nil, err 474 } 475 m := &protocol.ColumnMapper{ 476 URI: spn.URI(), 477 Converter: span.NewContentConverter(spn.URI().Filename(), content), 478 Content: content, 479 } 480 rng, err := m.Range(spn) 481 if err != nil { 482 return nil, err 483 } 484 category := GoCommandError 485 if fh != nil { 486 category = SyntaxError 487 } 488 return &source.Error{ 489 Category: category, 490 Message: msg, 491 Range: rng, 492 URI: spn.URI(), 493 }, nil 494} 495