1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "context" 9 "fmt" 10 "io/ioutil" 11 "os" 12 "regexp" 13 "strconv" 14 "strings" 15 16 "golang.org/x/mod/modfile" 17 "golang.org/x/tools/go/packages" 18 "golang.org/x/tools/internal/gocommand" 19 "golang.org/x/tools/internal/lsp/protocol" 20 "golang.org/x/tools/internal/lsp/source" 21 "golang.org/x/tools/internal/lsp/telemetry" 22 "golang.org/x/tools/internal/memoize" 23 "golang.org/x/tools/internal/span" 24 "golang.org/x/tools/internal/telemetry/log" 25 "golang.org/x/tools/internal/telemetry/trace" 26 errors "golang.org/x/xerrors" 27) 28 29const ( 30 ModTidyError = "go mod tidy" 31 SyntaxError = "syntax" 32) 33 34type modKey struct { 35 cfg string 36 gomod string 37 view string 38} 39 40type modTidyKey struct { 41 cfg string 42 gomod string 43 imports string 44 view string 45} 46 47type modHandle struct { 48 handle *memoize.Handle 49 file source.FileHandle 50 cfg *packages.Config 51} 52 53type modData struct { 54 memoize.NoCopy 55 56 // origfh is the file handle for the original go.mod file. 57 origfh source.FileHandle 58 59 // origParsedFile contains the parsed contents that are used to diff with 60 // the ideal contents. 61 origParsedFile *modfile.File 62 63 // origMapper is the column mapper for the original go.mod file. 64 origMapper *protocol.ColumnMapper 65 66 // idealParsedFile contains the parsed contents for the go.mod file 67 // after it has been "tidied". 68 idealParsedFile *modfile.File 69 70 // unusedDeps is the map containing the dependencies that are left after 71 // removing the ones that are identical in the original and ideal go.mods. 72 unusedDeps map[string]*modfile.Require 73 74 // missingDeps is the map containing the dependencies that are left after 75 // removing the ones that are identical in the original and ideal go.mods. 76 missingDeps map[string]*modfile.Require 77 78 // upgrades is a map of path->version that contains any upgrades for the go.mod. 79 upgrades map[string]string 80 81 // why is a map of path->explanation that contains all the "go mod why" contents 82 // for each require statement. 83 why map[string]string 84 85 // parseErrors are the errors that arise when we diff between a user's go.mod 86 // and the "tidied" go.mod. 87 parseErrors []source.Error 88 89 // err is any error that occurs while we are calculating the parseErrors. 90 err error 91} 92 93func (mh *modHandle) String() string { 94 return mh.File().Identity().URI.Filename() 95} 96 97func (mh *modHandle) File() source.FileHandle { 98 return mh.file 99} 100 101func (mh *modHandle) Parse(ctx context.Context) (*modfile.File, *protocol.ColumnMapper, error) { 102 v := mh.handle.Get(ctx) 103 if v == nil { 104 return nil, nil, errors.Errorf("no parsed file for %s", mh.File().Identity().URI) 105 } 106 data := v.(*modData) 107 return data.origParsedFile, data.origMapper, data.err 108} 109 110func (mh *modHandle) Upgrades(ctx context.Context) (*modfile.File, *protocol.ColumnMapper, map[string]string, error) { 111 v := mh.handle.Get(ctx) 112 if v == nil { 113 return nil, nil, nil, errors.Errorf("no parsed file for %s", mh.File().Identity().URI) 114 } 115 data := v.(*modData) 116 return data.origParsedFile, data.origMapper, data.upgrades, data.err 117} 118 119func (mh *modHandle) Why(ctx context.Context) (*modfile.File, *protocol.ColumnMapper, map[string]string, error) { 120 v := mh.handle.Get(ctx) 121 if v == nil { 122 return nil, nil, nil, errors.Errorf("no parsed file for %s", mh.File().Identity().URI) 123 } 124 data := v.(*modData) 125 return data.origParsedFile, data.origMapper, data.why, data.err 126} 127 128func (s *snapshot) ModHandle(ctx context.Context, fh source.FileHandle) source.ModHandle { 129 uri := fh.Identity().URI 130 if handle := s.getModHandle(uri); handle != nil { 131 return handle 132 } 133 134 realURI, tempURI := s.view.ModFiles() 135 folder := s.View().Folder().Filename() 136 cfg := s.Config(ctx) 137 138 key := modKey{ 139 cfg: hashConfig(cfg), 140 gomod: fh.Identity().String(), 141 view: folder, 142 } 143 h := s.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} { 144 ctx, done := trace.StartSpan(ctx, "cache.ModHandle", telemetry.File.Of(uri)) 145 defer done() 146 147 contents, _, err := fh.Read(ctx) 148 if err != nil { 149 return &modData{ 150 err: err, 151 } 152 } 153 parsedFile, err := modfile.Parse(uri.Filename(), contents, nil) 154 if err != nil { 155 return &modData{ 156 err: err, 157 } 158 } 159 data := &modData{ 160 origfh: fh, 161 origParsedFile: parsedFile, 162 origMapper: &protocol.ColumnMapper{ 163 URI: uri, 164 Converter: span.NewContentConverter(uri.Filename(), contents), 165 Content: contents, 166 }, 167 } 168 // If the go.mod file is not the view's go.mod file, then we just want to parse. 169 if uri != realURI { 170 return data 171 } 172 173 // If we have a tempModfile, copy the real go.mod file content into the temp go.mod file. 174 if tempURI != "" { 175 if err := ioutil.WriteFile(tempURI.Filename(), contents, os.ModePerm); err != nil { 176 data.err = err 177 return data 178 } 179 } 180 // Only get dependency upgrades if the go.mod file is the same as the view's. 181 if err := dependencyUpgrades(ctx, cfg, folder, data); err != nil { 182 data.err = err 183 return data 184 } 185 // Only run "go mod why" if the go.mod file is the same as the view's. 186 if err := goModWhy(ctx, cfg, folder, data); err != nil { 187 data.err = err 188 return data 189 } 190 return data 191 }) 192 s.mu.Lock() 193 defer s.mu.Unlock() 194 s.modHandles[uri] = &modHandle{ 195 handle: h, 196 file: fh, 197 cfg: cfg, 198 } 199 return s.modHandles[uri] 200} 201 202func goModWhy(ctx context.Context, cfg *packages.Config, folder string, data *modData) error { 203 if len(data.origParsedFile.Require) == 0 { 204 return nil 205 } 206 // Run "go mod why" on all the dependencies to get information about the usages. 207 inv := gocommand.Invocation{ 208 Verb: "mod", 209 Args: []string{"why", "-m"}, 210 BuildFlags: cfg.BuildFlags, 211 Env: cfg.Env, 212 WorkingDir: folder, 213 } 214 for _, req := range data.origParsedFile.Require { 215 inv.Args = append(inv.Args, req.Mod.Path) 216 } 217 stdout, err := inv.Run(ctx) 218 if err != nil { 219 return err 220 } 221 whyList := strings.Split(stdout.String(), "\n\n") 222 if len(whyList) <= 1 || len(whyList) > len(data.origParsedFile.Require) { 223 return nil 224 } 225 data.why = make(map[string]string) 226 for i, req := range data.origParsedFile.Require { 227 data.why[req.Mod.Path] = whyList[i] 228 } 229 return nil 230} 231 232func dependencyUpgrades(ctx context.Context, cfg *packages.Config, folder string, data *modData) error { 233 if len(data.origParsedFile.Require) == 0 { 234 return nil 235 } 236 // Run "go list -u -m all" to be able to see which deps can be upgraded. 237 inv := gocommand.Invocation{ 238 Verb: "list", 239 Args: []string{"-u", "-m", "all"}, 240 BuildFlags: cfg.BuildFlags, 241 Env: cfg.Env, 242 WorkingDir: folder, 243 } 244 stdout, err := inv.Run(ctx) 245 if err != nil { 246 return err 247 } 248 upgradesList := strings.Split(stdout.String(), "\n") 249 if len(upgradesList) <= 1 { 250 return nil 251 } 252 data.upgrades = make(map[string]string) 253 for _, upgrade := range upgradesList[1:] { 254 // Example: "github.com/x/tools v1.1.0 [v1.2.0]" 255 info := strings.Split(upgrade, " ") 256 if len(info) < 3 { 257 continue 258 } 259 dep, version := info[0], info[2] 260 latest := version[1:] // remove the "[" 261 latest = strings.TrimSuffix(latest, "]") // remove the "]" 262 data.upgrades[dep] = latest 263 } 264 return nil 265} 266 267func (mh *modHandle) Tidy(ctx context.Context) (*modfile.File, *protocol.ColumnMapper, map[string]*modfile.Require, []source.Error, error) { 268 v := mh.handle.Get(ctx) 269 if v == nil { 270 return nil, nil, nil, nil, errors.Errorf("no parsed file for %s", mh.File().Identity().URI) 271 } 272 data := v.(*modData) 273 return data.origParsedFile, data.origMapper, data.missingDeps, data.parseErrors, data.err 274} 275 276func (s *snapshot) ModTidyHandle(ctx context.Context, realfh source.FileHandle) (source.ModTidyHandle, error) { 277 realURI, tempURI := s.view.ModFiles() 278 cfg := s.Config(ctx) 279 options := s.View().Options() 280 folder := s.View().Folder().Filename() 281 282 wsPackages, err := s.WorkspacePackages(ctx) 283 if ctx.Err() != nil { 284 return nil, ctx.Err() 285 } 286 if err != nil { 287 return nil, err 288 } 289 imports, err := hashImports(ctx, wsPackages) 290 if err != nil { 291 return nil, err 292 } 293 key := modTidyKey{ 294 view: folder, 295 imports: imports, 296 gomod: realfh.Identity().Identifier, 297 cfg: hashConfig(cfg), 298 } 299 h := s.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} { 300 data := &modData{} 301 302 // Check the case when the tempModfile flag is turned off. 303 if realURI == "" || tempURI == "" { 304 return data 305 } 306 307 ctx, done := trace.StartSpan(ctx, "cache.ModTidyHandle", telemetry.File.Of(realURI)) 308 defer done() 309 310 realContents, _, err := realfh.Read(ctx) 311 if err != nil { 312 data.err = err 313 return data 314 } 315 realMapper := &protocol.ColumnMapper{ 316 URI: realURI, 317 Converter: span.NewContentConverter(realURI.Filename(), realContents), 318 Content: realContents, 319 } 320 origParsedFile, err := modfile.Parse(realURI.Filename(), realContents, nil) 321 if err != nil { 322 if parseErr, err := extractModParseErrors(ctx, realURI, realMapper, err, realContents); err == nil { 323 data.parseErrors = []source.Error{parseErr} 324 return data 325 } 326 data.err = err 327 return data 328 } 329 330 // Copy the real go.mod file content into the temp go.mod file. 331 if err := ioutil.WriteFile(tempURI.Filename(), realContents, os.ModePerm); err != nil { 332 data.err = err 333 return data 334 } 335 336 // We want to run "go mod tidy" to be able to diff between the real and the temp files. 337 inv := gocommand.Invocation{ 338 Verb: "mod", 339 Args: []string{"tidy"}, 340 BuildFlags: cfg.BuildFlags, 341 Env: cfg.Env, 342 WorkingDir: folder, 343 } 344 if _, err := inv.Run(ctx); err != nil { 345 // Ignore concurrency errors here. 346 if !modConcurrencyError.MatchString(err.Error()) { 347 data.err = err 348 return data 349 } 350 } 351 352 // Go directly to disk to get the temporary mod file, since it is always on disk. 353 tempContents, err := ioutil.ReadFile(tempURI.Filename()) 354 if err != nil { 355 data.err = err 356 return data 357 } 358 idealParsedFile, err := modfile.Parse(tempURI.Filename(), tempContents, nil) 359 if err != nil { 360 // We do not need to worry about the temporary file's parse errors since it has been "tidied". 361 data.err = err 362 return data 363 } 364 365 data = &modData{ 366 origfh: realfh, 367 origParsedFile: origParsedFile, 368 origMapper: realMapper, 369 idealParsedFile: idealParsedFile, 370 unusedDeps: make(map[string]*modfile.Require, len(origParsedFile.Require)), 371 missingDeps: make(map[string]*modfile.Require, len(idealParsedFile.Require)), 372 } 373 // Get the dependencies that are different between the original and ideal mod files. 374 for _, req := range origParsedFile.Require { 375 data.unusedDeps[req.Mod.Path] = req 376 } 377 for _, req := range idealParsedFile.Require { 378 origDep := data.unusedDeps[req.Mod.Path] 379 if origDep != nil && origDep.Indirect == req.Indirect { 380 delete(data.unusedDeps, req.Mod.Path) 381 } else { 382 data.missingDeps[req.Mod.Path] = req 383 } 384 } 385 data.parseErrors, data.err = modRequireErrors(ctx, options, data) 386 387 for _, req := range data.missingDeps { 388 if data.unusedDeps[req.Mod.Path] != nil { 389 delete(data.missingDeps, req.Mod.Path) 390 } 391 } 392 return data 393 }) 394 return &modHandle{ 395 handle: h, 396 file: realfh, 397 cfg: cfg, 398 }, nil 399} 400 401// extractModParseErrors processes the raw errors returned by modfile.Parse, 402// extracting the filenames and line numbers that correspond to the errors. 403func extractModParseErrors(ctx context.Context, uri span.URI, m *protocol.ColumnMapper, parseErr error, content []byte) (source.Error, error) { 404 re := regexp.MustCompile(`.*:([\d]+): (.+)`) 405 matches := re.FindStringSubmatch(strings.TrimSpace(parseErr.Error())) 406 if len(matches) < 3 { 407 log.Error(ctx, "could not parse golang/x/mod error message", parseErr) 408 return source.Error{}, parseErr 409 } 410 line, err := strconv.Atoi(matches[1]) 411 if err != nil { 412 return source.Error{}, parseErr 413 } 414 lines := strings.Split(string(content), "\n") 415 if len(lines) <= line { 416 return source.Error{}, errors.Errorf("could not parse goland/x/mod error message, line number out of range") 417 } 418 // The error returned from the modfile package only returns a line number, 419 // so we assume that the diagnostic should be for the entire line. 420 endOfLine := len(lines[line-1]) 421 sOffset, err := m.Converter.ToOffset(line, 0) 422 if err != nil { 423 return source.Error{}, err 424 } 425 eOffset, err := m.Converter.ToOffset(line, endOfLine) 426 if err != nil { 427 return source.Error{}, err 428 } 429 spn := span.New(uri, span.NewPoint(line, 0, sOffset), span.NewPoint(line, endOfLine, eOffset)) 430 rng, err := m.Range(spn) 431 if err != nil { 432 return source.Error{}, err 433 } 434 return source.Error{ 435 Category: SyntaxError, 436 Message: matches[2], 437 Range: rng, 438 URI: uri, 439 }, nil 440} 441 442// modRequireErrors extracts the errors that occur on the require directives. 443// It checks for directness issues and unused dependencies. 444func modRequireErrors(ctx context.Context, options source.Options, data *modData) ([]source.Error, error) { 445 var errors []source.Error 446 for dep, req := range data.unusedDeps { 447 if req.Syntax == nil { 448 continue 449 } 450 // Handle dependencies that are incorrectly labeled indirect and vice versa. 451 if data.missingDeps[dep] != nil && req.Indirect != data.missingDeps[dep].Indirect { 452 directErr, err := modDirectnessErrors(ctx, options, data, req) 453 if err != nil { 454 return nil, err 455 } 456 errors = append(errors, directErr) 457 } 458 // Handle unused dependencies. 459 if data.missingDeps[dep] == nil { 460 rng, err := rangeFromPositions(data.origfh.Identity().URI, data.origMapper, req.Syntax.Start, req.Syntax.End) 461 if err != nil { 462 return nil, err 463 } 464 edits, err := dropDependencyEdits(ctx, options, data, req) 465 if err != nil { 466 return nil, err 467 } 468 errors = append(errors, source.Error{ 469 Category: ModTidyError, 470 Message: fmt.Sprintf("%s is not used in this module.", dep), 471 Range: rng, 472 URI: data.origfh.Identity().URI, 473 SuggestedFixes: []source.SuggestedFix{{ 474 Title: fmt.Sprintf("Remove dependency: %s", dep), 475 Edits: map[span.URI][]protocol.TextEdit{data.origfh.Identity().URI: edits}, 476 }}, 477 }) 478 } 479 } 480 return errors, nil 481} 482 483// modDirectnessErrors extracts errors when a dependency is labeled indirect when it should be direct and vice versa. 484func modDirectnessErrors(ctx context.Context, options source.Options, data *modData, req *modfile.Require) (source.Error, error) { 485 rng, err := rangeFromPositions(data.origfh.Identity().URI, data.origMapper, req.Syntax.Start, req.Syntax.End) 486 if err != nil { 487 return source.Error{}, err 488 } 489 if req.Indirect { 490 // If the dependency should be direct, just highlight the // indirect. 491 if comments := req.Syntax.Comment(); comments != nil && len(comments.Suffix) > 0 { 492 end := comments.Suffix[0].Start 493 end.LineRune += len(comments.Suffix[0].Token) 494 end.Byte += len([]byte(comments.Suffix[0].Token)) 495 rng, err = rangeFromPositions(data.origfh.Identity().URI, data.origMapper, comments.Suffix[0].Start, end) 496 if err != nil { 497 return source.Error{}, err 498 } 499 } 500 edits, err := changeDirectnessEdits(ctx, options, data, req, false) 501 if err != nil { 502 return source.Error{}, err 503 } 504 return source.Error{ 505 Category: ModTidyError, 506 Message: fmt.Sprintf("%s should be a direct dependency.", req.Mod.Path), 507 Range: rng, 508 URI: data.origfh.Identity().URI, 509 SuggestedFixes: []source.SuggestedFix{{ 510 Title: fmt.Sprintf("Make %s direct", req.Mod.Path), 511 Edits: map[span.URI][]protocol.TextEdit{data.origfh.Identity().URI: edits}, 512 }}, 513 }, nil 514 } 515 // If the dependency should be indirect, add the // indirect. 516 edits, err := changeDirectnessEdits(ctx, options, data, req, true) 517 if err != nil { 518 return source.Error{}, err 519 } 520 return source.Error{ 521 Category: ModTidyError, 522 Message: fmt.Sprintf("%s should be an indirect dependency.", req.Mod.Path), 523 Range: rng, 524 URI: data.origfh.Identity().URI, 525 SuggestedFixes: []source.SuggestedFix{{ 526 Title: fmt.Sprintf("Make %s indirect", req.Mod.Path), 527 Edits: map[span.URI][]protocol.TextEdit{data.origfh.Identity().URI: edits}, 528 }}, 529 }, nil 530} 531 532// dropDependencyEdits gets the edits needed to remove the dependency from the go.mod file. 533// As an example, this function will codify the edits needed to convert the before go.mod file to the after. 534// Before: 535// module t 536// 537// go 1.11 538// 539// require golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee 540// After: 541// module t 542// 543// go 1.11 544func dropDependencyEdits(ctx context.Context, options source.Options, data *modData, req *modfile.Require) ([]protocol.TextEdit, error) { 545 if err := data.origParsedFile.DropRequire(req.Mod.Path); err != nil { 546 return nil, err 547 } 548 data.origParsedFile.Cleanup() 549 newContents, err := data.origParsedFile.Format() 550 if err != nil { 551 return nil, err 552 } 553 // Reset the *modfile.File back to before we dropped the dependency. 554 data.origParsedFile.AddNewRequire(req.Mod.Path, req.Mod.Version, req.Indirect) 555 // Calculate the edits to be made due to the change. 556 diff := options.ComputeEdits(data.origfh.Identity().URI, string(data.origMapper.Content), string(newContents)) 557 edits, err := source.ToProtocolEdits(data.origMapper, diff) 558 if err != nil { 559 return nil, err 560 } 561 return edits, nil 562} 563 564// changeDirectnessEdits gets the edits needed to change an indirect dependency to direct and vice versa. 565// As an example, this function will codify the edits needed to convert the before go.mod file to the after. 566// Before: 567// module t 568// 569// go 1.11 570// 571// require golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee 572// After: 573// module t 574// 575// go 1.11 576// 577// require golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee // indirect 578func changeDirectnessEdits(ctx context.Context, options source.Options, data *modData, req *modfile.Require, indirect bool) ([]protocol.TextEdit, error) { 579 var newReq []*modfile.Require 580 prevIndirect := false 581 // Change the directness in the matching require statement. 582 for _, r := range data.origParsedFile.Require { 583 if req.Mod.Path == r.Mod.Path { 584 prevIndirect = req.Indirect 585 req.Indirect = indirect 586 } 587 newReq = append(newReq, r) 588 } 589 data.origParsedFile.SetRequire(newReq) 590 data.origParsedFile.Cleanup() 591 newContents, err := data.origParsedFile.Format() 592 if err != nil { 593 return nil, err 594 } 595 // Change the dependency back to the way it was before we got the newContents. 596 for _, r := range data.origParsedFile.Require { 597 if req.Mod.Path == r.Mod.Path { 598 req.Indirect = prevIndirect 599 } 600 newReq = append(newReq, r) 601 } 602 data.origParsedFile.SetRequire(newReq) 603 // Calculate the edits to be made due to the change. 604 diff := options.ComputeEdits(data.origfh.Identity().URI, string(data.origMapper.Content), string(newContents)) 605 edits, err := source.ToProtocolEdits(data.origMapper, diff) 606 if err != nil { 607 return nil, err 608 } 609 return edits, nil 610} 611 612func rangeFromPositions(uri span.URI, m *protocol.ColumnMapper, s, e modfile.Position) (protocol.Range, error) { 613 line, col, err := m.Converter.ToPosition(s.Byte) 614 if err != nil { 615 return protocol.Range{}, err 616 } 617 start := span.NewPoint(line, col, s.Byte) 618 619 line, col, err = m.Converter.ToPosition(e.Byte) 620 if err != nil { 621 return protocol.Range{}, err 622 } 623 end := span.NewPoint(line, col, e.Byte) 624 625 spn := span.New(uri, start, end) 626 rng, err := m.Range(spn) 627 if err != nil { 628 return protocol.Range{}, err 629 } 630 return rng, nil 631} 632