1// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package cache
6
7import (
8	"context"
9	"crypto/sha256"
10	"fmt"
11	"go/types"
12	"io/ioutil"
13	"os"
14	"path/filepath"
15	"sort"
16	"strings"
17	"time"
18
19	"golang.org/x/tools/go/packages"
20	"golang.org/x/tools/internal/event"
21	"golang.org/x/tools/internal/gocommand"
22	"golang.org/x/tools/internal/lsp/debug/tag"
23	"golang.org/x/tools/internal/lsp/protocol"
24	"golang.org/x/tools/internal/lsp/source"
25	"golang.org/x/tools/internal/memoize"
26	"golang.org/x/tools/internal/packagesinternal"
27	"golang.org/x/tools/internal/span"
28	errors "golang.org/x/xerrors"
29)
30
31// metadata holds package metadata extracted from a call to packages.Load.
32type metadata struct {
33	id              packageID
34	pkgPath         packagePath
35	name            packageName
36	goFiles         []span.URI
37	compiledGoFiles []span.URI
38	forTest         packagePath
39	typesSizes      types.Sizes
40	errors          []packages.Error
41	deps            []packageID
42	missingDeps     map[packagePath]struct{}
43	module          *packages.Module
44
45	// config is the *packages.Config associated with the loaded package.
46	config *packages.Config
47}
48
49// load calls packages.Load for the given scopes, updating package metadata,
50// import graph, and mapped files with the result.
51func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...interface{}) error {
52	var query []string
53	var containsDir bool // for logging
54	for _, scope := range scopes {
55		switch scope := scope.(type) {
56		case packagePath:
57			if isCommandLineArguments(string(scope)) {
58				panic("attempted to load command-line-arguments")
59			}
60			// The only time we pass package paths is when we're doing a
61			// partial workspace load. In those cases, the paths came back from
62			// go list and should already be GOPATH-vendorized when appropriate.
63			query = append(query, string(scope))
64		case fileURI:
65			uri := span.URI(scope)
66			// Don't try to load a file that doesn't exist.
67			fh := s.FindFile(uri)
68			if fh == nil || fh.Kind() != source.Go {
69				continue
70			}
71			query = append(query, fmt.Sprintf("file=%s", uri.Filename()))
72		case moduleLoadScope:
73			query = append(query, fmt.Sprintf("%s/...", scope))
74		case viewLoadScope:
75			// If we are outside of GOPATH, a module, or some other known
76			// build system, don't load subdirectories.
77			if !s.ValidBuildConfiguration() {
78				query = append(query, "./")
79			} else {
80				query = append(query, "./...")
81			}
82		default:
83			panic(fmt.Sprintf("unknown scope type %T", scope))
84		}
85		switch scope.(type) {
86		case viewLoadScope, moduleLoadScope:
87			containsDir = true
88		}
89	}
90	if len(query) == 0 {
91		return nil
92	}
93	sort.Strings(query) // for determinism
94
95	ctx, done := event.Start(ctx, "cache.view.load", tag.Query.Of(query))
96	defer done()
97
98	flags := source.LoadWorkspace
99	if allowNetwork {
100		flags |= source.AllowNetwork
101	}
102	_, inv, cleanup, err := s.goCommandInvocation(ctx, flags, &gocommand.Invocation{
103		WorkingDir: s.view.rootURI.Filename(),
104	})
105	if err != nil {
106		return err
107	}
108
109	// Set a last resort deadline on packages.Load since it calls the go
110	// command, which may hang indefinitely if it has a bug. golang/go#42132
111	// and golang/go#42255 have more context.
112	ctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
113	defer cancel()
114
115	cfg := s.config(ctx, inv)
116	pkgs, err := packages.Load(cfg, query...)
117	cleanup()
118
119	// If the context was canceled, return early. Otherwise, we might be
120	// type-checking an incomplete result. Check the context directly,
121	// because go/packages adds extra information to the error.
122	if ctx.Err() != nil {
123		return ctx.Err()
124	}
125	if err != nil {
126		event.Error(ctx, "go/packages.Load", err, tag.Snapshot.Of(s.ID()), tag.Directory.Of(cfg.Dir), tag.Query.Of(query), tag.PackageCount.Of(len(pkgs)))
127	} else {
128		event.Log(ctx, "go/packages.Load", tag.Snapshot.Of(s.ID()), tag.Directory.Of(cfg.Dir), tag.Query.Of(query), tag.PackageCount.Of(len(pkgs)))
129	}
130	if len(pkgs) == 0 {
131		if err == nil {
132			err = fmt.Errorf("no packages returned")
133		}
134		return errors.Errorf("%v: %w", err, source.PackagesLoadError)
135	}
136	for _, pkg := range pkgs {
137		if !containsDir || s.view.Options().VerboseOutput {
138			event.Log(ctx, "go/packages.Load",
139				tag.Snapshot.Of(s.ID()),
140				tag.Package.Of(pkg.ID),
141				tag.Files.Of(pkg.CompiledGoFiles))
142		}
143		// Ignore packages with no sources, since we will never be able to
144		// correctly invalidate that metadata.
145		if len(pkg.GoFiles) == 0 && len(pkg.CompiledGoFiles) == 0 {
146			continue
147		}
148		// Special case for the builtin package, as it has no dependencies.
149		if pkg.PkgPath == "builtin" {
150			if err := s.buildBuiltinPackage(ctx, pkg.GoFiles); err != nil {
151				return err
152			}
153			continue
154		}
155		// Skip test main packages.
156		if isTestMain(pkg, s.view.gocache) {
157			continue
158		}
159		// Skip filtered packages. They may be added anyway if they're
160		// dependencies of non-filtered packages.
161		if s.view.allFilesExcluded(pkg) {
162			continue
163		}
164		// Set the metadata for this package.
165		m, err := s.setMetadata(ctx, packagePath(pkg.PkgPath), pkg, cfg, map[packageID]struct{}{})
166		if err != nil {
167			return err
168		}
169		if _, err := s.buildPackageHandle(ctx, m.id, s.workspaceParseMode(m.id)); err != nil {
170			return err
171		}
172	}
173	// Rebuild the import graph when the metadata is updated.
174	s.clearAndRebuildImportGraph()
175
176	return nil
177}
178
179// workspaceLayoutErrors returns a diagnostic for every open file, as well as
180// an error message if there are no open files.
181func (s *snapshot) workspaceLayoutError(ctx context.Context) *source.CriticalError {
182	if len(s.workspace.getKnownModFiles()) == 0 {
183		return nil
184	}
185	if s.view.userGo111Module == off {
186		return nil
187	}
188	if s.workspace.moduleSource != legacyWorkspace {
189		return nil
190	}
191	// If the user has one module per view, there is nothing to warn about.
192	if s.ValidBuildConfiguration() && len(s.workspace.getKnownModFiles()) == 1 {
193		return nil
194	}
195
196	// Apply diagnostics about the workspace configuration to relevant open
197	// files.
198	openFiles := s.openFiles()
199
200	// If the snapshot does not have a valid build configuration, it may be
201	// that the user has opened a directory that contains multiple modules.
202	// Check for that an warn about it.
203	if !s.ValidBuildConfiguration() {
204		msg := `gopls requires a module at the root of your workspace.
205You can work with multiple modules by opening each one as a workspace folder.
206Improvements to this workflow will be coming soon (https://github.com/golang/go/issues/32394),
207and you can learn more here: https://github.com/golang/go/issues/36899.`
208		return &source.CriticalError{
209			MainError: errors.Errorf(msg),
210			ErrorList: s.applyCriticalErrorToFiles(ctx, msg, openFiles),
211		}
212	}
213
214	// If the user has one active go.mod file, they may still be editing files
215	// in nested modules. Check the module of each open file and add warnings
216	// that the nested module must be opened as a workspace folder.
217	if len(s.workspace.getActiveModFiles()) == 1 {
218		// Get the active root go.mod file to compare against.
219		var rootModURI span.URI
220		for uri := range s.workspace.getActiveModFiles() {
221			rootModURI = uri
222		}
223		nestedModules := map[string][]source.VersionedFileHandle{}
224		for _, fh := range openFiles {
225			modURI := moduleForURI(s.workspace.knownModFiles, fh.URI())
226			if modURI != rootModURI {
227				modDir := filepath.Dir(modURI.Filename())
228				nestedModules[modDir] = append(nestedModules[modDir], fh)
229			}
230		}
231		// Add a diagnostic to each file in a nested module to mark it as
232		// "orphaned". Don't show a general diagnostic in the progress bar,
233		// because the user may still want to edit a file in a nested module.
234		var srcErrs []*source.Error
235		for modDir, uris := range nestedModules {
236			msg := fmt.Sprintf(`This file is in %s, which is a nested module in the %s module.
237gopls currently requires one module per workspace folder.
238Please open %s as a separate workspace folder.
239You can learn more here: https://github.com/golang/go/issues/36899.
240`, modDir, filepath.Dir(rootModURI.Filename()), modDir)
241			srcErrs = append(srcErrs, s.applyCriticalErrorToFiles(ctx, msg, uris)...)
242		}
243		if len(srcErrs) != 0 {
244			return &source.CriticalError{
245				MainError: errors.Errorf(`You are working in a nested module. Please open it as a separate workspace folder.`),
246				ErrorList: srcErrs,
247			}
248		}
249	}
250	return nil
251}
252
253func (s *snapshot) applyCriticalErrorToFiles(ctx context.Context, msg string, files []source.VersionedFileHandle) []*source.Error {
254	var srcErrs []*source.Error
255	for _, fh := range files {
256		// Place the diagnostics on the package or module declarations.
257		var rng protocol.Range
258		switch fh.Kind() {
259		case source.Go:
260			if pgf, err := s.ParseGo(ctx, fh, source.ParseHeader); err == nil {
261				pkgDecl := span.NewRange(s.FileSet(), pgf.File.Package, pgf.File.Name.End())
262				if spn, err := pkgDecl.Span(); err == nil {
263					rng, _ = pgf.Mapper.Range(spn)
264				}
265			}
266		case source.Mod:
267			if pmf, err := s.ParseMod(ctx, fh); err == nil {
268				if pmf.File.Module != nil && pmf.File.Module.Syntax != nil {
269					rng, _ = rangeFromPositions(pmf.Mapper, pmf.File.Module.Syntax.Start, pmf.File.Module.Syntax.End)
270				}
271			}
272		}
273		srcErrs = append(srcErrs, &source.Error{
274			URI:     fh.URI(),
275			Range:   rng,
276			Kind:    source.ListError,
277			Message: msg,
278		})
279	}
280	return srcErrs
281}
282
283type workspaceDirKey string
284
285type workspaceDirData struct {
286	dir string
287	err error
288}
289
290// getWorkspaceDir gets the URI for the workspace directory associated with
291// this snapshot. The workspace directory is a temp directory containing the
292// go.mod file computed from all active modules.
293func (s *snapshot) getWorkspaceDir(ctx context.Context) (span.URI, error) {
294	s.mu.Lock()
295	h := s.workspaceDirHandle
296	s.mu.Unlock()
297	if h != nil {
298		return getWorkspaceDir(ctx, h, s.generation)
299	}
300	file, err := s.workspace.modFile(ctx, s)
301	if err != nil {
302		return "", err
303	}
304	hash := sha256.New()
305	modContent, err := file.Format()
306	if err != nil {
307		return "", err
308	}
309	sumContent, err := s.workspace.sumFile(ctx, s)
310	if err != nil {
311		return "", err
312	}
313	hash.Write(modContent)
314	hash.Write(sumContent)
315	key := workspaceDirKey(hash.Sum(nil))
316	s.mu.Lock()
317	h = s.generation.Bind(key, func(context.Context, memoize.Arg) interface{} {
318		tmpdir, err := ioutil.TempDir("", "gopls-workspace-mod")
319		if err != nil {
320			return &workspaceDirData{err: err}
321		}
322
323		for name, content := range map[string][]byte{
324			"go.mod": modContent,
325			"go.sum": sumContent,
326		} {
327			filename := filepath.Join(tmpdir, name)
328			if err := ioutil.WriteFile(filename, content, 0644); err != nil {
329				os.RemoveAll(tmpdir)
330				return &workspaceDirData{err: err}
331			}
332		}
333
334		return &workspaceDirData{dir: tmpdir}
335	}, func(v interface{}) {
336		d := v.(*workspaceDirData)
337		if d.dir != "" {
338			if err := os.RemoveAll(d.dir); err != nil {
339				event.Error(context.Background(), "cleaning workspace dir", err)
340			}
341		}
342	})
343	s.workspaceDirHandle = h
344	s.mu.Unlock()
345	return getWorkspaceDir(ctx, h, s.generation)
346}
347
348func getWorkspaceDir(ctx context.Context, h *memoize.Handle, g *memoize.Generation) (span.URI, error) {
349	v, err := h.Get(ctx, g, nil)
350	if err != nil {
351		return "", err
352	}
353	return span.URIFromPath(v.(*workspaceDirData).dir), nil
354}
355
356// setMetadata extracts metadata from pkg and records it in s. It
357// recurses through pkg.Imports to ensure that metadata exists for all
358// dependencies.
359func (s *snapshot) setMetadata(ctx context.Context, pkgPath packagePath, pkg *packages.Package, cfg *packages.Config, seen map[packageID]struct{}) (*metadata, error) {
360	id := packageID(pkg.ID)
361	if _, ok := seen[id]; ok {
362		return nil, errors.Errorf("import cycle detected: %q", id)
363	}
364	// Recreate the metadata rather than reusing it to avoid locking.
365	m := &metadata{
366		id:         id,
367		pkgPath:    pkgPath,
368		name:       packageName(pkg.Name),
369		forTest:    packagePath(packagesinternal.GetForTest(pkg)),
370		typesSizes: pkg.TypesSizes,
371		errors:     pkg.Errors,
372		config:     cfg,
373		module:     pkg.Module,
374	}
375
376	for _, filename := range pkg.CompiledGoFiles {
377		uri := span.URIFromPath(filename)
378		m.compiledGoFiles = append(m.compiledGoFiles, uri)
379		s.addID(uri, m.id)
380	}
381	for _, filename := range pkg.GoFiles {
382		uri := span.URIFromPath(filename)
383		m.goFiles = append(m.goFiles, uri)
384		s.addID(uri, m.id)
385	}
386
387	// TODO(rstambler): is this still necessary?
388	copied := map[packageID]struct{}{
389		id: {},
390	}
391	for k, v := range seen {
392		copied[k] = v
393	}
394	for importPath, importPkg := range pkg.Imports {
395		importPkgPath := packagePath(importPath)
396		importID := packageID(importPkg.ID)
397
398		m.deps = append(m.deps, importID)
399
400		// Don't remember any imports with significant errors.
401		if importPkgPath != "unsafe" && len(importPkg.CompiledGoFiles) == 0 {
402			if m.missingDeps == nil {
403				m.missingDeps = make(map[packagePath]struct{})
404			}
405			m.missingDeps[importPkgPath] = struct{}{}
406			continue
407		}
408		if s.getMetadata(importID) == nil {
409			if _, err := s.setMetadata(ctx, importPkgPath, importPkg, cfg, copied); err != nil {
410				event.Error(ctx, "error in dependency", err)
411			}
412		}
413	}
414
415	// Add the metadata to the cache.
416	s.mu.Lock()
417	defer s.mu.Unlock()
418
419	// TODO: We should make sure not to set duplicate metadata,
420	// and instead panic here. This can be done by making sure not to
421	// reset metadata information for packages we've already seen.
422	if original, ok := s.metadata[m.id]; ok {
423		m = original
424	} else {
425		s.metadata[m.id] = m
426	}
427
428	// Set the workspace packages. If any of the package's files belong to the
429	// view, then the package may be a workspace package.
430	for _, uri := range append(m.compiledGoFiles, m.goFiles...) {
431		if !s.view.contains(uri) {
432			continue
433		}
434
435		// The package's files are in this view. It may be a workspace package.
436		if strings.Contains(string(uri), "/vendor/") {
437			// Vendored packages are not likely to be interesting to the user.
438			continue
439		}
440
441		switch {
442		case m.forTest == "":
443			// A normal package.
444			s.workspacePackages[m.id] = pkgPath
445		case m.forTest == m.pkgPath, m.forTest+"_test" == m.pkgPath:
446			// The test variant of some workspace package or its x_test.
447			// To load it, we need to load the non-test variant with -test.
448			s.workspacePackages[m.id] = m.forTest
449		default:
450			// A test variant of some intermediate package. We don't care about it.
451		}
452	}
453	return m, nil
454}
455
456func isTestMain(pkg *packages.Package, gocache string) bool {
457	// Test mains must have an import path that ends with ".test".
458	if !strings.HasSuffix(pkg.PkgPath, ".test") {
459		return false
460	}
461	// Test main packages are always named "main".
462	if pkg.Name != "main" {
463		return false
464	}
465	// Test mains always have exactly one GoFile that is in the build cache.
466	if len(pkg.GoFiles) > 1 {
467		return false
468	}
469	if !source.InDir(gocache, pkg.GoFiles[0]) {
470		return false
471	}
472	return true
473}
474