1// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package cache
6
7import (
8	"bytes"
9	"context"
10	"fmt"
11	"go/ast"
12	"go/token"
13	"go/types"
14	"path"
15	"sort"
16	"strings"
17	"sync"
18
19	"golang.org/x/tools/go/packages"
20	"golang.org/x/tools/internal/lsp/source"
21	"golang.org/x/tools/internal/lsp/telemetry"
22	"golang.org/x/tools/internal/memoize"
23	"golang.org/x/tools/internal/span"
24	"golang.org/x/tools/internal/telemetry/log"
25	"golang.org/x/tools/internal/telemetry/trace"
26	errors "golang.org/x/xerrors"
27)
28
29type packageHandleKey string
30
31// packageHandle implements source.PackageHandle.
32type packageHandle struct {
33	handle *memoize.Handle
34
35	goFiles []source.ParseGoHandle
36
37	// compiledGoFiles are the ParseGoHandles that compose the package.
38	compiledGoFiles []source.ParseGoHandle
39
40	// mode is the mode the the files were parsed in.
41	mode source.ParseMode
42
43	// m is the metadata associated with the package.
44	m *metadata
45
46	// key is the hashed key for the package.
47	key packageHandleKey
48}
49
50func (ph *packageHandle) packageKey() packageKey {
51	return packageKey{
52		id:   ph.m.id,
53		mode: ph.mode,
54	}
55}
56
57func (ph *packageHandle) isValidImportFor(parentPkgPath string) bool {
58	importPath := string(ph.m.pkgPath)
59
60	pkgRootIndex := strings.Index(importPath, "/internal/")
61	if pkgRootIndex != -1 && parentPkgPath != "command-line-arguments" {
62		if !strings.HasPrefix(parentPkgPath, importPath[0:pkgRootIndex]) {
63			return false
64		}
65	}
66
67	return true
68}
69
70// packageData contains the data produced by type-checking a package.
71type packageData struct {
72	memoize.NoCopy
73
74	pkg *pkg
75	err error
76}
77
78// buildPackageHandle returns a source.PackageHandle for a given package and config.
79func (s *snapshot) buildPackageHandle(ctx context.Context, id packageID, mode source.ParseMode) (*packageHandle, error) {
80	if ph := s.getPackage(id, mode); ph != nil {
81		return ph, nil
82	}
83
84	// Build the PackageHandle for this ID and its dependencies.
85	ph, deps, err := s.buildKey(ctx, id, mode)
86	if err != nil {
87		return nil, err
88	}
89
90	// Do not close over the packageHandle or the snapshot in the Bind function.
91	// This creates a cycle, which causes the finalizers to never run on the handles.
92	// The possible cycles are:
93	//
94	//     packageHandle.h.function -> packageHandle
95	//     packageHandle.h.function -> snapshot -> packageHandle
96	//
97
98	m := ph.m
99	goFiles := ph.goFiles
100	compiledGoFiles := ph.compiledGoFiles
101	key := ph.key
102	fset := s.view.session.cache.fset
103
104	h := s.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} {
105		// Begin loading the direct dependencies, in parallel.
106		for _, dep := range deps {
107			go func(dep *packageHandle) {
108				dep.check(ctx)
109			}(dep)
110		}
111		data := &packageData{}
112		data.pkg, data.err = typeCheck(ctx, fset, m, mode, goFiles, compiledGoFiles, deps)
113		return data
114	})
115	ph.handle = h
116
117	// Cache the PackageHandle in the snapshot.
118	s.addPackage(ph)
119
120	return ph, nil
121}
122
123// buildKey computes the key for a given packageHandle.
124func (s *snapshot) buildKey(ctx context.Context, id packageID, mode source.ParseMode) (*packageHandle, map[packagePath]*packageHandle, error) {
125	m := s.getMetadata(id)
126	if m == nil {
127		return nil, nil, errors.Errorf("no metadata for %s", id)
128	}
129	goFiles, err := s.parseGoHandles(ctx, m.goFiles, mode)
130	if err != nil {
131		return nil, nil, err
132	}
133	compiledGoFiles, err := s.parseGoHandles(ctx, m.compiledGoFiles, mode)
134	if err != nil {
135		return nil, nil, err
136	}
137	ph := &packageHandle{
138		m:               m,
139		goFiles:         goFiles,
140		compiledGoFiles: compiledGoFiles,
141		mode:            mode,
142	}
143	// Make sure all of the depList are sorted.
144	depList := append([]packageID{}, m.deps...)
145	sort.Slice(depList, func(i, j int) bool {
146		return depList[i] < depList[j]
147	})
148
149	deps := make(map[packagePath]*packageHandle)
150
151	// Begin computing the key by getting the depKeys for all dependencies.
152	var depKeys []packageHandleKey
153	for _, depID := range depList {
154		mode := source.ParseExported
155		if _, ok := s.isWorkspacePackage(depID); ok {
156			mode = source.ParseFull
157		}
158		depHandle, err := s.buildPackageHandle(ctx, depID, mode)
159		if err != nil {
160			log.Error(ctx, "no dep handle", err, telemetry.Package.Of(depID))
161
162			// One bad dependency should not prevent us from checking the entire package.
163			// Add a special key to mark a bad dependency.
164			depKeys = append(depKeys, packageHandleKey(fmt.Sprintf("%s import not found", id)))
165			continue
166		}
167		deps[depHandle.m.pkgPath] = depHandle
168		depKeys = append(depKeys, depHandle.key)
169	}
170	ph.key = checkPackageKey(ph.m.id, ph.compiledGoFiles, m.config, depKeys)
171	return ph, deps, nil
172}
173
174func checkPackageKey(id packageID, pghs []source.ParseGoHandle, cfg *packages.Config, deps []packageHandleKey) packageHandleKey {
175	var depBytes []byte
176	for _, dep := range deps {
177		depBytes = append(depBytes, []byte(dep)...)
178	}
179	return packageHandleKey(hashContents([]byte(fmt.Sprintf("%s%s%s%s", id, hashParseKeys(pghs), hashConfig(cfg), hashContents(depBytes)))))
180}
181
182// hashConfig returns the hash for the *packages.Config.
183func hashConfig(config *packages.Config) string {
184	b := bytes.NewBuffer(nil)
185
186	// Dir, Mode, Env, BuildFlags are the parts of the config that can change.
187	b.WriteString(config.Dir)
188	b.WriteString(string(config.Mode))
189
190	for _, e := range config.Env {
191		b.WriteString(e)
192	}
193	for _, f := range config.BuildFlags {
194		b.WriteString(f)
195	}
196	return hashContents(b.Bytes())
197}
198
199func (ph *packageHandle) Check(ctx context.Context) (source.Package, error) {
200	return ph.check(ctx)
201}
202
203func (ph *packageHandle) check(ctx context.Context) (*pkg, error) {
204	v := ph.handle.Get(ctx)
205	if v == nil {
206		return nil, ctx.Err()
207	}
208	data := v.(*packageData)
209	return data.pkg, data.err
210}
211
212func (ph *packageHandle) CompiledGoFiles() []source.ParseGoHandle {
213	return ph.compiledGoFiles
214}
215
216func (ph *packageHandle) ID() string {
217	return string(ph.m.id)
218}
219
220func (ph *packageHandle) MissingDependencies() []string {
221	var md []string
222	for i := range ph.m.missingDeps {
223		md = append(md, string(i))
224	}
225	return md
226}
227
228func hashImports(ctx context.Context, wsPackages []source.PackageHandle) (string, error) {
229	results := make(map[string]bool)
230	var imports []string
231	for _, ph := range wsPackages {
232		// Check package since we do not always invalidate the metadata.
233		pkg, err := ph.Check(ctx)
234		if err != nil {
235			return "", err
236		}
237		for _, path := range pkg.Imports() {
238			imp := path.PkgPath()
239			if _, ok := results[imp]; !ok {
240				results[imp] = true
241				imports = append(imports, imp)
242			}
243		}
244	}
245	sort.Strings(imports)
246	hashed := strings.Join(imports, ",")
247	return hashContents([]byte(hashed)), nil
248}
249
250func (ph *packageHandle) Cached() (source.Package, error) {
251	return ph.cached()
252}
253
254func (ph *packageHandle) cached() (*pkg, error) {
255	v := ph.handle.Cached()
256	if v == nil {
257		return nil, errors.Errorf("no cached type information for %s", ph.m.pkgPath)
258	}
259	data := v.(*packageData)
260	return data.pkg, data.err
261}
262
263func (s *snapshot) parseGoHandles(ctx context.Context, files []span.URI, mode source.ParseMode) ([]source.ParseGoHandle, error) {
264	phs := make([]source.ParseGoHandle, 0, len(files))
265	for _, uri := range files {
266		fh, err := s.GetFile(uri)
267		if err != nil {
268			return nil, err
269		}
270		phs = append(phs, s.view.session.cache.ParseGoHandle(fh, mode))
271	}
272	return phs, nil
273}
274
275func typeCheck(ctx context.Context, fset *token.FileSet, m *metadata, mode source.ParseMode, goFiles []source.ParseGoHandle, compiledGoFiles []source.ParseGoHandle, deps map[packagePath]*packageHandle) (*pkg, error) {
276	ctx, done := trace.StartSpan(ctx, "cache.importer.typeCheck", telemetry.Package.Of(m.id))
277	defer done()
278
279	var rawErrors []error
280	for _, err := range m.errors {
281		rawErrors = append(rawErrors, err)
282	}
283
284	pkg := &pkg{
285		id:              m.id,
286		pkgPath:         m.pkgPath,
287		mode:            mode,
288		goFiles:         goFiles,
289		compiledGoFiles: compiledGoFiles,
290		module:          m.module,
291		imports:         make(map[packagePath]*pkg),
292		typesSizes:      m.typesSizes,
293		typesInfo: &types.Info{
294			Types:      make(map[ast.Expr]types.TypeAndValue),
295			Defs:       make(map[*ast.Ident]types.Object),
296			Uses:       make(map[*ast.Ident]types.Object),
297			Implicits:  make(map[ast.Node]types.Object),
298			Selections: make(map[*ast.SelectorExpr]*types.Selection),
299			Scopes:     make(map[ast.Node]*types.Scope),
300		},
301		forTest: m.forTest,
302	}
303	var (
304		files        = make([]*ast.File, len(pkg.compiledGoFiles))
305		parseErrors  = make([]error, len(pkg.compiledGoFiles))
306		actualErrors = make([]error, len(pkg.compiledGoFiles))
307		wg           sync.WaitGroup
308	)
309	for i, ph := range pkg.compiledGoFiles {
310		wg.Add(1)
311		go func(i int, ph source.ParseGoHandle) {
312			files[i], _, _, parseErrors[i], actualErrors[i] = ph.Parse(ctx)
313			wg.Done()
314		}(i, ph)
315	}
316	for _, ph := range pkg.goFiles {
317		wg.Add(1)
318		// We need to parse the non-compiled go files, but we don't care about their errors.
319		go func(ph source.ParseGoHandle) {
320			ph.Parse(ctx)
321			wg.Done()
322		}(ph)
323	}
324	wg.Wait()
325
326	for _, e := range parseErrors {
327		if e != nil {
328			rawErrors = append(rawErrors, e)
329		}
330	}
331
332	var i int
333	for _, f := range files {
334		if f != nil {
335			files[i] = f
336			i++
337		}
338	}
339	files = files[:i]
340
341	// Use the default type information for the unsafe package.
342	if pkg.pkgPath == "unsafe" {
343		pkg.types = types.Unsafe
344		// Don't type check Unsafe: it's unnecessary, and doing so exposes a data
345		// race to Unsafe.completed.
346		return pkg, nil
347	} else if len(files) == 0 { // not the unsafe package, no parsed files
348		return nil, errors.Errorf("no parsed files for package %s, expected: %s, errors: %v, list errors: %v", pkg.pkgPath, pkg.compiledGoFiles, actualErrors, rawErrors)
349	} else {
350		pkg.types = types.NewPackage(string(m.pkgPath), m.name)
351	}
352
353	cfg := &types.Config{
354		Error: func(e error) {
355			rawErrors = append(rawErrors, e)
356		},
357		Importer: importerFunc(func(pkgPath string) (*types.Package, error) {
358			// If the context was cancelled, we should abort.
359			if ctx.Err() != nil {
360				return nil, ctx.Err()
361			}
362			dep := deps[packagePath(pkgPath)]
363			if dep == nil {
364				// We may be in GOPATH mode, in which case we need to check vendor dirs.
365				searchDir := path.Dir(pkg.PkgPath())
366				for {
367					vdir := packagePath(path.Join(searchDir, "vendor", pkgPath))
368					if vdep := deps[vdir]; vdep != nil {
369						dep = vdep
370						break
371					}
372
373					// Search until Dir doesn't take us anywhere new, e.g. "." or "/".
374					next := path.Dir(searchDir)
375					if searchDir == next {
376						break
377					}
378					searchDir = next
379				}
380			}
381			if dep == nil {
382				return nil, errors.Errorf("no package for import %s", pkgPath)
383			}
384			if !dep.isValidImportFor(pkg.PkgPath()) {
385				return nil, errors.Errorf("invalid use of internal package %s", pkgPath)
386			}
387			depPkg, err := dep.check(ctx)
388			if err != nil {
389				return nil, err
390			}
391			pkg.imports[depPkg.pkgPath] = depPkg
392			return depPkg.types, nil
393		}),
394	}
395	check := types.NewChecker(cfg, fset, pkg.types, pkg.typesInfo)
396
397	// Type checking errors are handled via the config, so ignore them here.
398	_ = check.Files(files)
399	// If the context was cancelled, we may have returned a ton of transient
400	// errors to the type checker. Swallow them.
401	if ctx.Err() != nil {
402		return nil, ctx.Err()
403	}
404
405	// We don't care about a package's errors unless we have parsed it in full.
406	if mode == source.ParseFull {
407		for _, e := range rawErrors {
408			srcErr, err := sourceError(ctx, fset, pkg, e)
409			if err != nil {
410				log.Error(ctx, "unable to compute error positions", err, telemetry.Package.Of(pkg.ID()))
411				continue
412			}
413			pkg.errors = append(pkg.errors, srcErr)
414		}
415	}
416	return pkg, nil
417}
418
419// An importFunc is an implementation of the single-method
420// types.Importer interface based on a function value.
421type importerFunc func(path string) (*types.Package, error)
422
423func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
424