1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package cache 6 7import ( 8 "bytes" 9 "context" 10 "fmt" 11 "go/ast" 12 "go/token" 13 "go/types" 14 "path" 15 "sort" 16 "strings" 17 "sync" 18 19 "golang.org/x/tools/go/packages" 20 "golang.org/x/tools/internal/lsp/source" 21 "golang.org/x/tools/internal/lsp/telemetry" 22 "golang.org/x/tools/internal/memoize" 23 "golang.org/x/tools/internal/span" 24 "golang.org/x/tools/internal/telemetry/log" 25 "golang.org/x/tools/internal/telemetry/trace" 26 errors "golang.org/x/xerrors" 27) 28 29type packageHandleKey string 30 31// packageHandle implements source.PackageHandle. 32type packageHandle struct { 33 handle *memoize.Handle 34 35 goFiles []source.ParseGoHandle 36 37 // compiledGoFiles are the ParseGoHandles that compose the package. 38 compiledGoFiles []source.ParseGoHandle 39 40 // mode is the mode the the files were parsed in. 41 mode source.ParseMode 42 43 // m is the metadata associated with the package. 44 m *metadata 45 46 // key is the hashed key for the package. 47 key packageHandleKey 48} 49 50func (ph *packageHandle) packageKey() packageKey { 51 return packageKey{ 52 id: ph.m.id, 53 mode: ph.mode, 54 } 55} 56 57func (ph *packageHandle) isValidImportFor(parentPkgPath string) bool { 58 importPath := string(ph.m.pkgPath) 59 60 pkgRootIndex := strings.Index(importPath, "/internal/") 61 if pkgRootIndex != -1 && parentPkgPath != "command-line-arguments" { 62 if !strings.HasPrefix(parentPkgPath, importPath[0:pkgRootIndex]) { 63 return false 64 } 65 } 66 67 return true 68} 69 70// packageData contains the data produced by type-checking a package. 71type packageData struct { 72 memoize.NoCopy 73 74 pkg *pkg 75 err error 76} 77 78// buildPackageHandle returns a source.PackageHandle for a given package and config. 79func (s *snapshot) buildPackageHandle(ctx context.Context, id packageID, mode source.ParseMode) (*packageHandle, error) { 80 if ph := s.getPackage(id, mode); ph != nil { 81 return ph, nil 82 } 83 84 // Build the PackageHandle for this ID and its dependencies. 85 ph, deps, err := s.buildKey(ctx, id, mode) 86 if err != nil { 87 return nil, err 88 } 89 90 // Do not close over the packageHandle or the snapshot in the Bind function. 91 // This creates a cycle, which causes the finalizers to never run on the handles. 92 // The possible cycles are: 93 // 94 // packageHandle.h.function -> packageHandle 95 // packageHandle.h.function -> snapshot -> packageHandle 96 // 97 98 m := ph.m 99 goFiles := ph.goFiles 100 compiledGoFiles := ph.compiledGoFiles 101 key := ph.key 102 fset := s.view.session.cache.fset 103 104 h := s.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} { 105 // Begin loading the direct dependencies, in parallel. 106 for _, dep := range deps { 107 go func(dep *packageHandle) { 108 dep.check(ctx) 109 }(dep) 110 } 111 data := &packageData{} 112 data.pkg, data.err = typeCheck(ctx, fset, m, mode, goFiles, compiledGoFiles, deps) 113 return data 114 }) 115 ph.handle = h 116 117 // Cache the PackageHandle in the snapshot. 118 s.addPackage(ph) 119 120 return ph, nil 121} 122 123// buildKey computes the key for a given packageHandle. 124func (s *snapshot) buildKey(ctx context.Context, id packageID, mode source.ParseMode) (*packageHandle, map[packagePath]*packageHandle, error) { 125 m := s.getMetadata(id) 126 if m == nil { 127 return nil, nil, errors.Errorf("no metadata for %s", id) 128 } 129 goFiles, err := s.parseGoHandles(ctx, m.goFiles, mode) 130 if err != nil { 131 return nil, nil, err 132 } 133 compiledGoFiles, err := s.parseGoHandles(ctx, m.compiledGoFiles, mode) 134 if err != nil { 135 return nil, nil, err 136 } 137 ph := &packageHandle{ 138 m: m, 139 goFiles: goFiles, 140 compiledGoFiles: compiledGoFiles, 141 mode: mode, 142 } 143 // Make sure all of the depList are sorted. 144 depList := append([]packageID{}, m.deps...) 145 sort.Slice(depList, func(i, j int) bool { 146 return depList[i] < depList[j] 147 }) 148 149 deps := make(map[packagePath]*packageHandle) 150 151 // Begin computing the key by getting the depKeys for all dependencies. 152 var depKeys []packageHandleKey 153 for _, depID := range depList { 154 mode := source.ParseExported 155 if _, ok := s.isWorkspacePackage(depID); ok { 156 mode = source.ParseFull 157 } 158 depHandle, err := s.buildPackageHandle(ctx, depID, mode) 159 if err != nil { 160 log.Error(ctx, "no dep handle", err, telemetry.Package.Of(depID)) 161 162 // One bad dependency should not prevent us from checking the entire package. 163 // Add a special key to mark a bad dependency. 164 depKeys = append(depKeys, packageHandleKey(fmt.Sprintf("%s import not found", id))) 165 continue 166 } 167 deps[depHandle.m.pkgPath] = depHandle 168 depKeys = append(depKeys, depHandle.key) 169 } 170 ph.key = checkPackageKey(ph.m.id, ph.compiledGoFiles, m.config, depKeys) 171 return ph, deps, nil 172} 173 174func checkPackageKey(id packageID, pghs []source.ParseGoHandle, cfg *packages.Config, deps []packageHandleKey) packageHandleKey { 175 var depBytes []byte 176 for _, dep := range deps { 177 depBytes = append(depBytes, []byte(dep)...) 178 } 179 return packageHandleKey(hashContents([]byte(fmt.Sprintf("%s%s%s%s", id, hashParseKeys(pghs), hashConfig(cfg), hashContents(depBytes))))) 180} 181 182// hashConfig returns the hash for the *packages.Config. 183func hashConfig(config *packages.Config) string { 184 b := bytes.NewBuffer(nil) 185 186 // Dir, Mode, Env, BuildFlags are the parts of the config that can change. 187 b.WriteString(config.Dir) 188 b.WriteString(string(config.Mode)) 189 190 for _, e := range config.Env { 191 b.WriteString(e) 192 } 193 for _, f := range config.BuildFlags { 194 b.WriteString(f) 195 } 196 return hashContents(b.Bytes()) 197} 198 199func (ph *packageHandle) Check(ctx context.Context) (source.Package, error) { 200 return ph.check(ctx) 201} 202 203func (ph *packageHandle) check(ctx context.Context) (*pkg, error) { 204 v := ph.handle.Get(ctx) 205 if v == nil { 206 return nil, ctx.Err() 207 } 208 data := v.(*packageData) 209 return data.pkg, data.err 210} 211 212func (ph *packageHandle) CompiledGoFiles() []source.ParseGoHandle { 213 return ph.compiledGoFiles 214} 215 216func (ph *packageHandle) ID() string { 217 return string(ph.m.id) 218} 219 220func (ph *packageHandle) MissingDependencies() []string { 221 var md []string 222 for i := range ph.m.missingDeps { 223 md = append(md, string(i)) 224 } 225 return md 226} 227 228func hashImports(ctx context.Context, wsPackages []source.PackageHandle) (string, error) { 229 results := make(map[string]bool) 230 var imports []string 231 for _, ph := range wsPackages { 232 // Check package since we do not always invalidate the metadata. 233 pkg, err := ph.Check(ctx) 234 if err != nil { 235 return "", err 236 } 237 for _, path := range pkg.Imports() { 238 imp := path.PkgPath() 239 if _, ok := results[imp]; !ok { 240 results[imp] = true 241 imports = append(imports, imp) 242 } 243 } 244 } 245 sort.Strings(imports) 246 hashed := strings.Join(imports, ",") 247 return hashContents([]byte(hashed)), nil 248} 249 250func (ph *packageHandle) Cached() (source.Package, error) { 251 return ph.cached() 252} 253 254func (ph *packageHandle) cached() (*pkg, error) { 255 v := ph.handle.Cached() 256 if v == nil { 257 return nil, errors.Errorf("no cached type information for %s", ph.m.pkgPath) 258 } 259 data := v.(*packageData) 260 return data.pkg, data.err 261} 262 263func (s *snapshot) parseGoHandles(ctx context.Context, files []span.URI, mode source.ParseMode) ([]source.ParseGoHandle, error) { 264 phs := make([]source.ParseGoHandle, 0, len(files)) 265 for _, uri := range files { 266 fh, err := s.GetFile(uri) 267 if err != nil { 268 return nil, err 269 } 270 phs = append(phs, s.view.session.cache.ParseGoHandle(fh, mode)) 271 } 272 return phs, nil 273} 274 275func typeCheck(ctx context.Context, fset *token.FileSet, m *metadata, mode source.ParseMode, goFiles []source.ParseGoHandle, compiledGoFiles []source.ParseGoHandle, deps map[packagePath]*packageHandle) (*pkg, error) { 276 ctx, done := trace.StartSpan(ctx, "cache.importer.typeCheck", telemetry.Package.Of(m.id)) 277 defer done() 278 279 var rawErrors []error 280 for _, err := range m.errors { 281 rawErrors = append(rawErrors, err) 282 } 283 284 pkg := &pkg{ 285 id: m.id, 286 pkgPath: m.pkgPath, 287 mode: mode, 288 goFiles: goFiles, 289 compiledGoFiles: compiledGoFiles, 290 module: m.module, 291 imports: make(map[packagePath]*pkg), 292 typesSizes: m.typesSizes, 293 typesInfo: &types.Info{ 294 Types: make(map[ast.Expr]types.TypeAndValue), 295 Defs: make(map[*ast.Ident]types.Object), 296 Uses: make(map[*ast.Ident]types.Object), 297 Implicits: make(map[ast.Node]types.Object), 298 Selections: make(map[*ast.SelectorExpr]*types.Selection), 299 Scopes: make(map[ast.Node]*types.Scope), 300 }, 301 forTest: m.forTest, 302 } 303 var ( 304 files = make([]*ast.File, len(pkg.compiledGoFiles)) 305 parseErrors = make([]error, len(pkg.compiledGoFiles)) 306 actualErrors = make([]error, len(pkg.compiledGoFiles)) 307 wg sync.WaitGroup 308 ) 309 for i, ph := range pkg.compiledGoFiles { 310 wg.Add(1) 311 go func(i int, ph source.ParseGoHandle) { 312 files[i], _, _, parseErrors[i], actualErrors[i] = ph.Parse(ctx) 313 wg.Done() 314 }(i, ph) 315 } 316 for _, ph := range pkg.goFiles { 317 wg.Add(1) 318 // We need to parse the non-compiled go files, but we don't care about their errors. 319 go func(ph source.ParseGoHandle) { 320 ph.Parse(ctx) 321 wg.Done() 322 }(ph) 323 } 324 wg.Wait() 325 326 for _, e := range parseErrors { 327 if e != nil { 328 rawErrors = append(rawErrors, e) 329 } 330 } 331 332 var i int 333 for _, f := range files { 334 if f != nil { 335 files[i] = f 336 i++ 337 } 338 } 339 files = files[:i] 340 341 // Use the default type information for the unsafe package. 342 if pkg.pkgPath == "unsafe" { 343 pkg.types = types.Unsafe 344 // Don't type check Unsafe: it's unnecessary, and doing so exposes a data 345 // race to Unsafe.completed. 346 return pkg, nil 347 } else if len(files) == 0 { // not the unsafe package, no parsed files 348 return nil, errors.Errorf("no parsed files for package %s, expected: %s, errors: %v, list errors: %v", pkg.pkgPath, pkg.compiledGoFiles, actualErrors, rawErrors) 349 } else { 350 pkg.types = types.NewPackage(string(m.pkgPath), m.name) 351 } 352 353 cfg := &types.Config{ 354 Error: func(e error) { 355 rawErrors = append(rawErrors, e) 356 }, 357 Importer: importerFunc(func(pkgPath string) (*types.Package, error) { 358 // If the context was cancelled, we should abort. 359 if ctx.Err() != nil { 360 return nil, ctx.Err() 361 } 362 dep := deps[packagePath(pkgPath)] 363 if dep == nil { 364 // We may be in GOPATH mode, in which case we need to check vendor dirs. 365 searchDir := path.Dir(pkg.PkgPath()) 366 for { 367 vdir := packagePath(path.Join(searchDir, "vendor", pkgPath)) 368 if vdep := deps[vdir]; vdep != nil { 369 dep = vdep 370 break 371 } 372 373 // Search until Dir doesn't take us anywhere new, e.g. "." or "/". 374 next := path.Dir(searchDir) 375 if searchDir == next { 376 break 377 } 378 searchDir = next 379 } 380 } 381 if dep == nil { 382 return nil, errors.Errorf("no package for import %s", pkgPath) 383 } 384 if !dep.isValidImportFor(pkg.PkgPath()) { 385 return nil, errors.Errorf("invalid use of internal package %s", pkgPath) 386 } 387 depPkg, err := dep.check(ctx) 388 if err != nil { 389 return nil, err 390 } 391 pkg.imports[depPkg.pkgPath] = depPkg 392 return depPkg.types, nil 393 }), 394 } 395 check := types.NewChecker(cfg, fset, pkg.types, pkg.typesInfo) 396 397 // Type checking errors are handled via the config, so ignore them here. 398 _ = check.Files(files) 399 // If the context was cancelled, we may have returned a ton of transient 400 // errors to the type checker. Swallow them. 401 if ctx.Err() != nil { 402 return nil, ctx.Err() 403 } 404 405 // We don't care about a package's errors unless we have parsed it in full. 406 if mode == source.ParseFull { 407 for _, e := range rawErrors { 408 srcErr, err := sourceError(ctx, fset, pkg, e) 409 if err != nil { 410 log.Error(ctx, "unable to compute error positions", err, telemetry.Package.Of(pkg.ID())) 411 continue 412 } 413 pkg.errors = append(pkg.errors, srcErr) 414 } 415 } 416 return pkg, nil 417} 418 419// An importFunc is an implementation of the single-method 420// types.Importer interface based on a function value. 421type importerFunc func(path string) (*types.Package, error) 422 423func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } 424