1package loader 2 3import ( 4 "errors" 5 "fmt" 6 "go/ast" 7 "go/parser" 8 "go/scanner" 9 "go/token" 10 "go/types" 11 "os" 12 "time" 13 14 "honnef.co/go/tools/config" 15 "honnef.co/go/tools/internal/cache" 16 "honnef.co/go/tools/internal/go/gcimporter" 17 18 "golang.org/x/tools/go/packages" 19) 20 21const MaxFileSize = 50 * 1024 * 1024 // 50 MB 22 23var errMaxFileSize = errors.New("file exceeds max file size") 24 25type PackageSpec struct { 26 ID string 27 Name string 28 PkgPath string 29 // Errors that occured while building the import graph. These will 30 // primarily be parse errors or failure to resolve imports, but 31 // may also be other errors. 32 Errors []packages.Error 33 GoFiles []string 34 CompiledGoFiles []string 35 OtherFiles []string 36 ExportFile string 37 Imports map[string]*PackageSpec 38 TypesSizes types.Sizes 39 Hash cache.ActionID 40 41 Config config.Config 42} 43 44func (spec *PackageSpec) String() string { 45 return spec.ID 46} 47 48type Package struct { 49 *PackageSpec 50 51 // Errors that occured while loading the package. These will 52 // primarily be parse or type errors, but may also be lower-level 53 // failures such as file-system ones. 54 Errors []packages.Error 55 Types *types.Package 56 Fset *token.FileSet 57 Syntax []*ast.File 58 TypesInfo *types.Info 59} 60 61// Graph resolves patterns and returns packages with all the 62// information required to later load type information, and optionally 63// syntax trees. 64// 65// The provided config can set any setting with the exception of Mode. 66func Graph(cfg *packages.Config, patterns ...string) ([]*PackageSpec, error) { 67 var dcfg packages.Config 68 if cfg != nil { 69 dcfg = *cfg 70 } 71 dcfg.Mode = packages.NeedName | 72 packages.NeedImports | 73 packages.NeedDeps | 74 packages.NeedExportsFile | 75 packages.NeedFiles | 76 packages.NeedCompiledGoFiles | 77 packages.NeedTypesSizes 78 pkgs, err := packages.Load(&dcfg, patterns...) 79 if err != nil { 80 return nil, err 81 } 82 83 m := map[*packages.Package]*PackageSpec{} 84 packages.Visit(pkgs, nil, func(pkg *packages.Package) { 85 spec := &PackageSpec{ 86 ID: pkg.ID, 87 Name: pkg.Name, 88 PkgPath: pkg.PkgPath, 89 Errors: pkg.Errors, 90 GoFiles: pkg.GoFiles, 91 CompiledGoFiles: pkg.CompiledGoFiles, 92 OtherFiles: pkg.OtherFiles, 93 ExportFile: pkg.ExportFile, 94 Imports: map[string]*PackageSpec{}, 95 TypesSizes: pkg.TypesSizes, 96 } 97 for path, imp := range pkg.Imports { 98 spec.Imports[path] = m[imp] 99 } 100 if cdir := config.Dir(pkg.GoFiles); cdir != "" { 101 cfg, err := config.Load(cdir) 102 if err != nil { 103 spec.Errors = append(spec.Errors, convertError(err)...) 104 } 105 spec.Config = cfg 106 } else { 107 spec.Config = config.DefaultConfig 108 } 109 spec.Hash, err = computeHash(spec) 110 if err != nil { 111 spec.Errors = append(spec.Errors, convertError(err)...) 112 } 113 m[pkg] = spec 114 }) 115 out := make([]*PackageSpec, 0, len(pkgs)) 116 for _, pkg := range pkgs { 117 if len(pkg.CompiledGoFiles) == 0 && len(pkg.Errors) == 0 && pkg.PkgPath != "unsafe" { 118 // If a package consists only of test files, then 119 // go/packages incorrectly(?) returns an empty package for 120 // the non-test variant. Get rid of those packages. See 121 // #646. 122 // 123 // Do not, however, skip packages that have errors. Those, 124 // too, may have no files, but we want to print the 125 // errors. 126 continue 127 } 128 out = append(out, m[pkg]) 129 } 130 131 return out, nil 132} 133 134type program struct { 135 fset *token.FileSet 136 packages map[string]*types.Package 137} 138 139type Stats struct { 140 Source time.Duration 141 Export map[*PackageSpec]time.Duration 142} 143 144// Load loads the package described in spec. Imports will be loaded 145// from export data, while the package itself will be loaded from 146// source. 147// 148// An error will only be returned for system failures, such as failure 149// to read export data from disk. Syntax and type errors, among 150// others, will only populate the returned package's Errors field. 151func Load(spec *PackageSpec) (*Package, Stats, error) { 152 prog := &program{ 153 fset: token.NewFileSet(), 154 packages: map[string]*types.Package{}, 155 } 156 157 stats := Stats{ 158 Export: map[*PackageSpec]time.Duration{}, 159 } 160 var b []byte 161 for _, imp := range spec.Imports { 162 if imp.PkgPath == "unsafe" { 163 continue 164 } 165 t := time.Now() 166 var err error 167 _, b, err = prog.loadFromExport(imp, b) 168 stats.Export[imp] = time.Since(t) 169 if err != nil { 170 return nil, stats, err 171 } 172 } 173 t := time.Now() 174 pkg, err := prog.loadFromSource(spec) 175 if err == errMaxFileSize { 176 pkg, _, err = prog.loadFromExport(spec, b) 177 } 178 stats.Source = time.Since(t) 179 return pkg, stats, err 180} 181 182// loadFromExport loads a package from export data. 183func (prog *program) loadFromExport(spec *PackageSpec, b []byte) (*Package, []byte, error) { 184 // log.Printf("Loading package %s from export", spec) 185 if spec.ExportFile == "" { 186 return nil, b, fmt.Errorf("no export data for %q", spec.ID) 187 } 188 f, err := os.Open(spec.ExportFile) 189 if err != nil { 190 return nil, b, err 191 } 192 defer f.Close() 193 194 b, err = gcimporter.GetExportData(f, b) 195 if err != nil { 196 return nil, b, err 197 } 198 199 _, tpkg, err := gcimporter.IImportData(prog.fset, prog.packages, b[1:], spec.PkgPath) 200 if err != nil { 201 return nil, b, err 202 } 203 pkg := &Package{ 204 PackageSpec: spec, 205 Types: tpkg, 206 Fset: prog.fset, 207 } 208 // runtime.SetFinalizer(pkg, func(pkg *Package) { 209 // log.Println("Unloading package", pkg.PkgPath) 210 // }) 211 return pkg, b, nil 212} 213 214// loadFromSource loads a package from source. All of its dependencies 215// must have been loaded already. 216func (prog *program) loadFromSource(spec *PackageSpec) (*Package, error) { 217 if len(spec.Errors) > 0 { 218 panic("LoadFromSource called on package with errors") 219 } 220 221 pkg := &Package{ 222 PackageSpec: spec, 223 Types: types.NewPackage(spec.PkgPath, spec.Name), 224 Syntax: make([]*ast.File, len(spec.CompiledGoFiles)), 225 Fset: prog.fset, 226 TypesInfo: &types.Info{ 227 Types: make(map[ast.Expr]types.TypeAndValue), 228 Defs: make(map[*ast.Ident]types.Object), 229 Uses: make(map[*ast.Ident]types.Object), 230 Implicits: make(map[ast.Node]types.Object), 231 Scopes: make(map[ast.Node]*types.Scope), 232 Selections: make(map[*ast.SelectorExpr]*types.Selection), 233 }, 234 } 235 // runtime.SetFinalizer(pkg, func(pkg *Package) { 236 // log.Println("Unloading package", pkg.PkgPath) 237 // }) 238 239 // OPT(dh): many packages have few files, much fewer than there 240 // are CPU cores. Additionally, parsing each individual file is 241 // very fast. A naive parallel implementation of this loop won't 242 // be faster, and tends to be slower due to extra scheduling, 243 // bookkeeping and potentially false sharing of cache lines. 244 for i, file := range spec.CompiledGoFiles { 245 f, err := os.Open(file) 246 if err != nil { 247 return nil, err 248 } 249 fi, err := f.Stat() 250 if err != nil { 251 return nil, err 252 } 253 if fi.Size() >= MaxFileSize { 254 return nil, errMaxFileSize 255 } 256 af, err := parser.ParseFile(prog.fset, file, f, parser.ParseComments) 257 f.Close() 258 if err != nil { 259 pkg.Errors = append(pkg.Errors, convertError(err)...) 260 return pkg, nil 261 } 262 pkg.Syntax[i] = af 263 } 264 importer := func(path string) (*types.Package, error) { 265 if path == "unsafe" { 266 return types.Unsafe, nil 267 } 268 if path == "C" { 269 // go/packages doesn't tell us that cgo preprocessing 270 // failed. When we subsequently try to parse the package, 271 // we'll encounter the raw C import. 272 return nil, errors.New("cgo preprocessing failed") 273 } 274 ispecpkg := spec.Imports[path] 275 if ispecpkg == nil { 276 return nil, fmt.Errorf("trying to import %q in the context of %q returned nil PackageSpec", path, spec) 277 } 278 ipkg := prog.packages[ispecpkg.PkgPath] 279 if ipkg == nil { 280 return nil, fmt.Errorf("trying to import %q (%q) in the context of %q returned nil PackageSpec", ispecpkg.PkgPath, path, spec) 281 } 282 return ipkg, nil 283 } 284 tc := &types.Config{ 285 Importer: importerFunc(importer), 286 Error: func(err error) { 287 pkg.Errors = append(pkg.Errors, convertError(err)...) 288 }, 289 } 290 types.NewChecker(tc, pkg.Fset, pkg.Types, pkg.TypesInfo).Files(pkg.Syntax) 291 return pkg, nil 292} 293 294func convertError(err error) []packages.Error { 295 var errs []packages.Error 296 // taken from go/packages 297 switch err := err.(type) { 298 case packages.Error: 299 // from driver 300 errs = append(errs, err) 301 302 case *os.PathError: 303 // from parser 304 errs = append(errs, packages.Error{ 305 Pos: err.Path + ":1", 306 Msg: err.Err.Error(), 307 Kind: packages.ParseError, 308 }) 309 310 case scanner.ErrorList: 311 // from parser 312 for _, err := range err { 313 errs = append(errs, packages.Error{ 314 Pos: err.Pos.String(), 315 Msg: err.Msg, 316 Kind: packages.ParseError, 317 }) 318 } 319 320 case types.Error: 321 // from type checker 322 errs = append(errs, packages.Error{ 323 Pos: err.Fset.Position(err.Pos).String(), 324 Msg: err.Msg, 325 Kind: packages.TypeError, 326 }) 327 328 default: 329 errs = append(errs, packages.Error{ 330 Pos: "-", 331 Msg: err.Error(), 332 Kind: packages.UnknownError, 333 }) 334 } 335 return errs 336} 337 338type importerFunc func(path string) (*types.Package, error) 339 340func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } 341