1// Copyright 2018 The CUE Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package load 16 17import ( 18 // TODO: remove this usage 19 20 "os" 21 "path" 22 "path/filepath" 23 "regexp" 24 "strings" 25 26 "cuelang.org/go/cue/build" 27 "cuelang.org/go/cue/errors" 28 "cuelang.org/go/cue/token" 29) 30 31// A match represents the result of matching a single package pattern. 32type match struct { 33 Pattern string // the pattern itself 34 Literal bool // whether it is a literal (no wildcards) 35 Pkgs []*build.Instance 36 Err errors.Error 37} 38 39// TODO: should be matched from module file only. 40// The pattern is either "all" (all packages), "std" (standard packages), 41// "cmd" (standard commands), or a path including "...". 42func (l *loader) matchPackages(pattern, pkgName string) *match { 43 // cfg := l.cfg 44 m := &match{ 45 Pattern: pattern, 46 Literal: false, 47 } 48 // match := func(string) bool { return true } 49 // treeCanMatch := func(string) bool { return true } 50 // if !isMetaPackage(pattern) { 51 // match = matchPattern(pattern) 52 // treeCanMatch = treeCanMatchPattern(pattern) 53 // } 54 55 // have := map[string]bool{ 56 // "builtin": true, // ignore pseudo-package that exists only for documentation 57 // } 58 59 // for _, src := range cfg.srcDirs() { 60 // if pattern == "std" || pattern == "cmd" { 61 // continue 62 // } 63 // src = filepath.Clean(src) + string(filepath.Separator) 64 // root := src 65 // if pattern == "cmd" { 66 // root += "cmd" + string(filepath.Separator) 67 // } 68 // filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { 69 // if err != nil || path == src { 70 // return nil 71 // } 72 73 // want := true 74 // // Avoid .foo, _foo, and testdata directory trees. 75 // _, elem := filepath.Split(path) 76 // if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { 77 // want = false 78 // } 79 80 // name := filepath.ToSlash(path[len(src):]) 81 // if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") { 82 // // The name "std" is only the standard library. 83 // // If the name is cmd, it's the root of the command tree. 84 // want = false 85 // } 86 // if !treeCanMatch(name) { 87 // want = false 88 // } 89 90 // if !fi.IsDir() { 91 // if fi.Mode()&os.ModeSymlink != 0 && want { 92 // if target, err := os.Stat(path); err == nil && target.IsDir() { 93 // fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path) 94 // } 95 // } 96 // return nil 97 // } 98 // if !want { 99 // return skipDir 100 // } 101 102 // if have[name] { 103 // return nil 104 // } 105 // have[name] = true 106 // if !match(name) { 107 // return nil 108 // } 109 // pkg := l.importPkg(".", path) 110 // if err := pkg.Error; err != nil { 111 // if _, noGo := err.(*noCUEError); noGo { 112 // return nil 113 // } 114 // } 115 116 // // If we are expanding "cmd", skip main 117 // // packages under cmd/vendor. At least as of 118 // // March, 2017, there is one there for the 119 // // vendored pprof tool. 120 // if pattern == "cmd" && strings.HasPrefix(pkg.DisplayPath, "cmd/vendor") && pkg.PkgName == "main" { 121 // return nil 122 // } 123 124 // m.Pkgs = append(m.Pkgs, pkg) 125 // return nil 126 // }) 127 // } 128 return m 129} 130 131// matchPackagesInFS is like allPackages but is passed a pattern 132// beginning ./ or ../, meaning it should scan the tree rooted 133// at the given directory. There are ... in the pattern too. 134// (See go help packages for pattern syntax.) 135func (l *loader) matchPackagesInFS(pattern, pkgName string) *match { 136 c := l.cfg 137 m := &match{ 138 Pattern: pattern, 139 Literal: false, 140 } 141 142 // Find directory to begin the scan. 143 // Could be smarter but this one optimization 144 // is enough for now, since ... is usually at the 145 // end of a path. 146 i := strings.Index(pattern, "...") 147 dir, _ := path.Split(pattern[:i]) 148 149 root := l.abs(dir) 150 151 // Find new module root from here or check there are no additional 152 // cue.mod files between here and the next module. 153 154 if !hasFilepathPrefix(root, c.ModuleRoot) { 155 m.Err = errors.Newf(token.NoPos, 156 "cue: pattern %s refers to dir %s, outside module root %s", 157 pattern, root, c.ModuleRoot) 158 return m 159 } 160 161 pkgDir := filepath.Join(root, modDir) 162 // TODO(legacy): remove 163 pkgDir2 := filepath.Join(root, "pkg") 164 165 _ = c.fileSystem.walk(root, func(path string, fi os.FileInfo, err errors.Error) errors.Error { 166 if err != nil || !fi.IsDir() { 167 return nil 168 } 169 if path == pkgDir || path == pkgDir2 { 170 return skipDir 171 } 172 173 top := path == root 174 175 // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". 176 _, elem := filepath.Split(path) 177 dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." 178 if dot || strings.HasPrefix(elem, "_") || (elem == "testdata" && !top) { 179 return skipDir 180 } 181 182 if !top { 183 // Ignore other modules found in subdirectories. 184 if _, err := c.fileSystem.stat(filepath.Join(path, modDir)); err == nil { 185 return skipDir 186 } 187 } 188 189 // name := prefix + filepath.ToSlash(path) 190 // if !match(name) { 191 // return nil 192 // } 193 194 // We keep the directory if we can import it, or if we can't import it 195 // due to invalid CUE source files. This means that directories 196 // containing parse errors will be built (and fail) instead of being 197 // silently skipped as not matching the pattern. 198 // Do not take root, as we want to stay relative 199 // to one dir only. 200 dir, e := filepath.Rel(c.Dir, path) 201 if e != nil { 202 panic(err) 203 } else { 204 dir = "./" + dir 205 } 206 // TODO: consider not doing these checks here. 207 inst := c.newRelInstance(token.NoPos, dir, pkgName) 208 pkgs := l.importPkg(token.NoPos, inst) 209 for _, p := range pkgs { 210 if err := p.Err; err != nil && (p == nil || len(p.InvalidFiles) == 0) { 211 switch err.(type) { 212 case nil: 213 break 214 case *NoFilesError: 215 if c.DataFiles && len(p.OrphanedFiles) > 0 { 216 break 217 } 218 return nil 219 default: 220 m.Err = errors.Append(m.Err, err) 221 } 222 } 223 } 224 225 m.Pkgs = append(m.Pkgs, pkgs...) 226 return nil 227 }) 228 return m 229} 230 231// treeCanMatchPattern(pattern)(name) reports whether 232// name or children of name can possibly match pattern. 233// Pattern is the same limited glob accepted by matchPattern. 234func treeCanMatchPattern(pattern string) func(name string) bool { 235 wildCard := false 236 if i := strings.Index(pattern, "..."); i >= 0 { 237 wildCard = true 238 pattern = pattern[:i] 239 } 240 return func(name string) bool { 241 return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || 242 wildCard && strings.HasPrefix(name, pattern) 243 } 244} 245 246// matchPattern(pattern)(name) reports whether 247// name matches pattern. Pattern is a limited glob 248// pattern in which '...' means 'any string' and there 249// is no other special syntax. 250// Unfortunately, there are two special cases. Quoting "go help packages": 251// 252// First, /... at the end of the pattern can match an empty string, 253// so that net/... matches both net and packages in its subdirectories, like net/http. 254// Second, any slash-separted pattern element containing a wildcard never 255// participates in a match of the "vendor" element in the path of a vendored 256// package, so that ./... does not match packages in subdirectories of 257// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. 258// Note, however, that a directory named vendor that itself contains code 259// is not a vendored package: cmd/vendor would be a command named vendor, 260// and the pattern cmd/... matches it. 261func matchPattern(pattern string) func(name string) bool { 262 // Convert pattern to regular expression. 263 // The strategy for the trailing /... is to nest it in an explicit ? expression. 264 // The strategy for the vendor exclusion is to change the unmatchable 265 // vendor strings to a disallowed code point (vendorChar) and to use 266 // "(anything but that codepoint)*" as the implementation of the ... wildcard. 267 // This is a bit complicated but the obvious alternative, 268 // namely a hand-written search like in most shell glob matchers, 269 // is too easy to make accidentally exponential. 270 // Using package regexp guarantees linear-time matching. 271 272 const vendorChar = "\x00" 273 274 if strings.Contains(pattern, vendorChar) { 275 return func(name string) bool { return false } 276 } 277 278 re := regexp.QuoteMeta(pattern) 279 re = replaceVendor(re, vendorChar) 280 switch { 281 case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): 282 re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` 283 case re == vendorChar+`/\.\.\.`: 284 re = `(/vendor|/` + vendorChar + `/\.\.\.)` 285 case strings.HasSuffix(re, `/\.\.\.`): 286 re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` 287 } 288 re = strings.Replace(re, `\.\.\.`, `[^`+vendorChar+`]*`, -1) 289 290 reg := regexp.MustCompile(`^` + re + `$`) 291 292 return func(name string) bool { 293 if strings.Contains(name, vendorChar) { 294 return false 295 } 296 return reg.MatchString(replaceVendor(name, vendorChar)) 297 } 298} 299 300// replaceVendor returns the result of replacing 301// non-trailing vendor path elements in x with repl. 302func replaceVendor(x, repl string) string { 303 if !strings.Contains(x, "vendor") { 304 return x 305 } 306 elem := strings.Split(x, "/") 307 for i := 0; i < len(elem)-1; i++ { 308 if elem[i] == "vendor" { 309 elem[i] = repl 310 } 311 } 312 return strings.Join(elem, "/") 313} 314 315// warnUnmatched warns about patterns that didn't match any packages. 316func warnUnmatched(matches []*match) { 317 for _, m := range matches { 318 if len(m.Pkgs) == 0 { 319 m.Err = 320 errors.Newf(token.NoPos, "cue: %q matched no packages\n", m.Pattern) 321 } 322 } 323} 324 325// importPaths returns the matching paths to use for the given command line. 326// It calls ImportPathsQuiet and then WarnUnmatched. 327func (l *loader) importPaths(patterns []string) []*match { 328 matches := l.importPathsQuiet(patterns) 329 warnUnmatched(matches) 330 return matches 331} 332 333// importPathsQuiet is like ImportPaths but does not warn about patterns with no matches. 334func (l *loader) importPathsQuiet(patterns []string) []*match { 335 var out []*match 336 for _, a := range cleanPatterns(patterns) { 337 if isMetaPackage(a) { 338 out = append(out, l.matchPackages(a, l.cfg.Package)) 339 continue 340 } 341 342 orig := a 343 pkgName := l.cfg.Package 344 switch p := strings.IndexByte(a, ':'); { 345 case p < 0: 346 case p == 0: 347 pkgName = a[1:] 348 a = "." 349 default: 350 pkgName = a[p+1:] 351 a = a[:p] 352 } 353 if pkgName == "*" { 354 pkgName = "" 355 } 356 357 if strings.Contains(a, "...") { 358 if isLocalImport(a) { 359 out = append(out, l.matchPackagesInFS(a, pkgName)) 360 } else { 361 out = append(out, l.matchPackages(a, pkgName)) 362 } 363 continue 364 } 365 366 var p *build.Instance 367 if isLocalImport(a) { 368 p = l.cfg.newRelInstance(token.NoPos, a, pkgName) 369 } else { 370 p = l.cfg.newInstance(token.NoPos, importPath(orig)) 371 } 372 373 pkgs := l.importPkg(token.NoPos, p) 374 out = append(out, &match{Pattern: a, Literal: true, Pkgs: pkgs}) 375 } 376 return out 377} 378 379// cleanPatterns returns the patterns to use for the given 380// command line. It canonicalizes the patterns but does not 381// evaluate any matches. 382func cleanPatterns(patterns []string) []string { 383 if len(patterns) == 0 { 384 return []string{"."} 385 } 386 var out []string 387 for _, a := range patterns { 388 // Arguments are supposed to be import paths, but 389 // as a courtesy to Windows developers, rewrite \ to / 390 // in command-line arguments. Handles .\... and so on. 391 if filepath.Separator == '\\' { 392 a = strings.Replace(a, `\`, `/`, -1) 393 } 394 395 // Put argument in canonical form, but preserve leading ./. 396 if strings.HasPrefix(a, "./") { 397 a = "./" + path.Clean(a) 398 if a == "./." { 399 a = "." 400 } 401 } else { 402 a = path.Clean(a) 403 } 404 out = append(out, a) 405 } 406 return out 407} 408 409// isMetaPackage checks if name is a reserved package name that expands to multiple packages. 410func isMetaPackage(name string) bool { 411 return name == "std" || name == "cmd" || name == "all" 412} 413 414// hasPathPrefix reports whether the path s begins with the 415// elements in prefix. 416func hasPathPrefix(s, prefix string) bool { 417 switch { 418 default: 419 return false 420 case len(s) == len(prefix): 421 return s == prefix 422 case len(s) > len(prefix): 423 if prefix != "" && prefix[len(prefix)-1] == '/' { 424 return strings.HasPrefix(s, prefix) 425 } 426 return s[len(prefix)] == '/' && s[:len(prefix)] == prefix 427 } 428} 429 430// hasFilepathPrefix reports whether the path s begins with the 431// elements in prefix. 432func hasFilepathPrefix(s, prefix string) bool { 433 switch { 434 default: 435 return false 436 case len(s) == len(prefix): 437 return s == prefix 438 case len(s) > len(prefix): 439 if prefix != "" && prefix[len(prefix)-1] == filepath.Separator { 440 return strings.HasPrefix(s, prefix) 441 } 442 return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix 443 } 444} 445 446// isStandardImportPath reports whether $GOROOT/src/path should be considered 447// part of the standard distribution. For historical reasons we allow people to add 448// their own code to $GOROOT instead of using $GOPATH, but we assume that 449// code will start with a domain name (dot in the first element). 450// 451// Note that this function is meant to evaluate whether a directory found in GOROOT 452// should be treated as part of the standard library. It should not be used to decide 453// that a directory found in GOPATH should be rejected: directories in GOPATH 454// need not have dots in the first element, and they just take their chances 455// with future collisions in the standard library. 456func isStandardImportPath(path string) bool { 457 i := strings.Index(path, "/") 458 if i < 0 { 459 i = len(path) 460 } 461 elem := path[:i] 462 return !strings.Contains(elem, ".") 463} 464 465// isRelativePath reports whether pattern should be interpreted as a directory 466// path relative to the current directory, as opposed to a pattern matching 467// import paths. 468func isRelativePath(pattern string) bool { 469 return strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == ".." 470} 471 472// inDir checks whether path is in the file tree rooted at dir. 473// If so, inDir returns an equivalent path relative to dir. 474// If not, inDir returns an empty string. 475// inDir makes some effort to succeed even in the presence of symbolic links. 476// TODO(rsc): Replace internal/test.inDir with a call to this function for Go 1.12. 477func inDir(path, dir string) string { 478 if rel := inDirLex(path, dir); rel != "" { 479 return rel 480 } 481 xpath, err := filepath.EvalSymlinks(path) 482 if err != nil || xpath == path { 483 xpath = "" 484 } else { 485 if rel := inDirLex(xpath, dir); rel != "" { 486 return rel 487 } 488 } 489 490 xdir, err := filepath.EvalSymlinks(dir) 491 if err == nil && xdir != dir { 492 if rel := inDirLex(path, xdir); rel != "" { 493 return rel 494 } 495 if xpath != "" { 496 if rel := inDirLex(xpath, xdir); rel != "" { 497 return rel 498 } 499 } 500 } 501 return "" 502} 503 504// inDirLex is like inDir but only checks the lexical form of the file names. 505// It does not consider symbolic links. 506// TODO(rsc): This is a copy of str.HasFilePathPrefix, modified to 507// return the suffix. Most uses of str.HasFilePathPrefix should probably 508// be calling InDir instead. 509func inDirLex(path, dir string) string { 510 pv := strings.ToUpper(filepath.VolumeName(path)) 511 dv := strings.ToUpper(filepath.VolumeName(dir)) 512 path = path[len(pv):] 513 dir = dir[len(dv):] 514 switch { 515 default: 516 return "" 517 case pv != dv: 518 return "" 519 case len(path) == len(dir): 520 if path == dir { 521 return "." 522 } 523 return "" 524 case dir == "": 525 return path 526 case len(path) > len(dir): 527 if dir[len(dir)-1] == filepath.Separator { 528 if path[:len(dir)] == dir { 529 return path[len(dir):] 530 } 531 return "" 532 } 533 if path[len(dir)] == filepath.Separator && path[:len(dir)] == dir { 534 if len(path) == len(dir)+1 { 535 return "." 536 } 537 return path[len(dir)+1:] 538 } 539 return "" 540 } 541} 542