1// Package fsys is an abstraction for reading files that 2// allows for virtual overlays on top of the files on disk. 3package fsys 4 5import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io/fs" 10 "io/ioutil" 11 "os" 12 "path/filepath" 13 "runtime" 14 "sort" 15 "strings" 16 "time" 17) 18 19// OverlayFile is the path to a text file in the OverlayJSON format. 20// It is the value of the -overlay flag. 21var OverlayFile string 22 23// OverlayJSON is the format overlay files are expected to be in. 24// The Replace map maps from overlaid paths to replacement paths: 25// the Go command will forward all reads trying to open 26// each overlaid path to its replacement path, or consider the overlaid 27// path not to exist if the replacement path is empty. 28type OverlayJSON struct { 29 Replace map[string]string 30} 31 32type node struct { 33 actualFilePath string // empty if a directory 34 children map[string]*node // path element → file or directory 35} 36 37func (n *node) isDir() bool { 38 return n.actualFilePath == "" && n.children != nil 39} 40 41func (n *node) isDeleted() bool { 42 return n.actualFilePath == "" && n.children == nil 43} 44 45// TODO(matloob): encapsulate these in an io/fs-like interface 46var overlay map[string]*node // path -> file or directory node 47var cwd string // copy of base.Cwd() to avoid dependency 48 49// Canonicalize a path for looking it up in the overlay. 50// Important: filepath.Join(cwd, path) doesn't always produce 51// the correct absolute path if path is relative, because on 52// Windows producing the correct absolute path requires making 53// a syscall. So this should only be used when looking up paths 54// in the overlay, or canonicalizing the paths in the overlay. 55func canonicalize(path string) string { 56 if path == "" { 57 return "" 58 } 59 if filepath.IsAbs(path) { 60 return filepath.Clean(path) 61 } 62 63 if v := filepath.VolumeName(cwd); v != "" && path[0] == filepath.Separator { 64 // On Windows filepath.Join(cwd, path) doesn't always work. In general 65 // filepath.Abs needs to make a syscall on Windows. Elsewhere in cmd/go 66 // use filepath.Join(cwd, path), but cmd/go specifically supports Windows 67 // paths that start with "\" which implies the path is relative to the 68 // volume of the working directory. See golang.org/issue/8130. 69 return filepath.Join(v, path) 70 } 71 72 // Make the path absolute. 73 return filepath.Join(cwd, path) 74} 75 76// Init initializes the overlay, if one is being used. 77func Init(wd string) error { 78 if overlay != nil { 79 // already initialized 80 return nil 81 } 82 83 cwd = wd 84 85 if OverlayFile == "" { 86 return nil 87 } 88 89 b, err := os.ReadFile(OverlayFile) 90 if err != nil { 91 return fmt.Errorf("reading overlay file: %v", err) 92 } 93 94 var overlayJSON OverlayJSON 95 if err := json.Unmarshal(b, &overlayJSON); err != nil { 96 return fmt.Errorf("parsing overlay JSON: %v", err) 97 } 98 99 return initFromJSON(overlayJSON) 100} 101 102func initFromJSON(overlayJSON OverlayJSON) error { 103 // Canonicalize the paths in the overlay map. 104 // Use reverseCanonicalized to check for collisions: 105 // no two 'from' paths should canonicalize to the same path. 106 overlay = make(map[string]*node) 107 reverseCanonicalized := make(map[string]string) // inverse of canonicalize operation, to check for duplicates 108 // Build a table of file and directory nodes from the replacement map. 109 110 // Remove any potential non-determinism from iterating over map by sorting it. 111 replaceFrom := make([]string, 0, len(overlayJSON.Replace)) 112 for k := range overlayJSON.Replace { 113 replaceFrom = append(replaceFrom, k) 114 } 115 sort.Strings(replaceFrom) 116 117 for _, from := range replaceFrom { 118 to := overlayJSON.Replace[from] 119 // Canonicalize paths and check for a collision. 120 if from == "" { 121 return fmt.Errorf("empty string key in overlay file Replace map") 122 } 123 cfrom := canonicalize(from) 124 if to != "" { 125 // Don't canonicalize "", meaning to delete a file, because then it will turn into ".". 126 to = canonicalize(to) 127 } 128 if otherFrom, seen := reverseCanonicalized[cfrom]; seen { 129 return fmt.Errorf( 130 "paths %q and %q both canonicalize to %q in overlay file Replace map", otherFrom, from, cfrom) 131 } 132 reverseCanonicalized[cfrom] = from 133 from = cfrom 134 135 // Create node for overlaid file. 136 dir, base := filepath.Dir(from), filepath.Base(from) 137 if n, ok := overlay[from]; ok { 138 // All 'from' paths in the overlay are file paths. Since the from paths 139 // are in a map, they are unique, so if the node already exists we added 140 // it below when we create parent directory nodes. That is, that 141 // both a file and a path to one of its parent directories exist as keys 142 // in the Replace map. 143 // 144 // This only applies if the overlay directory has any files or directories 145 // in it: placeholder directories that only contain deleted files don't 146 // count. They are safe to be overwritten with actual files. 147 for _, f := range n.children { 148 if !f.isDeleted() { 149 return fmt.Errorf("invalid overlay: path %v is used as both file and directory", from) 150 } 151 } 152 } 153 overlay[from] = &node{actualFilePath: to} 154 155 // Add parent directory nodes to overlay structure. 156 childNode := overlay[from] 157 for { 158 dirNode := overlay[dir] 159 if dirNode == nil || dirNode.isDeleted() { 160 dirNode = &node{children: make(map[string]*node)} 161 overlay[dir] = dirNode 162 } 163 if childNode.isDeleted() { 164 // Only create one parent for a deleted file: 165 // the directory only conditionally exists if 166 // there are any non-deleted children, so 167 // we don't create their parents. 168 if dirNode.isDir() { 169 dirNode.children[base] = childNode 170 } 171 break 172 } 173 if !dirNode.isDir() { 174 // This path already exists as a file, so it can't be a parent 175 // directory. See comment at error above. 176 return fmt.Errorf("invalid overlay: path %v is used as both file and directory", dir) 177 } 178 dirNode.children[base] = childNode 179 parent := filepath.Dir(dir) 180 if parent == dir { 181 break // reached the top; there is no parent 182 } 183 dir, base = parent, filepath.Base(dir) 184 childNode = dirNode 185 } 186 } 187 188 return nil 189} 190 191// IsDir returns true if path is a directory on disk or in the 192// overlay. 193func IsDir(path string) (bool, error) { 194 path = canonicalize(path) 195 196 if _, ok := parentIsOverlayFile(path); ok { 197 return false, nil 198 } 199 200 if n, ok := overlay[path]; ok { 201 return n.isDir(), nil 202 } 203 204 fi, err := os.Stat(path) 205 if err != nil { 206 return false, err 207 } 208 209 return fi.IsDir(), nil 210} 211 212// parentIsOverlayFile returns whether name or any of 213// its parents are files in the overlay, and the first parent found, 214// including name itself, that's a file in the overlay. 215func parentIsOverlayFile(name string) (string, bool) { 216 if overlay != nil { 217 // Check if name can't possibly be a directory because 218 // it or one of its parents is overlaid with a file. 219 // TODO(matloob): Maybe save this to avoid doing it every time? 220 prefix := name 221 for { 222 node := overlay[prefix] 223 if node != nil && !node.isDir() { 224 return prefix, true 225 } 226 parent := filepath.Dir(prefix) 227 if parent == prefix { 228 break 229 } 230 prefix = parent 231 } 232 } 233 234 return "", false 235} 236 237// errNotDir is used to communicate from ReadDir to IsDirWithGoFiles 238// that the argument is not a directory, so that IsDirWithGoFiles doesn't 239// return an error. 240var errNotDir = errors.New("not a directory") 241 242// readDir reads a dir on disk, returning an error that is errNotDir if the dir is not a directory. 243// Unfortunately, the error returned by ioutil.ReadDir if dir is not a directory 244// can vary depending on the OS (Linux, Mac, Windows return ENOTDIR; BSD returns EINVAL). 245func readDir(dir string) ([]fs.FileInfo, error) { 246 fis, err := ioutil.ReadDir(dir) 247 if err == nil { 248 return fis, nil 249 } 250 251 if os.IsNotExist(err) { 252 return nil, err 253 } 254 if dirfi, staterr := os.Stat(dir); staterr == nil && !dirfi.IsDir() { 255 return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} 256 } 257 return nil, err 258} 259 260// ReadDir provides a slice of fs.FileInfo entries corresponding 261// to the overlaid files in the directory. 262func ReadDir(dir string) ([]fs.FileInfo, error) { 263 dir = canonicalize(dir) 264 if _, ok := parentIsOverlayFile(dir); ok { 265 return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: errNotDir} 266 } 267 268 dirNode := overlay[dir] 269 if dirNode == nil { 270 return readDir(dir) 271 } 272 if dirNode.isDeleted() { 273 return nil, &fs.PathError{Op: "ReadDir", Path: dir, Err: fs.ErrNotExist} 274 } 275 diskfis, err := readDir(dir) 276 if err != nil && !os.IsNotExist(err) && !errors.Is(err, errNotDir) { 277 return nil, err 278 } 279 280 // Stat files in overlay to make composite list of fileinfos 281 files := make(map[string]fs.FileInfo) 282 for _, f := range diskfis { 283 files[f.Name()] = f 284 } 285 for name, to := range dirNode.children { 286 switch { 287 case to.isDir(): 288 files[name] = fakeDir(name) 289 case to.isDeleted(): 290 delete(files, name) 291 default: 292 // This is a regular file. 293 f, err := os.Lstat(to.actualFilePath) 294 if err != nil { 295 files[name] = missingFile(name) 296 continue 297 } else if f.IsDir() { 298 return nil, fmt.Errorf("for overlay of %q to %q: overlay Replace entries can't point to dirctories", 299 filepath.Join(dir, name), to.actualFilePath) 300 } 301 // Add a fileinfo for the overlaid file, so that it has 302 // the original file's name, but the overlaid file's metadata. 303 files[name] = fakeFile{name, f} 304 } 305 } 306 sortedFiles := diskfis[:0] 307 for _, f := range files { 308 sortedFiles = append(sortedFiles, f) 309 } 310 sort.Slice(sortedFiles, func(i, j int) bool { return sortedFiles[i].Name() < sortedFiles[j].Name() }) 311 return sortedFiles, nil 312} 313 314// OverlayPath returns the path to the overlaid contents of the 315// file, the empty string if the overlay deletes the file, or path 316// itself if the file is not in the overlay, the file is a directory 317// in the overlay, or there is no overlay. 318// It returns true if the path is overlaid with a regular file 319// or deleted, and false otherwise. 320func OverlayPath(path string) (string, bool) { 321 if p, ok := overlay[canonicalize(path)]; ok && !p.isDir() { 322 return p.actualFilePath, ok 323 } 324 325 return path, false 326} 327 328// Open opens the file at or overlaid on the given path. 329func Open(path string) (*os.File, error) { 330 return OpenFile(path, os.O_RDONLY, 0) 331} 332 333// OpenFile opens the file at or overlaid on the given path with the flag and perm. 334func OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { 335 cpath := canonicalize(path) 336 if node, ok := overlay[cpath]; ok { 337 // Opening a file in the overlay. 338 if node.isDir() { 339 return nil, &fs.PathError{Op: "OpenFile", Path: path, Err: errors.New("fsys.OpenFile doesn't support opening directories yet")} 340 } 341 // We can't open overlaid paths for write. 342 if perm != os.FileMode(os.O_RDONLY) { 343 return nil, &fs.PathError{Op: "OpenFile", Path: path, Err: errors.New("overlaid files can't be opened for write")} 344 } 345 return os.OpenFile(node.actualFilePath, flag, perm) 346 } 347 if parent, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { 348 // The file is deleted explicitly in the Replace map, 349 // or implicitly because one of its parent directories was 350 // replaced by a file. 351 return nil, &fs.PathError{ 352 Op: "Open", 353 Path: path, 354 Err: fmt.Errorf("file %s does not exist: parent directory %s is replaced by a file in overlay", path, parent), 355 } 356 } 357 return os.OpenFile(cpath, flag, perm) 358} 359 360// IsDirWithGoFiles reports whether dir is a directory containing Go files 361// either on disk or in the overlay. 362func IsDirWithGoFiles(dir string) (bool, error) { 363 fis, err := ReadDir(dir) 364 if os.IsNotExist(err) || errors.Is(err, errNotDir) { 365 return false, nil 366 } 367 if err != nil { 368 return false, err 369 } 370 371 var firstErr error 372 for _, fi := range fis { 373 if fi.IsDir() { 374 continue 375 } 376 377 // TODO(matloob): this enforces that the "from" in the map 378 // has a .go suffix, but the actual destination file 379 // doesn't need to have a .go suffix. Is this okay with the 380 // compiler? 381 if !strings.HasSuffix(fi.Name(), ".go") { 382 continue 383 } 384 if fi.Mode().IsRegular() { 385 return true, nil 386 } 387 388 // fi is the result of an Lstat, so it doesn't follow symlinks. 389 // But it's okay if the file is a symlink pointing to a regular 390 // file, so use os.Stat to follow symlinks and check that. 391 actualFilePath, _ := OverlayPath(filepath.Join(dir, fi.Name())) 392 fi, err := os.Stat(actualFilePath) 393 if err == nil && fi.Mode().IsRegular() { 394 return true, nil 395 } 396 if err != nil && firstErr == nil { 397 firstErr = err 398 } 399 } 400 401 // No go files found in directory. 402 return false, firstErr 403} 404 405// walk recursively descends path, calling walkFn. Copied, with some 406// modifications from path/filepath.walk. 407func walk(path string, info fs.FileInfo, walkFn filepath.WalkFunc) error { 408 if !info.IsDir() { 409 return walkFn(path, info, nil) 410 } 411 412 fis, readErr := ReadDir(path) 413 walkErr := walkFn(path, info, readErr) 414 // If readErr != nil, walk can't walk into this directory. 415 // walkErr != nil means walkFn want walk to skip this directory or stop walking. 416 // Therefore, if one of readErr and walkErr isn't nil, walk will return. 417 if readErr != nil || walkErr != nil { 418 // The caller's behavior is controlled by the return value, which is decided 419 // by walkFn. walkFn may ignore readErr and return nil. 420 // If walkFn returns SkipDir, it will be handled by the caller. 421 // So walk should return whatever walkFn returns. 422 return walkErr 423 } 424 425 for _, fi := range fis { 426 filename := filepath.Join(path, fi.Name()) 427 if walkErr = walk(filename, fi, walkFn); walkErr != nil { 428 if !fi.IsDir() || walkErr != filepath.SkipDir { 429 return walkErr 430 } 431 } 432 } 433 return nil 434} 435 436// Walk walks the file tree rooted at root, calling walkFn for each file or 437// directory in the tree, including root. 438func Walk(root string, walkFn filepath.WalkFunc) error { 439 info, err := Lstat(root) 440 if err != nil { 441 err = walkFn(root, nil, err) 442 } else { 443 err = walk(root, info, walkFn) 444 } 445 if err == filepath.SkipDir { 446 return nil 447 } 448 return err 449} 450 451// lstat implements a version of os.Lstat that operates on the overlay filesystem. 452func Lstat(path string) (fs.FileInfo, error) { 453 return overlayStat(path, os.Lstat, "lstat") 454} 455 456// Stat implements a version of os.Stat that operates on the overlay filesystem. 457func Stat(path string) (fs.FileInfo, error) { 458 return overlayStat(path, os.Stat, "stat") 459} 460 461// overlayStat implements lstat or Stat (depending on whether os.Lstat or os.Stat is passed in). 462func overlayStat(path string, osStat func(string) (fs.FileInfo, error), opName string) (fs.FileInfo, error) { 463 cpath := canonicalize(path) 464 465 if _, ok := parentIsOverlayFile(filepath.Dir(cpath)); ok { 466 return nil, &fs.PathError{Op: opName, Path: cpath, Err: fs.ErrNotExist} 467 } 468 469 node, ok := overlay[cpath] 470 if !ok { 471 // The file or directory is not overlaid. 472 return osStat(path) 473 } 474 475 switch { 476 case node.isDeleted(): 477 return nil, &fs.PathError{Op: "lstat", Path: cpath, Err: fs.ErrNotExist} 478 case node.isDir(): 479 return fakeDir(filepath.Base(path)), nil 480 default: 481 fi, err := osStat(node.actualFilePath) 482 if err != nil { 483 return nil, err 484 } 485 return fakeFile{name: filepath.Base(path), real: fi}, nil 486 } 487} 488 489// fakeFile provides an fs.FileInfo implementation for an overlaid file, 490// so that the file has the name of the overlaid file, but takes all 491// other characteristics of the replacement file. 492type fakeFile struct { 493 name string 494 real fs.FileInfo 495} 496 497func (f fakeFile) Name() string { return f.name } 498func (f fakeFile) Size() int64 { return f.real.Size() } 499func (f fakeFile) Mode() fs.FileMode { return f.real.Mode() } 500func (f fakeFile) ModTime() time.Time { return f.real.ModTime() } 501func (f fakeFile) IsDir() bool { return f.real.IsDir() } 502func (f fakeFile) Sys() any { return f.real.Sys() } 503 504// missingFile provides an fs.FileInfo for an overlaid file where the 505// destination file in the overlay doesn't exist. It returns zero values 506// for the fileInfo methods other than Name, set to the file's name, and Mode 507// set to ModeIrregular. 508type missingFile string 509 510func (f missingFile) Name() string { return string(f) } 511func (f missingFile) Size() int64 { return 0 } 512func (f missingFile) Mode() fs.FileMode { return fs.ModeIrregular } 513func (f missingFile) ModTime() time.Time { return time.Unix(0, 0) } 514func (f missingFile) IsDir() bool { return false } 515func (f missingFile) Sys() any { return nil } 516 517// fakeDir provides an fs.FileInfo implementation for directories that are 518// implicitly created by overlaid files. Each directory in the 519// path of an overlaid file is considered to exist in the overlay filesystem. 520type fakeDir string 521 522func (f fakeDir) Name() string { return string(f) } 523func (f fakeDir) Size() int64 { return 0 } 524func (f fakeDir) Mode() fs.FileMode { return fs.ModeDir | 0500 } 525func (f fakeDir) ModTime() time.Time { return time.Unix(0, 0) } 526func (f fakeDir) IsDir() bool { return true } 527func (f fakeDir) Sys() any { return nil } 528 529// Glob is like filepath.Glob but uses the overlay file system. 530func Glob(pattern string) (matches []string, err error) { 531 // Check pattern is well-formed. 532 if _, err := filepath.Match(pattern, ""); err != nil { 533 return nil, err 534 } 535 if !hasMeta(pattern) { 536 if _, err = Lstat(pattern); err != nil { 537 return nil, nil 538 } 539 return []string{pattern}, nil 540 } 541 542 dir, file := filepath.Split(pattern) 543 volumeLen := 0 544 if runtime.GOOS == "windows" { 545 volumeLen, dir = cleanGlobPathWindows(dir) 546 } else { 547 dir = cleanGlobPath(dir) 548 } 549 550 if !hasMeta(dir[volumeLen:]) { 551 return glob(dir, file, nil) 552 } 553 554 // Prevent infinite recursion. See issue 15879. 555 if dir == pattern { 556 return nil, filepath.ErrBadPattern 557 } 558 559 var m []string 560 m, err = Glob(dir) 561 if err != nil { 562 return 563 } 564 for _, d := range m { 565 matches, err = glob(d, file, matches) 566 if err != nil { 567 return 568 } 569 } 570 return 571} 572 573// cleanGlobPath prepares path for glob matching. 574func cleanGlobPath(path string) string { 575 switch path { 576 case "": 577 return "." 578 case string(filepath.Separator): 579 // do nothing to the path 580 return path 581 default: 582 return path[0 : len(path)-1] // chop off trailing separator 583 } 584} 585 586func volumeNameLen(path string) int { 587 isSlash := func(c uint8) bool { 588 return c == '\\' || c == '/' 589 } 590 if len(path) < 2 { 591 return 0 592 } 593 // with drive letter 594 c := path[0] 595 if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { 596 return 2 597 } 598 // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx 599 if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && 600 !isSlash(path[2]) && path[2] != '.' { 601 // first, leading `\\` and next shouldn't be `\`. its server name. 602 for n := 3; n < l-1; n++ { 603 // second, next '\' shouldn't be repeated. 604 if isSlash(path[n]) { 605 n++ 606 // third, following something characters. its share name. 607 if !isSlash(path[n]) { 608 if path[n] == '.' { 609 break 610 } 611 for ; n < l; n++ { 612 if isSlash(path[n]) { 613 break 614 } 615 } 616 return n 617 } 618 break 619 } 620 } 621 } 622 return 0 623} 624 625// cleanGlobPathWindows is windows version of cleanGlobPath. 626func cleanGlobPathWindows(path string) (prefixLen int, cleaned string) { 627 vollen := volumeNameLen(path) 628 switch { 629 case path == "": 630 return 0, "." 631 case vollen+1 == len(path) && os.IsPathSeparator(path[len(path)-1]): // /, \, C:\ and C:/ 632 // do nothing to the path 633 return vollen + 1, path 634 case vollen == len(path) && len(path) == 2: // C: 635 return vollen, path + "." // convert C: into C:. 636 default: 637 if vollen >= len(path) { 638 vollen = len(path) - 1 639 } 640 return vollen, path[0 : len(path)-1] // chop off trailing separator 641 } 642} 643 644// glob searches for files matching pattern in the directory dir 645// and appends them to matches. If the directory cannot be 646// opened, it returns the existing matches. New matches are 647// added in lexicographical order. 648func glob(dir, pattern string, matches []string) (m []string, e error) { 649 m = matches 650 fi, err := Stat(dir) 651 if err != nil { 652 return // ignore I/O error 653 } 654 if !fi.IsDir() { 655 return // ignore I/O error 656 } 657 658 list, err := ReadDir(dir) 659 if err != nil { 660 return // ignore I/O error 661 } 662 663 var names []string 664 for _, info := range list { 665 names = append(names, info.Name()) 666 } 667 sort.Strings(names) 668 669 for _, n := range names { 670 matched, err := filepath.Match(pattern, n) 671 if err != nil { 672 return m, err 673 } 674 if matched { 675 m = append(m, filepath.Join(dir, n)) 676 } 677 } 678 return 679} 680 681// hasMeta reports whether path contains any of the magic characters 682// recognized by filepath.Match. 683func hasMeta(path string) bool { 684 magicChars := `*?[` 685 if runtime.GOOS != "windows" { 686 magicChars = `*?[\` 687 } 688 return strings.ContainsAny(path, magicChars) 689} 690