1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Package codehost defines the interface implemented by a code hosting source, 6// along with support code for use by implementations. 7package codehost 8 9import ( 10 "bytes" 11 "crypto/sha256" 12 "fmt" 13 "io" 14 "io/ioutil" 15 "os" 16 "os/exec" 17 "path/filepath" 18 "strings" 19 "sync" 20 "time" 21 22 "cmd/go/internal/cfg" 23 "cmd/go/internal/lockedfile" 24 "cmd/go/internal/str" 25) 26 27// Downloaded size limits. 28const ( 29 MaxGoMod = 16 << 20 // maximum size of go.mod file 30 MaxLICENSE = 16 << 20 // maximum size of LICENSE file 31 MaxZipFile = 500 << 20 // maximum size of downloaded zip file 32) 33 34// A Repo represents a code hosting source. 35// Typical implementations include local version control repositories, 36// remote version control servers, and code hosting sites. 37// A Repo must be safe for simultaneous use by multiple goroutines. 38type Repo interface { 39 // List lists all tags with the given prefix. 40 Tags(prefix string) (tags []string, err error) 41 42 // Stat returns information about the revision rev. 43 // A revision can be any identifier known to the underlying service: 44 // commit hash, branch, tag, and so on. 45 Stat(rev string) (*RevInfo, error) 46 47 // Latest returns the latest revision on the default branch, 48 // whatever that means in the underlying implementation. 49 Latest() (*RevInfo, error) 50 51 // ReadFile reads the given file in the file tree corresponding to revision rev. 52 // It should refuse to read more than maxSize bytes. 53 // 54 // If the requested file does not exist it should return an error for which 55 // os.IsNotExist(err) returns true. 56 ReadFile(rev, file string, maxSize int64) (data []byte, err error) 57 58 // ReadFileRevs reads a single file at multiple versions. 59 // It should refuse to read more than maxSize bytes. 60 // The result is a map from each requested rev strings 61 // to the associated FileRev. The map must have a non-nil 62 // entry for every requested rev (unless ReadFileRevs returned an error). 63 // A file simply being missing or even corrupted in revs[i] 64 // should be reported only in files[revs[i]].Err, not in the error result 65 // from ReadFileRevs. 66 // The overall call should return an error (and no map) only 67 // in the case of a problem with obtaining the data, such as 68 // a network failure. 69 // Implementations may assume that revs only contain tags, 70 // not direct commit hashes. 71 ReadFileRevs(revs []string, file string, maxSize int64) (files map[string]*FileRev, err error) 72 73 // ReadZip downloads a zip file for the subdir subdirectory 74 // of the given revision to a new file in a given temporary directory. 75 // It should refuse to read more than maxSize bytes. 76 // It returns a ReadCloser for a streamed copy of the zip file, 77 // along with the actual subdirectory (possibly shorter than subdir) 78 // contained in the zip file. All files in the zip file are expected to be 79 // nested in a single top-level directory, whose name is not specified. 80 ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) 81 82 // RecentTag returns the most recent tag at or before the given rev 83 // with the given prefix. It should make a best-effort attempt to 84 // find a tag that is a valid semantic version (following the prefix), 85 // or else the result is not useful to the caller, but it need not 86 // incur great expense in doing so. For example, the git implementation 87 // of RecentTag limits git's search to tags matching the glob expression 88 // "v[0-9]*.[0-9]*.[0-9]*" (after the prefix). 89 RecentTag(rev, prefix string) (tag string, err error) 90} 91 92// A Rev describes a single revision in a source code repository. 93type RevInfo struct { 94 Name string // complete ID in underlying repository 95 Short string // shortened ID, for use in pseudo-version 96 Version string // version used in lookup 97 Time time.Time // commit time 98 Tags []string // known tags for commit 99} 100 101// A FileRev describes the result of reading a file at a given revision. 102type FileRev struct { 103 Rev string // requested revision 104 Data []byte // file data 105 Err error // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev 106} 107 108// AllHex reports whether the revision rev is entirely lower-case hexadecimal digits. 109func AllHex(rev string) bool { 110 for i := 0; i < len(rev); i++ { 111 c := rev[i] 112 if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' { 113 continue 114 } 115 return false 116 } 117 return true 118} 119 120// ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length 121// used in pseudo-versions (12 hex digits). 122func ShortenSHA1(rev string) string { 123 if AllHex(rev) && len(rev) == 40 { 124 return rev[:12] 125 } 126 return rev 127} 128 129// WorkRoot is the root of the cached work directory. 130// It is set by cmd/go/internal/modload.InitMod. 131var WorkRoot string 132 133// WorkDir returns the name of the cached work directory to use for the 134// given repository type and name. 135func WorkDir(typ, name string) (dir, lockfile string, err error) { 136 if WorkRoot == "" { 137 return "", "", fmt.Errorf("codehost.WorkRoot not set") 138 } 139 140 // We name the work directory for the SHA256 hash of the type and name. 141 // We intentionally avoid the actual name both because of possible 142 // conflicts with valid file system paths and because we want to ensure 143 // that one checkout is never nested inside another. That nesting has 144 // led to security problems in the past. 145 if strings.Contains(typ, ":") { 146 return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon") 147 } 148 key := typ + ":" + name 149 dir = filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key)))) 150 151 if cfg.BuildX { 152 fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name) 153 } 154 if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil { 155 return "", "", err 156 } 157 158 lockfile = dir + ".lock" 159 if cfg.BuildX { 160 fmt.Fprintf(os.Stderr, "# lock %s", lockfile) 161 } 162 163 unlock, err := lockedfile.MutexAt(lockfile).Lock() 164 if err != nil { 165 return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err) 166 } 167 defer unlock() 168 169 data, err := ioutil.ReadFile(dir + ".info") 170 info, err2 := os.Stat(dir) 171 if err == nil && err2 == nil && info.IsDir() { 172 // Info file and directory both already exist: reuse. 173 have := strings.TrimSuffix(string(data), "\n") 174 if have != key { 175 return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key) 176 } 177 if cfg.BuildX { 178 fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name) 179 } 180 return dir, lockfile, nil 181 } 182 183 // Info file or directory missing. Start from scratch. 184 if cfg.BuildX { 185 fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name) 186 } 187 os.RemoveAll(dir) 188 if err := os.MkdirAll(dir, 0777); err != nil { 189 return "", "", err 190 } 191 if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil { 192 os.RemoveAll(dir) 193 return "", "", err 194 } 195 return dir, lockfile, nil 196} 197 198type RunError struct { 199 Cmd string 200 Err error 201 Stderr []byte 202 HelpText string 203} 204 205func (e *RunError) Error() string { 206 text := e.Cmd + ": " + e.Err.Error() 207 stderr := bytes.TrimRight(e.Stderr, "\n") 208 if len(stderr) > 0 { 209 text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t") 210 } 211 if len(e.HelpText) > 0 { 212 text += "\n" + e.HelpText 213 } 214 return text 215} 216 217var dirLock sync.Map 218 219// Run runs the command line in the given directory 220// (an empty dir means the current directory). 221// It returns the standard output and, for a non-zero exit, 222// a *RunError indicating the command, exit status, and standard error. 223// Standard error is unavailable for commands that exit successfully. 224func Run(dir string, cmdline ...interface{}) ([]byte, error) { 225 return RunWithStdin(dir, nil, cmdline...) 226} 227 228// bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell. 229// See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html. 230var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`) 231 232func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte, error) { 233 if dir != "" { 234 muIface, ok := dirLock.Load(dir) 235 if !ok { 236 muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex)) 237 } 238 mu := muIface.(*sync.Mutex) 239 mu.Lock() 240 defer mu.Unlock() 241 } 242 243 cmd := str.StringList(cmdline...) 244 if cfg.BuildX { 245 text := new(strings.Builder) 246 if dir != "" { 247 text.WriteString("cd ") 248 text.WriteString(dir) 249 text.WriteString("; ") 250 } 251 for i, arg := range cmd { 252 if i > 0 { 253 text.WriteByte(' ') 254 } 255 switch { 256 case strings.ContainsAny(arg, "'"): 257 // Quote args that could be mistaken for quoted args. 258 text.WriteByte('"') 259 text.WriteString(bashQuoter.Replace(arg)) 260 text.WriteByte('"') 261 case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"): 262 // Quote args that contain special characters, glob patterns, or spaces. 263 text.WriteByte('\'') 264 text.WriteString(arg) 265 text.WriteByte('\'') 266 default: 267 text.WriteString(arg) 268 } 269 } 270 fmt.Fprintf(os.Stderr, "%s\n", text) 271 start := time.Now() 272 defer func() { 273 fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text) 274 }() 275 } 276 // TODO: Impose limits on command output size. 277 // TODO: Set environment to get English error messages. 278 var stderr bytes.Buffer 279 var stdout bytes.Buffer 280 c := exec.Command(cmd[0], cmd[1:]...) 281 c.Dir = dir 282 c.Stdin = stdin 283 c.Stderr = &stderr 284 c.Stdout = &stdout 285 err := c.Run() 286 if err != nil { 287 err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err} 288 } 289 return stdout.Bytes(), err 290} 291