1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package web2 6 7import ( 8 "bytes" 9 "cmd/go/internal/base" 10 "cmd/go/internal/cfg" 11 "encoding/json" 12 "flag" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "log" 17 "net/http" 18 "os" 19 "path/filepath" 20 "runtime" 21 "runtime/debug" 22 "strings" 23 "sync" 24) 25 26var TraceGET = false 27var webstack = false 28 29func init() { 30 flag.BoolVar(&TraceGET, "webtrace", TraceGET, "trace GET requests") 31 flag.BoolVar(&webstack, "webstack", webstack, "print stack for GET requests") 32} 33 34type netrcLine struct { 35 machine string 36 login string 37 password string 38} 39 40var netrcOnce sync.Once 41var netrc []netrcLine 42 43func parseNetrc(data string) []netrcLine { 44 var nrc []netrcLine 45 var l netrcLine 46 for _, line := range strings.Split(data, "\n") { 47 f := strings.Fields(line) 48 for i := 0; i < len(f)-1; i += 2 { 49 switch f[i] { 50 case "machine": 51 l.machine = f[i+1] 52 case "login": 53 l.login = f[i+1] 54 case "password": 55 l.password = f[i+1] 56 } 57 } 58 if l.machine != "" && l.login != "" && l.password != "" { 59 nrc = append(nrc, l) 60 l = netrcLine{} 61 } 62 } 63 return nrc 64} 65 66func havePassword(machine string) bool { 67 netrcOnce.Do(readNetrc) 68 for _, line := range netrc { 69 if line.machine == machine { 70 return true 71 } 72 } 73 return false 74} 75 76func netrcPath() string { 77 switch runtime.GOOS { 78 case "windows": 79 return filepath.Join(os.Getenv("USERPROFILE"), "_netrc") 80 case "plan9": 81 return filepath.Join(os.Getenv("home"), ".netrc") 82 default: 83 return filepath.Join(os.Getenv("HOME"), ".netrc") 84 } 85} 86 87func readNetrc() { 88 data, err := ioutil.ReadFile(netrcPath()) 89 if err != nil { 90 return 91 } 92 netrc = parseNetrc(string(data)) 93} 94 95type getState struct { 96 req *http.Request 97 resp *http.Response 98 body io.ReadCloser 99 non200ok bool 100} 101 102type Option interface { 103 option(*getState) error 104} 105 106func Non200OK() Option { 107 return optionFunc(func(g *getState) error { 108 g.non200ok = true 109 return nil 110 }) 111} 112 113type optionFunc func(*getState) error 114 115func (f optionFunc) option(g *getState) error { 116 return f(g) 117} 118 119func DecodeJSON(dst interface{}) Option { 120 return optionFunc(func(g *getState) error { 121 if g.resp != nil { 122 return json.NewDecoder(g.body).Decode(dst) 123 } 124 return nil 125 }) 126} 127 128func ReadAllBody(body *[]byte) Option { 129 return optionFunc(func(g *getState) error { 130 if g.resp != nil { 131 var err error 132 *body, err = ioutil.ReadAll(g.body) 133 return err 134 } 135 return nil 136 }) 137} 138 139func Body(body *io.ReadCloser) Option { 140 return optionFunc(func(g *getState) error { 141 if g.resp != nil { 142 *body = g.body 143 g.body = nil 144 } 145 return nil 146 }) 147} 148 149func Header(hdr *http.Header) Option { 150 return optionFunc(func(g *getState) error { 151 if g.resp != nil { 152 *hdr = CopyHeader(g.resp.Header) 153 } 154 return nil 155 }) 156} 157 158func CopyHeader(hdr http.Header) http.Header { 159 if hdr == nil { 160 return nil 161 } 162 h2 := make(http.Header) 163 for k, v := range hdr { 164 v2 := make([]string, len(v)) 165 copy(v2, v) 166 h2[k] = v2 167 } 168 return h2 169} 170 171var cache struct { 172 mu sync.Mutex 173 byURL map[string]*cacheEntry 174} 175 176type cacheEntry struct { 177 mu sync.Mutex 178 resp *http.Response 179 body []byte 180} 181 182var httpDo = http.DefaultClient.Do 183 184func SetHTTPDoForTesting(do func(*http.Request) (*http.Response, error)) { 185 if do == nil { 186 do = http.DefaultClient.Do 187 } 188 httpDo = do 189} 190 191func Get(url string, options ...Option) error { 192 if TraceGET || webstack || cfg.BuildV { 193 log.Printf("Fetching %s", url) 194 if webstack { 195 log.Println(string(debug.Stack())) 196 } 197 } 198 199 req, err := http.NewRequest("GET", url, nil) 200 if err != nil { 201 return err 202 } 203 204 netrcOnce.Do(readNetrc) 205 for _, l := range netrc { 206 if l.machine == req.URL.Host { 207 req.SetBasicAuth(l.login, l.password) 208 break 209 } 210 } 211 212 g := &getState{req: req} 213 for _, o := range options { 214 if err := o.option(g); err != nil { 215 return err 216 } 217 } 218 219 cache.mu.Lock() 220 e := cache.byURL[url] 221 if e == nil { 222 e = new(cacheEntry) 223 if !strings.HasPrefix(url, "file:") { 224 if cache.byURL == nil { 225 cache.byURL = make(map[string]*cacheEntry) 226 } 227 cache.byURL[url] = e 228 } 229 } 230 cache.mu.Unlock() 231 232 e.mu.Lock() 233 if strings.HasPrefix(url, "file:") { 234 body, err := ioutil.ReadFile(req.URL.Path) 235 if err != nil { 236 e.mu.Unlock() 237 return err 238 } 239 e.body = body 240 e.resp = &http.Response{ 241 StatusCode: 200, 242 } 243 } else if e.resp == nil { 244 resp, err := httpDo(req) 245 if err != nil { 246 e.mu.Unlock() 247 return err 248 } 249 e.resp = resp 250 // TODO: Spool to temp file. 251 body, err := ioutil.ReadAll(resp.Body) 252 resp.Body.Close() 253 resp.Body = nil 254 if err != nil { 255 e.mu.Unlock() 256 return err 257 } 258 e.body = body 259 } 260 g.resp = e.resp 261 g.body = ioutil.NopCloser(bytes.NewReader(e.body)) 262 e.mu.Unlock() 263 264 defer func() { 265 if g.body != nil { 266 g.body.Close() 267 } 268 }() 269 270 if g.resp.StatusCode == 403 && req.URL.Host == "api.github.com" && !havePassword("api.github.com") { 271 base.Errorf("%s", githubMessage) 272 } 273 if !g.non200ok && g.resp.StatusCode != 200 { 274 return fmt.Errorf("unexpected status (%s): %v", url, g.resp.Status) 275 } 276 277 for _, o := range options { 278 if err := o.option(g); err != nil { 279 return err 280 } 281 } 282 return err 283} 284 285var githubMessage = `go: 403 response from api.github.com 286 287GitHub applies fairly small rate limits to unauthenticated users, and 288you appear to be hitting them. To authenticate, please visit 289https://github.com/settings/tokens and click "Generate New Token" to 290create a Personal Access Token. The token only needs "public_repo" 291scope, but you can add "repo" if you want to access private 292repositories too. 293 294Add the token to your $HOME/.netrc (%USERPROFILE%\_netrc on Windows): 295 296 machine api.github.com login YOU password TOKEN 297 298Sorry for the interruption. 299` 300