1// Copyright 2017 The Go Authors. All rights reserved. 2// 3// Use of this source code is governed by a BSD-style 4// license that can be found in the LICENSE file or at 5// https://developers.google.com/open-source/licenses/bsd. 6 7// Command gddo-server is the GoPkgDoc server. 8package main 9 10import ( 11 "bytes" 12 "context" 13 "crypto/md5" 14 "encoding/json" 15 "errors" 16 "fmt" 17 "go/build" 18 "html/template" 19 "io" 20 "log" 21 "net/http" 22 "net/url" 23 "os" 24 "path" 25 "regexp" 26 "runtime/debug" 27 "sort" 28 "strconv" 29 "strings" 30 "time" 31 32 "cloud.google.com/go/logging" 33 "cloud.google.com/go/pubsub" 34 "cloud.google.com/go/trace" 35 "github.com/spf13/viper" 36 37 "github.com/golang/gddo/database" 38 "github.com/golang/gddo/doc" 39 "github.com/golang/gddo/gosrc" 40 "github.com/golang/gddo/httputil" 41 "github.com/golang/gddo/internal/health" 42) 43 44const ( 45 jsonMIMEType = "application/json; charset=utf-8" 46 textMIMEType = "text/plain; charset=utf-8" 47 htmlMIMEType = "text/html; charset=utf-8" 48) 49 50var errUpdateTimeout = errors.New("refresh timeout") 51 52type httpError struct { 53 status int // HTTP status code. 54 err error // Optional reason for the HTTP error. 55} 56 57func (err *httpError) Error() string { 58 if err.err != nil { 59 return fmt.Sprintf("status %d, reason %s", err.status, err.err.Error()) 60 } 61 return fmt.Sprintf("Status %d", err.status) 62} 63 64const ( 65 humanRequest = iota 66 robotRequest 67 queryRequest 68 refreshRequest 69 apiRequest 70) 71 72type crawlResult struct { 73 pdoc *doc.Package 74 err error 75} 76 77// getDoc gets the package documentation from the database or from the version 78// control system as needed. 79func (s *server) getDoc(ctx context.Context, path string, requestType int) (*doc.Package, []database.Package, error) { 80 if path == "-" { 81 // A hack in the database package uses the path "-" to represent the 82 // next document to crawl. Block "-" here so that requests to /- always 83 // return not found. 84 return nil, nil, &httpError{status: http.StatusNotFound} 85 } 86 87 pdoc, pkgs, nextCrawl, err := s.db.Get(ctx, path) 88 if err != nil { 89 return nil, nil, err 90 } 91 92 needsCrawl := false 93 switch requestType { 94 case queryRequest, apiRequest: 95 needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0 96 case humanRequest: 97 needsCrawl = nextCrawl.Before(time.Now()) 98 case robotRequest: 99 needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0 100 } 101 102 if !needsCrawl { 103 return pdoc, pkgs, nil 104 } 105 106 c := make(chan crawlResult, 1) 107 go func() { 108 pdoc, err := s.crawlDoc(ctx, "web ", path, pdoc, len(pkgs) > 0, nextCrawl) 109 c <- crawlResult{pdoc, err} 110 }() 111 112 timeout := s.v.GetDuration(ConfigGetTimeout) 113 if pdoc == nil { 114 timeout = s.v.GetDuration(ConfigFirstGetTimeout) 115 } 116 117 select { 118 case cr := <-c: 119 err = cr.err 120 if err == nil { 121 pdoc = cr.pdoc 122 } 123 case <-time.After(timeout): 124 err = errUpdateTimeout 125 } 126 127 switch { 128 case err == nil: 129 return pdoc, pkgs, nil 130 case gosrc.IsNotFound(err): 131 return nil, nil, err 132 case pdoc != nil: 133 log.Printf("Serving %q from database after error getting doc: %v", path, err) 134 return pdoc, pkgs, nil 135 case err == errUpdateTimeout: 136 log.Printf("Serving %q as not found after timeout getting doc", path) 137 return nil, nil, &httpError{status: http.StatusNotFound} 138 default: 139 return nil, nil, err 140 } 141} 142 143func templateExt(req *http.Request) string { 144 if httputil.NegotiateContentType(req, []string{"text/html", "text/plain"}, "text/html") == "text/plain" { 145 return ".txt" 146 } 147 return ".html" 148} 149 150var robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)|(?:^Python-urllib)|(?:^Go )|(?:^Java/)`) 151 152func (s *server) isRobot(req *http.Request) bool { 153 if robotPat.MatchString(req.Header.Get("User-Agent")) { 154 return true 155 } 156 host := httputil.StripPort(req.RemoteAddr) 157 n, err := s.db.IncrementCounter(host, 1) 158 if err != nil { 159 log.Printf("error incrementing counter for %s, %v", host, err) 160 return false 161 } 162 if n > s.v.GetFloat64(ConfigRobotThreshold) { 163 log.Printf("robot %.2f %s %s", n, host, req.Header.Get("User-Agent")) 164 return true 165 } 166 return false 167} 168 169func popularLinkReferral(req *http.Request) bool { 170 return strings.HasSuffix(req.Header.Get("Referer"), "//"+req.Host+"/") 171} 172 173func isView(req *http.Request, key string) bool { 174 rq := req.URL.RawQuery 175 return strings.HasPrefix(rq, key) && 176 (len(rq) == len(key) || rq[len(key)] == '=' || rq[len(key)] == '&') 177} 178 179// httpEtag returns the package entity tag used in HTTP transactions. 180func (s *server) httpEtag(pdoc *doc.Package, pkgs []database.Package, importerCount int, flashMessages []flashMessage) string { 181 b := make([]byte, 0, 128) 182 b = strconv.AppendInt(b, pdoc.Updated.Unix(), 16) 183 b = append(b, 0) 184 b = append(b, pdoc.Etag...) 185 if importerCount >= 8 { 186 importerCount = 8 187 } 188 b = append(b, 0) 189 b = strconv.AppendInt(b, int64(importerCount), 16) 190 for _, pkg := range pkgs { 191 b = append(b, 0) 192 b = append(b, pkg.Path...) 193 b = append(b, 0) 194 b = append(b, pkg.Synopsis...) 195 } 196 if s.v.GetBool(ConfigSidebar) { 197 b = append(b, "\000xsb"...) 198 } 199 for _, m := range flashMessages { 200 b = append(b, 0) 201 b = append(b, m.ID...) 202 for _, a := range m.Args { 203 b = append(b, 1) 204 b = append(b, a...) 205 } 206 } 207 h := md5.New() 208 h.Write(b) 209 b = h.Sum(b[:0]) 210 return fmt.Sprintf("\"%x\"", b) 211} 212 213func (s *server) servePackage(resp http.ResponseWriter, req *http.Request) error { 214 p := path.Clean(req.URL.Path) 215 if strings.HasPrefix(p, "/pkg/") { 216 p = p[len("/pkg"):] 217 } 218 if p != req.URL.Path { 219 http.Redirect(resp, req, p, http.StatusMovedPermanently) 220 return nil 221 } 222 223 if isView(req, "status.svg") { 224 s.statusSVG.ServeHTTP(resp, req) 225 return nil 226 } 227 228 if isView(req, "status.png") { 229 s.statusPNG.ServeHTTP(resp, req) 230 return nil 231 } 232 233 requestType := humanRequest 234 if s.isRobot(req) { 235 requestType = robotRequest 236 } 237 238 importPath := strings.TrimPrefix(req.URL.Path, "/") 239 pdoc, pkgs, err := s.getDoc(req.Context(), importPath, requestType) 240 241 if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" { 242 // To prevent dumb clients from following redirect loops, respond with 243 // status 404 if the target document is not found. 244 if _, _, err := s.getDoc(req.Context(), e.Redirect, requestType); gosrc.IsNotFound(err) { 245 return &httpError{status: http.StatusNotFound} 246 } 247 u := "/" + e.Redirect 248 if req.URL.RawQuery != "" { 249 u += "?" + req.URL.RawQuery 250 } 251 setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}}) 252 http.Redirect(resp, req, u, http.StatusFound) 253 return nil 254 } 255 if err != nil { 256 return err 257 } 258 259 flashMessages := getFlashMessages(resp, req) 260 261 if pdoc == nil { 262 if len(pkgs) == 0 { 263 return &httpError{status: http.StatusNotFound} 264 } 265 pdocChild, _, _, err := s.db.Get(req.Context(), pkgs[0].Path) 266 if err != nil { 267 return err 268 } 269 pdoc = &doc.Package{ 270 ProjectName: pdocChild.ProjectName, 271 ProjectRoot: pdocChild.ProjectRoot, 272 ProjectURL: pdocChild.ProjectURL, 273 ImportPath: importPath, 274 } 275 } 276 277 showPkgGoDevRedirectToast := userReturningFromPkgGoDev(req) 278 279 switch { 280 case isView(req, "imports"): 281 if pdoc.Name == "" { 282 return &httpError{status: http.StatusNotFound} 283 } 284 pkgs, err = s.db.Packages(pdoc.Imports) 285 if err != nil { 286 return err 287 } 288 return s.templates.execute(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{ 289 "flashMessages": flashMessages, 290 "pkgs": pkgs, 291 "pdoc": newTDoc(s.v, pdoc), 292 "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast, 293 }) 294 case isView(req, "tools"): 295 proto := "http" 296 if req.Host == "godoc.org" { 297 proto = "https" 298 } 299 return s.templates.execute(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{ 300 "flashMessages": flashMessages, 301 "uri": fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath), 302 "pdoc": newTDoc(s.v, pdoc), 303 "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast, 304 }) 305 case isView(req, "importers"): 306 if pdoc.Name == "" { 307 return &httpError{status: http.StatusNotFound} 308 } 309 pkgs, err = s.db.Importers(importPath) 310 if err != nil { 311 return err 312 } 313 template := "importers.html" 314 if requestType == robotRequest { 315 // Hide back links from robots. 316 template = "importers_robot.html" 317 } 318 return s.templates.execute(resp, template, http.StatusOK, nil, map[string]interface{}{ 319 "flashMessages": flashMessages, 320 "pkgs": pkgs, 321 "pdoc": newTDoc(s.v, pdoc), 322 "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast, 323 }) 324 case isView(req, "import-graph"): 325 if requestType == robotRequest { 326 return &httpError{status: http.StatusForbidden} 327 } 328 if pdoc.Name == "" { 329 return &httpError{status: http.StatusNotFound} 330 } 331 332 // Throttle ?import-graph requests. 333 select { 334 case s.importGraphSem <- struct{}{}: 335 default: 336 return &httpError{status: http.StatusTooManyRequests} 337 } 338 defer func() { <-s.importGraphSem }() 339 340 hide := database.ShowAllDeps 341 switch req.Form.Get("hide") { 342 case "1": 343 hide = database.HideStandardDeps 344 case "2": 345 hide = database.HideStandardAll 346 } 347 pkgs, edges, err := s.db.ImportGraph(pdoc, hide) 348 if err != nil { 349 return err 350 } 351 b, err := renderGraph(pdoc, pkgs, edges) 352 if err != nil { 353 return err 354 } 355 return s.templates.execute(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{ 356 "flashMessages": flashMessages, 357 "svg": template.HTML(b), 358 "pdoc": newTDoc(s.v, pdoc), 359 "hide": hide, 360 "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast, 361 }) 362 case isView(req, "play"): 363 u, err := s.playURL(pdoc, req.Form.Get("play"), req.Header.Get("X-AppEngine-Country")) 364 if err != nil { 365 return err 366 } 367 http.Redirect(resp, req, u, http.StatusMovedPermanently) 368 return nil 369 case req.Form.Get("view") != "": 370 // Redirect deprecated view= queries. 371 var q string 372 switch view := req.Form.Get("view"); view { 373 case "imports", "importers": 374 q = view 375 case "import-graph": 376 if req.Form.Get("hide") == "1" { 377 q = "import-graph&hide=1" 378 } else { 379 q = "import-graph" 380 } 381 } 382 if q != "" { 383 u := *req.URL 384 u.RawQuery = q 385 http.Redirect(resp, req, u.String(), http.StatusMovedPermanently) 386 return nil 387 } 388 return &httpError{status: http.StatusNotFound} 389 default: 390 importerCount := 0 391 if pdoc.Name != "" { 392 importerCount, err = s.db.ImporterCount(importPath) 393 if err != nil { 394 return err 395 } 396 } 397 398 etag := s.httpEtag(pdoc, pkgs, importerCount, flashMessages) 399 status := http.StatusOK 400 if req.Header.Get("If-None-Match") == etag { 401 status = http.StatusNotModified 402 } 403 404 if requestType == humanRequest && 405 pdoc.Name != "" && // not a directory 406 pdoc.ProjectRoot != "" && // not a standard package 407 !pdoc.IsCmd && 408 len(pdoc.Errors) == 0 && 409 !popularLinkReferral(req) { 410 if err := s.db.IncrementPopularScore(pdoc.ImportPath); err != nil { 411 log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err) 412 } 413 } 414 if s.gceLogger != nil { 415 s.gceLogger.LogEvent(resp, req, nil) 416 } 417 418 template := "dir" 419 switch { 420 case pdoc.IsCmd: 421 template = "cmd" 422 case pdoc.Name != "": 423 template = "pkg" 424 } 425 template += templateExt(req) 426 427 return s.templates.execute(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{ 428 "flashMessages": flashMessages, 429 "pkgs": pkgs, 430 "pdoc": newTDoc(s.v, pdoc), 431 "importerCount": importerCount, 432 "showPkgGoDevRedirectToast": showPkgGoDevRedirectToast, 433 }) 434 } 435} 436 437func (s *server) serveRefresh(resp http.ResponseWriter, req *http.Request) error { 438 importPath := req.Form.Get("path") 439 _, pkgs, _, err := s.db.Get(req.Context(), importPath) 440 if err != nil { 441 return err 442 } 443 c := make(chan error, 1) 444 go func() { 445 _, err := s.crawlDoc(req.Context(), "rfrsh", importPath, nil, len(pkgs) > 0, time.Time{}) 446 c <- err 447 }() 448 select { 449 case err = <-c: 450 case <-time.After(s.v.GetDuration(ConfigGetTimeout)): 451 err = errUpdateTimeout 452 } 453 if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" { 454 setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}}) 455 importPath = e.Redirect 456 err = nil 457 } else if err != nil { 458 setFlashMessages(resp, []flashMessage{{ID: "refresh", Args: []string{errorText(err)}}}) 459 } 460 http.Redirect(resp, req, "/"+importPath, http.StatusFound) 461 return nil 462} 463 464func (s *server) serveGoIndex(resp http.ResponseWriter, req *http.Request) error { 465 pkgs, err := s.db.GoIndex() 466 if err != nil { 467 return err 468 } 469 return s.templates.execute(resp, "std.html", http.StatusOK, nil, map[string]interface{}{ 470 "pkgs": pkgs, 471 }) 472} 473 474func (s *server) serveGoSubrepoIndex(resp http.ResponseWriter, req *http.Request) error { 475 pkgs, err := s.db.GoSubrepoIndex() 476 if err != nil { 477 return err 478 } 479 return s.templates.execute(resp, "subrepo.html", http.StatusOK, nil, map[string]interface{}{ 480 "pkgs": pkgs, 481 }) 482} 483 484type byPath struct { 485 pkgs []database.Package 486 rank []int 487} 488 489func (bp *byPath) Len() int { return len(bp.pkgs) } 490func (bp *byPath) Less(i, j int) bool { return bp.pkgs[i].Path < bp.pkgs[j].Path } 491func (bp *byPath) Swap(i, j int) { 492 bp.pkgs[i], bp.pkgs[j] = bp.pkgs[j], bp.pkgs[i] 493 bp.rank[i], bp.rank[j] = bp.rank[j], bp.rank[i] 494} 495 496type byRank struct { 497 pkgs []database.Package 498 rank []int 499} 500 501func (br *byRank) Len() int { return len(br.pkgs) } 502func (br *byRank) Less(i, j int) bool { return br.rank[i] < br.rank[j] } 503func (br *byRank) Swap(i, j int) { 504 br.pkgs[i], br.pkgs[j] = br.pkgs[j], br.pkgs[i] 505 br.rank[i], br.rank[j] = br.rank[j], br.rank[i] 506} 507 508func (s *server) popular() ([]database.Package, error) { 509 const n = 25 510 511 pkgs, err := s.db.Popular(2 * n) 512 if err != nil { 513 return nil, err 514 } 515 516 rank := make([]int, len(pkgs)) 517 for i := range pkgs { 518 rank[i] = i 519 } 520 521 sort.Sort(&byPath{pkgs, rank}) 522 523 j := 0 524 prev := "." 525 for i, pkg := range pkgs { 526 if strings.HasPrefix(pkg.Path, prev) { 527 if rank[j-1] < rank[i] { 528 rank[j-1] = rank[i] 529 } 530 continue 531 } 532 prev = pkg.Path + "/" 533 pkgs[j] = pkg 534 rank[j] = rank[i] 535 j++ 536 } 537 pkgs = pkgs[:j] 538 539 sort.Sort(&byRank{pkgs, rank}) 540 541 if len(pkgs) > n { 542 pkgs = pkgs[:n] 543 } 544 545 sort.Sort(&byPath{pkgs, rank}) 546 547 return pkgs, nil 548} 549 550func (s *server) serveHome(resp http.ResponseWriter, req *http.Request) error { 551 if req.URL.Path != "/" { 552 return s.servePackage(resp, req) 553 } 554 555 q := strings.TrimSpace(req.Form.Get("q")) 556 if q == "" { 557 pkgs, err := s.popular() 558 if err != nil { 559 return err 560 } 561 562 return s.templates.execute(resp, "home"+templateExt(req), http.StatusOK, nil, 563 map[string]interface{}{ 564 "Popular": pkgs, 565 566 "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req), 567 }) 568 } 569 570 if path, ok := isBrowseURL(q); ok { 571 q = path 572 } 573 574 if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) { 575 pdoc, pkgs, err := s.getDoc(req.Context(), q, queryRequest) 576 if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" { 577 http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound) 578 return nil 579 } 580 if err == nil && (pdoc != nil || len(pkgs) > 0) { 581 http.Redirect(resp, req, "/"+q, http.StatusFound) 582 return nil 583 } 584 } 585 586 pkgs, err := s.db.Search(req.Context(), q) 587 if err != nil { 588 return err 589 } 590 if s.gceLogger != nil { 591 // Log up to top 10 packages we served upon a search. 592 logPkgs := pkgs 593 if len(pkgs) > 10 { 594 logPkgs = pkgs[:10] 595 } 596 s.gceLogger.LogEvent(resp, req, logPkgs) 597 } 598 599 return s.templates.execute(resp, "results"+templateExt(req), http.StatusOK, nil, 600 map[string]interface{}{ 601 "q": q, 602 "pkgs": pkgs, 603 604 "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req), 605 }) 606} 607 608func (s *server) serveAbout(resp http.ResponseWriter, req *http.Request) error { 609 return s.templates.execute(resp, "about.html", http.StatusOK, nil, 610 map[string]interface{}{ 611 "Host": req.Host, 612 613 "showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req), 614 }) 615} 616 617func (s *server) serveBot(resp http.ResponseWriter, req *http.Request) error { 618 return s.templates.execute(resp, "bot.html", http.StatusOK, nil, nil) 619} 620 621func logError(req *http.Request, err error, rv interface{}) { 622 if err != nil { 623 var buf bytes.Buffer 624 fmt.Fprintf(&buf, "Error serving %s: %v\n", req.URL, err) 625 if rv != nil { 626 fmt.Fprintln(&buf, rv) 627 buf.Write(debug.Stack()) 628 } 629 log.Print(buf.String()) 630 } 631} 632 633func (s *server) serveAPISearch(resp http.ResponseWriter, req *http.Request) error { 634 q := strings.TrimSpace(req.Form.Get("q")) 635 636 var pkgs []database.Package 637 638 if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) { 639 pdoc, _, err := s.getDoc(req.Context(), q, apiRequest) 640 if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" { 641 pdoc, _, err = s.getDoc(req.Context(), e.Redirect, robotRequest) 642 } 643 if err == nil && pdoc != nil { 644 pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}} 645 } 646 } 647 648 if pkgs == nil { 649 var err error 650 pkgs, err = s.db.Search(req.Context(), q) 651 if err != nil { 652 return err 653 } 654 } 655 656 var data = struct { 657 Results []database.Package `json:"results"` 658 }{ 659 pkgs, 660 } 661 resp.Header().Set("Content-Type", jsonMIMEType) 662 return json.NewEncoder(resp).Encode(&data) 663} 664 665func (s *server) serveAPIPackages(resp http.ResponseWriter, req *http.Request) error { 666 pkgs, err := s.db.AllPackages() 667 if err != nil { 668 return err 669 } 670 data := struct { 671 Results []database.Package `json:"results"` 672 }{ 673 pkgs, 674 } 675 resp.Header().Set("Content-Type", jsonMIMEType) 676 return json.NewEncoder(resp).Encode(&data) 677} 678 679func (s *server) serveAPIImporters(resp http.ResponseWriter, req *http.Request) error { 680 importPath := strings.TrimPrefix(req.URL.Path, "/importers/") 681 pkgs, err := s.db.Importers(importPath) 682 if err != nil { 683 return err 684 } 685 data := struct { 686 Results []database.Package `json:"results"` 687 }{ 688 pkgs, 689 } 690 resp.Header().Set("Content-Type", jsonMIMEType) 691 return json.NewEncoder(resp).Encode(&data) 692} 693 694func (s *server) serveAPIImports(resp http.ResponseWriter, req *http.Request) error { 695 importPath := strings.TrimPrefix(req.URL.Path, "/imports/") 696 pdoc, _, err := s.getDoc(req.Context(), importPath, robotRequest) 697 if err != nil { 698 return err 699 } 700 if pdoc == nil || pdoc.Name == "" { 701 return &httpError{status: http.StatusNotFound} 702 } 703 imports, err := s.db.Packages(pdoc.Imports) 704 if err != nil { 705 return err 706 } 707 testImports, err := s.db.Packages(pdoc.TestImports) 708 if err != nil { 709 return err 710 } 711 data := struct { 712 Imports []database.Package `json:"imports"` 713 TestImports []database.Package `json:"testImports"` 714 }{ 715 imports, 716 testImports, 717 } 718 resp.Header().Set("Content-Type", jsonMIMEType) 719 return json.NewEncoder(resp).Encode(&data) 720} 721 722func serveAPIHome(resp http.ResponseWriter, req *http.Request) error { 723 return &httpError{status: http.StatusNotFound} 724} 725 726type requestCleaner struct { 727 h http.Handler 728 trustProxyHeaders bool 729} 730 731func (rc requestCleaner) ServeHTTP(w http.ResponseWriter, req *http.Request) { 732 req2 := new(http.Request) 733 *req2 = *req 734 if rc.trustProxyHeaders { 735 if s := req.Header.Get("X-Forwarded-For"); s != "" { 736 req2.RemoteAddr = s 737 } 738 } 739 req2.Body = http.MaxBytesReader(w, req.Body, 2048) 740 req2.ParseForm() 741 rc.h.ServeHTTP(w, req2) 742} 743 744type errorHandler struct { 745 fn func(resp http.ResponseWriter, req *http.Request) error 746 errFn httputil.Error 747} 748 749func (eh errorHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 750 defer func() { 751 if rv := recover(); rv != nil { 752 err := errors.New("handler panic") 753 logError(req, err, rv) 754 eh.errFn(resp, req, http.StatusInternalServerError, err) 755 } 756 }() 757 758 rb := new(httputil.ResponseBuffer) 759 err := eh.fn(rb, req) 760 if err == nil { 761 rb.WriteTo(resp) 762 } else if e, ok := err.(*httpError); ok { 763 if e.status >= 500 { 764 logError(req, err, nil) 765 } 766 eh.errFn(resp, req, e.status, e.err) 767 } else if gosrc.IsNotFound(err) { 768 eh.errFn(resp, req, http.StatusNotFound, nil) 769 } else { 770 logError(req, err, nil) 771 eh.errFn(resp, req, http.StatusInternalServerError, err) 772 } 773} 774 775func errorText(err error) string { 776 if err == errUpdateTimeout { 777 return "Timeout getting package files from the version control system." 778 } 779 if e, ok := err.(*gosrc.RemoteError); ok { 780 return "Error getting package files from " + e.Host + "." 781 } 782 return "Internal server error." 783} 784 785func (s *server) handleError(resp http.ResponseWriter, req *http.Request, status int, err error) { 786 switch status { 787 case http.StatusNotFound: 788 s.templates.execute(resp, "notfound"+templateExt(req), status, nil, map[string]interface{}{ 789 "flashMessages": getFlashMessages(resp, req), 790 }) 791 default: 792 resp.Header().Set("Content-Type", textMIMEType) 793 resp.WriteHeader(http.StatusInternalServerError) 794 io.WriteString(resp, errorText(err)) 795 } 796} 797 798func handleAPIError(resp http.ResponseWriter, req *http.Request, status int, err error) { 799 var data struct { 800 Error struct { 801 Message string `json:"message"` 802 } `json:"error"` 803 } 804 data.Error.Message = http.StatusText(status) 805 resp.Header().Set("Content-Type", jsonMIMEType) 806 resp.WriteHeader(status) 807 json.NewEncoder(resp).Encode(&data) 808} 809 810// httpsRedirectHandler redirects all requests with an X-Forwarded-Proto: http 811// handler to their https equivalent. 812type httpsRedirectHandler struct { 813 h http.Handler 814} 815 816func (h httpsRedirectHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 817 if req.Header.Get("X-Forwarded-Proto") == "http" { 818 u := *req.URL 819 u.Scheme = "https" 820 u.Host = req.Host 821 http.Redirect(resp, req, u.String(), http.StatusFound) 822 return 823 } 824 h.h.ServeHTTP(resp, req) 825} 826 827type rootHandler []struct { 828 prefix string 829 h http.Handler 830} 831 832func (m rootHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 833 var h http.Handler 834 for _, ph := range m { 835 if strings.HasPrefix(req.Host, ph.prefix) { 836 h = ph.h 837 break 838 } 839 } 840 841 h.ServeHTTP(resp, req) 842} 843 844// otherDomainHandler redirects to another domain keeping the rest of the URL. 845type otherDomainHandler struct { 846 scheme string 847 targetDomain string 848} 849 850func (h otherDomainHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { 851 u := *req.URL 852 u.Scheme = h.scheme 853 u.Host = h.targetDomain 854 http.Redirect(w, req, u.String(), http.StatusFound) 855} 856 857func defaultBase(path string) string { 858 p, err := build.Default.Import(path, "", build.FindOnly) 859 if err != nil { 860 return "." 861 } 862 return p.Dir 863} 864 865type server struct { 866 v *viper.Viper 867 db *database.Database 868 httpClient *http.Client 869 gceLogger *GCELogger 870 templates templateMap 871 traceClient *trace.Client 872 crawlTopic *pubsub.Topic 873 874 statusPNG http.Handler 875 statusSVG http.Handler 876 877 root rootHandler 878 879 // A semaphore to limit concurrent ?import-graph requests. 880 importGraphSem chan struct{} 881} 882 883func newServer(ctx context.Context, v *viper.Viper) (*server, error) { 884 s := &server{ 885 v: v, 886 httpClient: newHTTPClient(v), 887 importGraphSem: make(chan struct{}, 10), 888 } 889 890 var err error 891 if proj := s.v.GetString(ConfigProject); proj != "" { 892 if s.traceClient, err = trace.NewClient(ctx, proj); err != nil { 893 return nil, err 894 } 895 sp, err := trace.NewLimitedSampler(s.v.GetFloat64(ConfigTraceSamplerFraction), s.v.GetFloat64(ConfigTraceSamplerMaxQPS)) 896 if err != nil { 897 return nil, err 898 } 899 s.traceClient.SetSamplingPolicy(sp) 900 901 // This topic should be created in the cloud console. 902 ps, err := pubsub.NewClient(ctx, proj) 903 if err != nil { 904 return nil, err 905 } 906 s.crawlTopic = ps.Topic(ConfigCrawlPubSubTopic) 907 } 908 909 assets := v.GetString(ConfigAssetsDir) 910 staticServer := httputil.StaticServer{ 911 Dir: assets, 912 MaxAge: time.Hour, 913 MIMETypes: map[string]string{ 914 ".css": "text/css; charset=utf-8", 915 ".js": "text/javascript; charset=utf-8", 916 }, 917 } 918 s.statusPNG = staticServer.FileHandler("status.png") 919 s.statusSVG = staticServer.FileHandler("status.svg") 920 921 apiHandler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler { 922 return requestCleaner{ 923 h: errorHandler{ 924 fn: f, 925 errFn: handleAPIError, 926 }, 927 trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders), 928 } 929 } 930 apiMux := http.NewServeMux() 931 apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) 932 apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) 933 apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) 934 apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt")) 935 apiMux.Handle("/search", apiHandler(s.serveAPISearch)) 936 apiMux.Handle("/packages", apiHandler(s.serveAPIPackages)) 937 apiMux.Handle("/importers/", apiHandler(s.serveAPIImporters)) 938 apiMux.Handle("/imports/", apiHandler(s.serveAPIImports)) 939 apiMux.Handle("/", apiHandler(serveAPIHome)) 940 941 mux := http.NewServeMux() 942 mux.Handle("/-/site.js", staticServer.FilesHandler( 943 "third_party/jquery.timeago.js", 944 "site.js")) 945 mux.Handle("/-/site.css", staticServer.FilesHandler("site.css")) 946 mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css")) 947 mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js")) 948 mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js")) 949 if s.v.GetBool(ConfigSidebar) { 950 mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css")) 951 } 952 mux.Handle("/-/", http.NotFoundHandler()) 953 954 handler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler { 955 return requestCleaner{ 956 h: errorHandler{ 957 fn: f, 958 errFn: s.handleError, 959 }, 960 trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders), 961 } 962 } 963 964 mux.Handle("/-/about", handler(pkgGoDevRedirectHandler(s.serveAbout))) 965 mux.Handle("/-/bot", handler(s.serveBot)) 966 mux.Handle("/-/go", handler(pkgGoDevRedirectHandler(s.serveGoIndex))) 967 mux.Handle("/-/subrepo", handler(s.serveGoSubrepoIndex)) 968 mux.Handle("/-/refresh", handler(s.serveRefresh)) 969 mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently)) 970 mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico")) 971 mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html")) 972 mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt")) 973 mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt")) 974 mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml")) 975 mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently)) 976 mux.Handle("/code.jquery.com/", http.NotFoundHandler()) 977 mux.Handle("/", handler(pkgGoDevRedirectHandler(s.serveHome))) 978 979 ahMux := http.NewServeMux() 980 ready := new(health.Handler) 981 ahMux.HandleFunc("/_ah/health", health.HandleLive) 982 ahMux.Handle("/_ah/ready", ready) 983 984 mainMux := http.NewServeMux() 985 mainMux.Handle("/_ah/", ahMux) 986 mainMux.Handle("/", s.traceClient.HTTPHandler(mux)) 987 988 s.root = rootHandler{ 989 {"api.", httpsRedirectHandler{s.traceClient.HTTPHandler(apiMux)}}, 990 {"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}}, 991 {"", httpsRedirectHandler{mainMux}}, 992 } 993 994 cacheBusters := &httputil.CacheBusters{Handler: mux} 995 s.templates, err = parseTemplates(assets, cacheBusters, v) 996 if err != nil { 997 return nil, err 998 } 999 s.db, err = database.New( 1000 v.GetString(ConfigDBServer), 1001 v.GetDuration(ConfigDBIdleTimeout), 1002 v.GetBool(ConfigDBLog), 1003 v.GetString(ConfigGAERemoteAPI), 1004 ) 1005 if err != nil { 1006 return nil, fmt.Errorf("open database: %v", err) 1007 } 1008 ready.Add(s.db) 1009 if gceLogName := v.GetString(ConfigGCELogName); gceLogName != "" { 1010 logc, err := logging.NewClient(ctx, v.GetString(ConfigProject)) 1011 if err != nil { 1012 return nil, fmt.Errorf("create cloud logging client: %v", err) 1013 } 1014 logger := logc.Logger(gceLogName) 1015 if err := logc.Ping(ctx); err != nil { 1016 return nil, fmt.Errorf("pinging cloud logging: %v", err) 1017 } 1018 s.gceLogger = newGCELogger(logger) 1019 } 1020 return s, nil 1021} 1022 1023const ( 1024 pkgGoDevRedirectCookie = "pkggodev-redirect" 1025 pkgGoDevRedirectParam = "redirect" 1026 pkgGoDevRedirectOn = "on" 1027 pkgGoDevRedirectOff = "off" 1028 pkgGoDevHost = "pkg.go.dev" 1029 teeproxyHost = "teeproxy-dot-go-discovery.appspot.com" 1030) 1031 1032type responseWriter struct { 1033 http.ResponseWriter 1034 status int 1035} 1036 1037func (rw *responseWriter) WriteHeader(code int) { 1038 rw.status = code 1039 rw.ResponseWriter.WriteHeader(code) 1040} 1041 1042func translateStatus(code int) int { 1043 if code == 0 { 1044 return http.StatusOK 1045 } 1046 return code 1047} 1048 1049func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 1050 start := time.Now() 1051 s.logRequestStart(r) 1052 w2 := &responseWriter{ResponseWriter: w} 1053 s.root.ServeHTTP(w2, r) 1054 latency := time.Since(start) 1055 s.logRequestEnd(r, latency) 1056 if f, ok := w.(http.Flusher); ok { 1057 f.Flush() 1058 } 1059 // Don't tee App Engine requests to pkg.go.dev. 1060 if strings.HasPrefix(r.URL.Path, "/_ah/") { 1061 return 1062 } 1063 if err := makePkgGoDevRequest(r, latency, s.isRobot(r), translateStatus(w2.status)); err != nil { 1064 log.Printf("makePkgGoDevRequest(%q, %d) error: %v", r.URL, latency, err) 1065 } 1066} 1067 1068func makePkgGoDevRequest(r *http.Request, latency time.Duration, isRobot bool, status int) error { 1069 event := newGDDOEvent(r, latency, isRobot, status) 1070 b, err := json.Marshal(event) 1071 if err != nil { 1072 return fmt.Errorf("json.Marshal(%v): %v", event, err) 1073 } 1074 1075 teeproxyURL := url.URL{Scheme: "https", Host: teeproxyHost} 1076 if _, err := http.Post(teeproxyURL.String(), jsonMIMEType, bytes.NewReader(b)); err != nil { 1077 return fmt.Errorf("http.Post(%q, %q, %v): %v", teeproxyURL.String(), jsonMIMEType, event, err) 1078 } 1079 log.Printf("makePkgGoDevRequest: request made to %q for %+v", teeproxyURL.String(), event) 1080 return nil 1081} 1082 1083type gddoEvent struct { 1084 Host string 1085 Path string 1086 Status int 1087 URL string 1088 Header http.Header 1089 Latency time.Duration 1090 IsRobot bool 1091 UsePkgGoDev bool 1092} 1093 1094func newGDDOEvent(r *http.Request, latency time.Duration, isRobot bool, status int) *gddoEvent { 1095 targetURL := url.URL{ 1096 Scheme: "https", 1097 Host: r.URL.Host, 1098 Path: r.URL.Path, 1099 RawQuery: r.URL.RawQuery, 1100 } 1101 if targetURL.Host == "" && r.Host != "" { 1102 targetURL.Host = r.Host 1103 } 1104 return &gddoEvent{ 1105 Host: targetURL.Host, 1106 Path: r.URL.Path, 1107 Status: status, 1108 URL: targetURL.String(), 1109 Header: r.Header, 1110 Latency: latency, 1111 IsRobot: isRobot, 1112 UsePkgGoDev: shouldRedirectToPkgGoDev(r), 1113 } 1114} 1115 1116func (s *server) logRequestStart(req *http.Request) { 1117 if s.gceLogger == nil { 1118 return 1119 } 1120 s.gceLogger.Log(logging.Entry{ 1121 HTTPRequest: &logging.HTTPRequest{Request: req}, 1122 Payload: fmt.Sprintf("%s request start", req.Host), 1123 Severity: logging.Info, 1124 }) 1125} 1126 1127func (s *server) logRequestEnd(req *http.Request, latency time.Duration) { 1128 if s.gceLogger == nil { 1129 return 1130 } 1131 s.gceLogger.Log(logging.Entry{ 1132 HTTPRequest: &logging.HTTPRequest{ 1133 Request: req, 1134 Latency: latency, 1135 }, 1136 Payload: fmt.Sprintf("%s request end", req.Host), 1137 Severity: logging.Info, 1138 }) 1139} 1140 1141func userReturningFromPkgGoDev(req *http.Request) bool { 1142 return req.FormValue("utm_source") == "backtogodoc" 1143} 1144 1145func shouldRedirectToPkgGoDev(req *http.Request) bool { 1146 // API requests are not redirected. 1147 if strings.HasPrefix(req.URL.Host, "api") { 1148 return false 1149 } 1150 redirectParam := req.FormValue(pkgGoDevRedirectParam) 1151 if redirectParam == pkgGoDevRedirectOn || redirectParam == pkgGoDevRedirectOff { 1152 return redirectParam == pkgGoDevRedirectOn 1153 } 1154 cookie, err := req.Cookie(pkgGoDevRedirectCookie) 1155 return (err == nil && cookie.Value == pkgGoDevRedirectOn) 1156} 1157 1158// pkgGoDevRedirectHandler redirects requests from godoc.org to pkg.go.dev, 1159// based on whether a cookie is set for pkggodev-redirect. The cookie 1160// can be turned on/off using a query param. 1161func pkgGoDevRedirectHandler(f func(http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) error { 1162 return func(w http.ResponseWriter, r *http.Request) error { 1163 if userReturningFromPkgGoDev(r) { 1164 return f(w, r) 1165 } 1166 1167 redirectParam := r.FormValue(pkgGoDevRedirectParam) 1168 1169 if redirectParam == pkgGoDevRedirectOn { 1170 cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: redirectParam, Path: "/"} 1171 http.SetCookie(w, cookie) 1172 } 1173 if redirectParam == pkgGoDevRedirectOff { 1174 cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: "", MaxAge: -1, Path: "/"} 1175 http.SetCookie(w, cookie) 1176 } 1177 1178 if !shouldRedirectToPkgGoDev(r) { 1179 return f(w, r) 1180 } 1181 1182 http.Redirect(w, r, pkgGoDevURL(r.URL).String(), http.StatusFound) 1183 return nil 1184 } 1185} 1186 1187func pkgGoDevURL(godocURL *url.URL) *url.URL { 1188 u := &url.URL{Scheme: "https", Host: pkgGoDevHost} 1189 q := url.Values{"utm_source": []string{"godoc"}} 1190 1191 if strings.Contains(godocURL.Path, "/vendor/") || strings.HasSuffix(godocURL.Path, "/vendor") { 1192 u.Path = "/" 1193 u.RawQuery = q.Encode() 1194 return u 1195 } 1196 1197 switch godocURL.Path { 1198 case "/-/go": 1199 u.Path = "/std" 1200 q.Add("tab", "packages") 1201 case "/-/about": 1202 u.Path = "/about" 1203 case "/": 1204 if qparam := godocURL.Query().Get("q"); qparam != "" { 1205 u.Path = "/search" 1206 q.Set("q", qparam) 1207 } else { 1208 u.Path = "/" 1209 } 1210 default: 1211 { 1212 u.Path = godocURL.Path 1213 if _, ok := godocURL.Query()["imports"]; ok { 1214 q.Set("tab", "imports") 1215 } else if _, ok := godocURL.Query()["importers"]; ok { 1216 q.Set("tab", "importedby") 1217 } else { 1218 q.Set("tab", "doc") 1219 } 1220 } 1221 } 1222 1223 u.RawQuery = q.Encode() 1224 return u 1225} 1226 1227func main() { 1228 ctx := context.Background() 1229 v, err := loadConfig(ctx, os.Args) 1230 if err != nil { 1231 log.Fatal(ctx, "load config", "error", err.Error()) 1232 } 1233 doc.SetDefaultGOOS(v.GetString(ConfigDefaultGOOS)) 1234 1235 s, err := newServer(ctx, v) 1236 if err != nil { 1237 log.Fatal("error creating server:", err) 1238 } 1239 1240 go func() { 1241 for range time.Tick(s.v.GetDuration(ConfigCrawlInterval)) { 1242 if err := s.doCrawl(ctx); err != nil { 1243 log.Printf("Task Crawl: %v", err) 1244 } 1245 } 1246 }() 1247 go func() { 1248 for range time.Tick(s.v.GetDuration(ConfigGithubInterval)) { 1249 if err := s.readGitHubUpdates(ctx); err != nil { 1250 log.Printf("Task GitHub updates: %v", err) 1251 } 1252 } 1253 }() 1254 http.Handle("/", s) 1255 log.Fatal(http.ListenAndServe(s.v.GetString(ConfigBindAddress), s)) 1256} 1257