1// Copyright 2017 The Go Authors. All rights reserved.
2//
3// Use of this source code is governed by a BSD-style
4// license that can be found in the LICENSE file or at
5// https://developers.google.com/open-source/licenses/bsd.
6
7// Command gddo-server is the GoPkgDoc server.
8package main
9
10import (
11	"bytes"
12	"context"
13	"crypto/md5"
14	"encoding/json"
15	"errors"
16	"fmt"
17	"go/build"
18	"html/template"
19	"io"
20	"log"
21	"net/http"
22	"net/url"
23	"os"
24	"path"
25	"regexp"
26	"runtime/debug"
27	"sort"
28	"strconv"
29	"strings"
30	"time"
31
32	"cloud.google.com/go/logging"
33	"cloud.google.com/go/pubsub"
34	"cloud.google.com/go/trace"
35	"github.com/spf13/viper"
36
37	"github.com/golang/gddo/database"
38	"github.com/golang/gddo/doc"
39	"github.com/golang/gddo/gosrc"
40	"github.com/golang/gddo/httputil"
41	"github.com/golang/gddo/internal/health"
42)
43
44const (
45	jsonMIMEType = "application/json; charset=utf-8"
46	textMIMEType = "text/plain; charset=utf-8"
47	htmlMIMEType = "text/html; charset=utf-8"
48)
49
50var errUpdateTimeout = errors.New("refresh timeout")
51
52type httpError struct {
53	status int   // HTTP status code.
54	err    error // Optional reason for the HTTP error.
55}
56
57func (err *httpError) Error() string {
58	if err.err != nil {
59		return fmt.Sprintf("status %d, reason %s", err.status, err.err.Error())
60	}
61	return fmt.Sprintf("Status %d", err.status)
62}
63
64const (
65	humanRequest = iota
66	robotRequest
67	queryRequest
68	refreshRequest
69	apiRequest
70)
71
72type crawlResult struct {
73	pdoc *doc.Package
74	err  error
75}
76
77// getDoc gets the package documentation from the database or from the version
78// control system as needed.
79func (s *server) getDoc(ctx context.Context, path string, requestType int) (*doc.Package, []database.Package, error) {
80	if path == "-" {
81		// A hack in the database package uses the path "-" to represent the
82		// next document to crawl. Block "-" here so that requests to /- always
83		// return not found.
84		return nil, nil, &httpError{status: http.StatusNotFound}
85	}
86
87	pdoc, pkgs, nextCrawl, err := s.db.Get(ctx, path)
88	if err != nil {
89		return nil, nil, err
90	}
91
92	needsCrawl := false
93	switch requestType {
94	case queryRequest, apiRequest:
95		needsCrawl = nextCrawl.IsZero() && len(pkgs) == 0
96	case humanRequest:
97		needsCrawl = nextCrawl.Before(time.Now())
98	case robotRequest:
99		needsCrawl = nextCrawl.IsZero() && len(pkgs) > 0
100	}
101
102	if !needsCrawl {
103		return pdoc, pkgs, nil
104	}
105
106	c := make(chan crawlResult, 1)
107	go func() {
108		pdoc, err := s.crawlDoc(ctx, "web  ", path, pdoc, len(pkgs) > 0, nextCrawl)
109		c <- crawlResult{pdoc, err}
110	}()
111
112	timeout := s.v.GetDuration(ConfigGetTimeout)
113	if pdoc == nil {
114		timeout = s.v.GetDuration(ConfigFirstGetTimeout)
115	}
116
117	select {
118	case cr := <-c:
119		err = cr.err
120		if err == nil {
121			pdoc = cr.pdoc
122		}
123	case <-time.After(timeout):
124		err = errUpdateTimeout
125	}
126
127	switch {
128	case err == nil:
129		return pdoc, pkgs, nil
130	case gosrc.IsNotFound(err):
131		return nil, nil, err
132	case pdoc != nil:
133		log.Printf("Serving %q from database after error getting doc: %v", path, err)
134		return pdoc, pkgs, nil
135	case err == errUpdateTimeout:
136		log.Printf("Serving %q as not found after timeout getting doc", path)
137		return nil, nil, &httpError{status: http.StatusNotFound}
138	default:
139		return nil, nil, err
140	}
141}
142
143func templateExt(req *http.Request) string {
144	if httputil.NegotiateContentType(req, []string{"text/html", "text/plain"}, "text/html") == "text/plain" {
145		return ".txt"
146	}
147	return ".html"
148}
149
150var robotPat = regexp.MustCompile(`(:?\+https?://)|(?:\Wbot\W)|(?:^Python-urllib)|(?:^Go )|(?:^Java/)`)
151
152func (s *server) isRobot(req *http.Request) bool {
153	if robotPat.MatchString(req.Header.Get("User-Agent")) {
154		return true
155	}
156	host := httputil.StripPort(req.RemoteAddr)
157	n, err := s.db.IncrementCounter(host, 1)
158	if err != nil {
159		log.Printf("error incrementing counter for %s, %v", host, err)
160		return false
161	}
162	if n > s.v.GetFloat64(ConfigRobotThreshold) {
163		log.Printf("robot %.2f %s %s", n, host, req.Header.Get("User-Agent"))
164		return true
165	}
166	return false
167}
168
169func popularLinkReferral(req *http.Request) bool {
170	return strings.HasSuffix(req.Header.Get("Referer"), "//"+req.Host+"/")
171}
172
173func isView(req *http.Request, key string) bool {
174	rq := req.URL.RawQuery
175	return strings.HasPrefix(rq, key) &&
176		(len(rq) == len(key) || rq[len(key)] == '=' || rq[len(key)] == '&')
177}
178
179// httpEtag returns the package entity tag used in HTTP transactions.
180func (s *server) httpEtag(pdoc *doc.Package, pkgs []database.Package, importerCount int, flashMessages []flashMessage) string {
181	b := make([]byte, 0, 128)
182	b = strconv.AppendInt(b, pdoc.Updated.Unix(), 16)
183	b = append(b, 0)
184	b = append(b, pdoc.Etag...)
185	if importerCount >= 8 {
186		importerCount = 8
187	}
188	b = append(b, 0)
189	b = strconv.AppendInt(b, int64(importerCount), 16)
190	for _, pkg := range pkgs {
191		b = append(b, 0)
192		b = append(b, pkg.Path...)
193		b = append(b, 0)
194		b = append(b, pkg.Synopsis...)
195	}
196	if s.v.GetBool(ConfigSidebar) {
197		b = append(b, "\000xsb"...)
198	}
199	for _, m := range flashMessages {
200		b = append(b, 0)
201		b = append(b, m.ID...)
202		for _, a := range m.Args {
203			b = append(b, 1)
204			b = append(b, a...)
205		}
206	}
207	h := md5.New()
208	h.Write(b)
209	b = h.Sum(b[:0])
210	return fmt.Sprintf("\"%x\"", b)
211}
212
213func (s *server) servePackage(resp http.ResponseWriter, req *http.Request) error {
214	p := path.Clean(req.URL.Path)
215	if strings.HasPrefix(p, "/pkg/") {
216		p = p[len("/pkg"):]
217	}
218	if p != req.URL.Path {
219		http.Redirect(resp, req, p, http.StatusMovedPermanently)
220		return nil
221	}
222
223	if isView(req, "status.svg") {
224		s.statusSVG.ServeHTTP(resp, req)
225		return nil
226	}
227
228	if isView(req, "status.png") {
229		s.statusPNG.ServeHTTP(resp, req)
230		return nil
231	}
232
233	requestType := humanRequest
234	if s.isRobot(req) {
235		requestType = robotRequest
236	}
237
238	importPath := strings.TrimPrefix(req.URL.Path, "/")
239	pdoc, pkgs, err := s.getDoc(req.Context(), importPath, requestType)
240
241	if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
242		// To prevent dumb clients from following redirect loops, respond with
243		// status 404 if the target document is not found.
244		if _, _, err := s.getDoc(req.Context(), e.Redirect, requestType); gosrc.IsNotFound(err) {
245			return &httpError{status: http.StatusNotFound}
246		}
247		u := "/" + e.Redirect
248		if req.URL.RawQuery != "" {
249			u += "?" + req.URL.RawQuery
250		}
251		setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
252		http.Redirect(resp, req, u, http.StatusFound)
253		return nil
254	}
255	if err != nil {
256		return err
257	}
258
259	flashMessages := getFlashMessages(resp, req)
260
261	if pdoc == nil {
262		if len(pkgs) == 0 {
263			return &httpError{status: http.StatusNotFound}
264		}
265		pdocChild, _, _, err := s.db.Get(req.Context(), pkgs[0].Path)
266		if err != nil {
267			return err
268		}
269		pdoc = &doc.Package{
270			ProjectName: pdocChild.ProjectName,
271			ProjectRoot: pdocChild.ProjectRoot,
272			ProjectURL:  pdocChild.ProjectURL,
273			ImportPath:  importPath,
274		}
275	}
276
277	showPkgGoDevRedirectToast := userReturningFromPkgGoDev(req)
278
279	switch {
280	case isView(req, "imports"):
281		if pdoc.Name == "" {
282			return &httpError{status: http.StatusNotFound}
283		}
284		pkgs, err = s.db.Packages(pdoc.Imports)
285		if err != nil {
286			return err
287		}
288		return s.templates.execute(resp, "imports.html", http.StatusOK, nil, map[string]interface{}{
289			"flashMessages":             flashMessages,
290			"pkgs":                      pkgs,
291			"pdoc":                      newTDoc(s.v, pdoc),
292			"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
293		})
294	case isView(req, "tools"):
295		proto := "http"
296		if req.Host == "godoc.org" {
297			proto = "https"
298		}
299		return s.templates.execute(resp, "tools.html", http.StatusOK, nil, map[string]interface{}{
300			"flashMessages":             flashMessages,
301			"uri":                       fmt.Sprintf("%s://%s/%s", proto, req.Host, importPath),
302			"pdoc":                      newTDoc(s.v, pdoc),
303			"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
304		})
305	case isView(req, "importers"):
306		if pdoc.Name == "" {
307			return &httpError{status: http.StatusNotFound}
308		}
309		pkgs, err = s.db.Importers(importPath)
310		if err != nil {
311			return err
312		}
313		template := "importers.html"
314		if requestType == robotRequest {
315			// Hide back links from robots.
316			template = "importers_robot.html"
317		}
318		return s.templates.execute(resp, template, http.StatusOK, nil, map[string]interface{}{
319			"flashMessages":             flashMessages,
320			"pkgs":                      pkgs,
321			"pdoc":                      newTDoc(s.v, pdoc),
322			"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
323		})
324	case isView(req, "import-graph"):
325		if requestType == robotRequest {
326			return &httpError{status: http.StatusForbidden}
327		}
328		if pdoc.Name == "" {
329			return &httpError{status: http.StatusNotFound}
330		}
331
332		// Throttle ?import-graph requests.
333		select {
334		case s.importGraphSem <- struct{}{}:
335		default:
336			return &httpError{status: http.StatusTooManyRequests}
337		}
338		defer func() { <-s.importGraphSem }()
339
340		hide := database.ShowAllDeps
341		switch req.Form.Get("hide") {
342		case "1":
343			hide = database.HideStandardDeps
344		case "2":
345			hide = database.HideStandardAll
346		}
347		pkgs, edges, err := s.db.ImportGraph(pdoc, hide)
348		if err != nil {
349			return err
350		}
351		b, err := renderGraph(pdoc, pkgs, edges)
352		if err != nil {
353			return err
354		}
355		return s.templates.execute(resp, "graph.html", http.StatusOK, nil, map[string]interface{}{
356			"flashMessages":             flashMessages,
357			"svg":                       template.HTML(b),
358			"pdoc":                      newTDoc(s.v, pdoc),
359			"hide":                      hide,
360			"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
361		})
362	case isView(req, "play"):
363		u, err := s.playURL(pdoc, req.Form.Get("play"), req.Header.Get("X-AppEngine-Country"))
364		if err != nil {
365			return err
366		}
367		http.Redirect(resp, req, u, http.StatusMovedPermanently)
368		return nil
369	case req.Form.Get("view") != "":
370		// Redirect deprecated view= queries.
371		var q string
372		switch view := req.Form.Get("view"); view {
373		case "imports", "importers":
374			q = view
375		case "import-graph":
376			if req.Form.Get("hide") == "1" {
377				q = "import-graph&hide=1"
378			} else {
379				q = "import-graph"
380			}
381		}
382		if q != "" {
383			u := *req.URL
384			u.RawQuery = q
385			http.Redirect(resp, req, u.String(), http.StatusMovedPermanently)
386			return nil
387		}
388		return &httpError{status: http.StatusNotFound}
389	default:
390		importerCount := 0
391		if pdoc.Name != "" {
392			importerCount, err = s.db.ImporterCount(importPath)
393			if err != nil {
394				return err
395			}
396		}
397
398		etag := s.httpEtag(pdoc, pkgs, importerCount, flashMessages)
399		status := http.StatusOK
400		if req.Header.Get("If-None-Match") == etag {
401			status = http.StatusNotModified
402		}
403
404		if requestType == humanRequest &&
405			pdoc.Name != "" && // not a directory
406			pdoc.ProjectRoot != "" && // not a standard package
407			!pdoc.IsCmd &&
408			len(pdoc.Errors) == 0 &&
409			!popularLinkReferral(req) {
410			if err := s.db.IncrementPopularScore(pdoc.ImportPath); err != nil {
411				log.Printf("ERROR db.IncrementPopularScore(%s): %v", pdoc.ImportPath, err)
412			}
413		}
414		if s.gceLogger != nil {
415			s.gceLogger.LogEvent(resp, req, nil)
416		}
417
418		template := "dir"
419		switch {
420		case pdoc.IsCmd:
421			template = "cmd"
422		case pdoc.Name != "":
423			template = "pkg"
424		}
425		template += templateExt(req)
426
427		return s.templates.execute(resp, template, status, http.Header{"Etag": {etag}}, map[string]interface{}{
428			"flashMessages":             flashMessages,
429			"pkgs":                      pkgs,
430			"pdoc":                      newTDoc(s.v, pdoc),
431			"importerCount":             importerCount,
432			"showPkgGoDevRedirectToast": showPkgGoDevRedirectToast,
433		})
434	}
435}
436
437func (s *server) serveRefresh(resp http.ResponseWriter, req *http.Request) error {
438	importPath := req.Form.Get("path")
439	_, pkgs, _, err := s.db.Get(req.Context(), importPath)
440	if err != nil {
441		return err
442	}
443	c := make(chan error, 1)
444	go func() {
445		_, err := s.crawlDoc(req.Context(), "rfrsh", importPath, nil, len(pkgs) > 0, time.Time{})
446		c <- err
447	}()
448	select {
449	case err = <-c:
450	case <-time.After(s.v.GetDuration(ConfigGetTimeout)):
451		err = errUpdateTimeout
452	}
453	if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
454		setFlashMessages(resp, []flashMessage{{ID: "redir", Args: []string{importPath}}})
455		importPath = e.Redirect
456		err = nil
457	} else if err != nil {
458		setFlashMessages(resp, []flashMessage{{ID: "refresh", Args: []string{errorText(err)}}})
459	}
460	http.Redirect(resp, req, "/"+importPath, http.StatusFound)
461	return nil
462}
463
464func (s *server) serveGoIndex(resp http.ResponseWriter, req *http.Request) error {
465	pkgs, err := s.db.GoIndex()
466	if err != nil {
467		return err
468	}
469	return s.templates.execute(resp, "std.html", http.StatusOK, nil, map[string]interface{}{
470		"pkgs": pkgs,
471	})
472}
473
474func (s *server) serveGoSubrepoIndex(resp http.ResponseWriter, req *http.Request) error {
475	pkgs, err := s.db.GoSubrepoIndex()
476	if err != nil {
477		return err
478	}
479	return s.templates.execute(resp, "subrepo.html", http.StatusOK, nil, map[string]interface{}{
480		"pkgs": pkgs,
481	})
482}
483
484type byPath struct {
485	pkgs []database.Package
486	rank []int
487}
488
489func (bp *byPath) Len() int           { return len(bp.pkgs) }
490func (bp *byPath) Less(i, j int) bool { return bp.pkgs[i].Path < bp.pkgs[j].Path }
491func (bp *byPath) Swap(i, j int) {
492	bp.pkgs[i], bp.pkgs[j] = bp.pkgs[j], bp.pkgs[i]
493	bp.rank[i], bp.rank[j] = bp.rank[j], bp.rank[i]
494}
495
496type byRank struct {
497	pkgs []database.Package
498	rank []int
499}
500
501func (br *byRank) Len() int           { return len(br.pkgs) }
502func (br *byRank) Less(i, j int) bool { return br.rank[i] < br.rank[j] }
503func (br *byRank) Swap(i, j int) {
504	br.pkgs[i], br.pkgs[j] = br.pkgs[j], br.pkgs[i]
505	br.rank[i], br.rank[j] = br.rank[j], br.rank[i]
506}
507
508func (s *server) popular() ([]database.Package, error) {
509	const n = 25
510
511	pkgs, err := s.db.Popular(2 * n)
512	if err != nil {
513		return nil, err
514	}
515
516	rank := make([]int, len(pkgs))
517	for i := range pkgs {
518		rank[i] = i
519	}
520
521	sort.Sort(&byPath{pkgs, rank})
522
523	j := 0
524	prev := "."
525	for i, pkg := range pkgs {
526		if strings.HasPrefix(pkg.Path, prev) {
527			if rank[j-1] < rank[i] {
528				rank[j-1] = rank[i]
529			}
530			continue
531		}
532		prev = pkg.Path + "/"
533		pkgs[j] = pkg
534		rank[j] = rank[i]
535		j++
536	}
537	pkgs = pkgs[:j]
538
539	sort.Sort(&byRank{pkgs, rank})
540
541	if len(pkgs) > n {
542		pkgs = pkgs[:n]
543	}
544
545	sort.Sort(&byPath{pkgs, rank})
546
547	return pkgs, nil
548}
549
550func (s *server) serveHome(resp http.ResponseWriter, req *http.Request) error {
551	if req.URL.Path != "/" {
552		return s.servePackage(resp, req)
553	}
554
555	q := strings.TrimSpace(req.Form.Get("q"))
556	if q == "" {
557		pkgs, err := s.popular()
558		if err != nil {
559			return err
560		}
561
562		return s.templates.execute(resp, "home"+templateExt(req), http.StatusOK, nil,
563			map[string]interface{}{
564				"Popular": pkgs,
565
566				"showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
567			})
568	}
569
570	if path, ok := isBrowseURL(q); ok {
571		q = path
572	}
573
574	if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
575		pdoc, pkgs, err := s.getDoc(req.Context(), q, queryRequest)
576		if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
577			http.Redirect(resp, req, "/"+e.Redirect, http.StatusFound)
578			return nil
579		}
580		if err == nil && (pdoc != nil || len(pkgs) > 0) {
581			http.Redirect(resp, req, "/"+q, http.StatusFound)
582			return nil
583		}
584	}
585
586	pkgs, err := s.db.Search(req.Context(), q)
587	if err != nil {
588		return err
589	}
590	if s.gceLogger != nil {
591		// Log up to top 10 packages we served upon a search.
592		logPkgs := pkgs
593		if len(pkgs) > 10 {
594			logPkgs = pkgs[:10]
595		}
596		s.gceLogger.LogEvent(resp, req, logPkgs)
597	}
598
599	return s.templates.execute(resp, "results"+templateExt(req), http.StatusOK, nil,
600		map[string]interface{}{
601			"q":    q,
602			"pkgs": pkgs,
603
604			"showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
605		})
606}
607
608func (s *server) serveAbout(resp http.ResponseWriter, req *http.Request) error {
609	return s.templates.execute(resp, "about.html", http.StatusOK, nil,
610		map[string]interface{}{
611			"Host": req.Host,
612
613			"showPkgGoDevRedirectToast": userReturningFromPkgGoDev(req),
614		})
615}
616
617func (s *server) serveBot(resp http.ResponseWriter, req *http.Request) error {
618	return s.templates.execute(resp, "bot.html", http.StatusOK, nil, nil)
619}
620
621func logError(req *http.Request, err error, rv interface{}) {
622	if err != nil {
623		var buf bytes.Buffer
624		fmt.Fprintf(&buf, "Error serving %s: %v\n", req.URL, err)
625		if rv != nil {
626			fmt.Fprintln(&buf, rv)
627			buf.Write(debug.Stack())
628		}
629		log.Print(buf.String())
630	}
631}
632
633func (s *server) serveAPISearch(resp http.ResponseWriter, req *http.Request) error {
634	q := strings.TrimSpace(req.Form.Get("q"))
635
636	var pkgs []database.Package
637
638	if gosrc.IsValidRemotePath(q) || (strings.Contains(q, "/") && gosrc.IsGoRepoPath(q)) {
639		pdoc, _, err := s.getDoc(req.Context(), q, apiRequest)
640		if e, ok := err.(gosrc.NotFoundError); ok && e.Redirect != "" {
641			pdoc, _, err = s.getDoc(req.Context(), e.Redirect, robotRequest)
642		}
643		if err == nil && pdoc != nil {
644			pkgs = []database.Package{{Path: pdoc.ImportPath, Synopsis: pdoc.Synopsis}}
645		}
646	}
647
648	if pkgs == nil {
649		var err error
650		pkgs, err = s.db.Search(req.Context(), q)
651		if err != nil {
652			return err
653		}
654	}
655
656	var data = struct {
657		Results []database.Package `json:"results"`
658	}{
659		pkgs,
660	}
661	resp.Header().Set("Content-Type", jsonMIMEType)
662	return json.NewEncoder(resp).Encode(&data)
663}
664
665func (s *server) serveAPIPackages(resp http.ResponseWriter, req *http.Request) error {
666	pkgs, err := s.db.AllPackages()
667	if err != nil {
668		return err
669	}
670	data := struct {
671		Results []database.Package `json:"results"`
672	}{
673		pkgs,
674	}
675	resp.Header().Set("Content-Type", jsonMIMEType)
676	return json.NewEncoder(resp).Encode(&data)
677}
678
679func (s *server) serveAPIImporters(resp http.ResponseWriter, req *http.Request) error {
680	importPath := strings.TrimPrefix(req.URL.Path, "/importers/")
681	pkgs, err := s.db.Importers(importPath)
682	if err != nil {
683		return err
684	}
685	data := struct {
686		Results []database.Package `json:"results"`
687	}{
688		pkgs,
689	}
690	resp.Header().Set("Content-Type", jsonMIMEType)
691	return json.NewEncoder(resp).Encode(&data)
692}
693
694func (s *server) serveAPIImports(resp http.ResponseWriter, req *http.Request) error {
695	importPath := strings.TrimPrefix(req.URL.Path, "/imports/")
696	pdoc, _, err := s.getDoc(req.Context(), importPath, robotRequest)
697	if err != nil {
698		return err
699	}
700	if pdoc == nil || pdoc.Name == "" {
701		return &httpError{status: http.StatusNotFound}
702	}
703	imports, err := s.db.Packages(pdoc.Imports)
704	if err != nil {
705		return err
706	}
707	testImports, err := s.db.Packages(pdoc.TestImports)
708	if err != nil {
709		return err
710	}
711	data := struct {
712		Imports     []database.Package `json:"imports"`
713		TestImports []database.Package `json:"testImports"`
714	}{
715		imports,
716		testImports,
717	}
718	resp.Header().Set("Content-Type", jsonMIMEType)
719	return json.NewEncoder(resp).Encode(&data)
720}
721
722func serveAPIHome(resp http.ResponseWriter, req *http.Request) error {
723	return &httpError{status: http.StatusNotFound}
724}
725
726type requestCleaner struct {
727	h                 http.Handler
728	trustProxyHeaders bool
729}
730
731func (rc requestCleaner) ServeHTTP(w http.ResponseWriter, req *http.Request) {
732	req2 := new(http.Request)
733	*req2 = *req
734	if rc.trustProxyHeaders {
735		if s := req.Header.Get("X-Forwarded-For"); s != "" {
736			req2.RemoteAddr = s
737		}
738	}
739	req2.Body = http.MaxBytesReader(w, req.Body, 2048)
740	req2.ParseForm()
741	rc.h.ServeHTTP(w, req2)
742}
743
744type errorHandler struct {
745	fn    func(resp http.ResponseWriter, req *http.Request) error
746	errFn httputil.Error
747}
748
749func (eh errorHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
750	defer func() {
751		if rv := recover(); rv != nil {
752			err := errors.New("handler panic")
753			logError(req, err, rv)
754			eh.errFn(resp, req, http.StatusInternalServerError, err)
755		}
756	}()
757
758	rb := new(httputil.ResponseBuffer)
759	err := eh.fn(rb, req)
760	if err == nil {
761		rb.WriteTo(resp)
762	} else if e, ok := err.(*httpError); ok {
763		if e.status >= 500 {
764			logError(req, err, nil)
765		}
766		eh.errFn(resp, req, e.status, e.err)
767	} else if gosrc.IsNotFound(err) {
768		eh.errFn(resp, req, http.StatusNotFound, nil)
769	} else {
770		logError(req, err, nil)
771		eh.errFn(resp, req, http.StatusInternalServerError, err)
772	}
773}
774
775func errorText(err error) string {
776	if err == errUpdateTimeout {
777		return "Timeout getting package files from the version control system."
778	}
779	if e, ok := err.(*gosrc.RemoteError); ok {
780		return "Error getting package files from " + e.Host + "."
781	}
782	return "Internal server error."
783}
784
785func (s *server) handleError(resp http.ResponseWriter, req *http.Request, status int, err error) {
786	switch status {
787	case http.StatusNotFound:
788		s.templates.execute(resp, "notfound"+templateExt(req), status, nil, map[string]interface{}{
789			"flashMessages": getFlashMessages(resp, req),
790		})
791	default:
792		resp.Header().Set("Content-Type", textMIMEType)
793		resp.WriteHeader(http.StatusInternalServerError)
794		io.WriteString(resp, errorText(err))
795	}
796}
797
798func handleAPIError(resp http.ResponseWriter, req *http.Request, status int, err error) {
799	var data struct {
800		Error struct {
801			Message string `json:"message"`
802		} `json:"error"`
803	}
804	data.Error.Message = http.StatusText(status)
805	resp.Header().Set("Content-Type", jsonMIMEType)
806	resp.WriteHeader(status)
807	json.NewEncoder(resp).Encode(&data)
808}
809
810// httpsRedirectHandler redirects all requests with an X-Forwarded-Proto: http
811// handler to their https equivalent.
812type httpsRedirectHandler struct {
813	h http.Handler
814}
815
816func (h httpsRedirectHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
817	if req.Header.Get("X-Forwarded-Proto") == "http" {
818		u := *req.URL
819		u.Scheme = "https"
820		u.Host = req.Host
821		http.Redirect(resp, req, u.String(), http.StatusFound)
822		return
823	}
824	h.h.ServeHTTP(resp, req)
825}
826
827type rootHandler []struct {
828	prefix string
829	h      http.Handler
830}
831
832func (m rootHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
833	var h http.Handler
834	for _, ph := range m {
835		if strings.HasPrefix(req.Host, ph.prefix) {
836			h = ph.h
837			break
838		}
839	}
840
841	h.ServeHTTP(resp, req)
842}
843
844// otherDomainHandler redirects to another domain keeping the rest of the URL.
845type otherDomainHandler struct {
846	scheme       string
847	targetDomain string
848}
849
850func (h otherDomainHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
851	u := *req.URL
852	u.Scheme = h.scheme
853	u.Host = h.targetDomain
854	http.Redirect(w, req, u.String(), http.StatusFound)
855}
856
857func defaultBase(path string) string {
858	p, err := build.Default.Import(path, "", build.FindOnly)
859	if err != nil {
860		return "."
861	}
862	return p.Dir
863}
864
865type server struct {
866	v           *viper.Viper
867	db          *database.Database
868	httpClient  *http.Client
869	gceLogger   *GCELogger
870	templates   templateMap
871	traceClient *trace.Client
872	crawlTopic  *pubsub.Topic
873
874	statusPNG http.Handler
875	statusSVG http.Handler
876
877	root rootHandler
878
879	// A semaphore to limit concurrent ?import-graph requests.
880	importGraphSem chan struct{}
881}
882
883func newServer(ctx context.Context, v *viper.Viper) (*server, error) {
884	s := &server{
885		v:              v,
886		httpClient:     newHTTPClient(v),
887		importGraphSem: make(chan struct{}, 10),
888	}
889
890	var err error
891	if proj := s.v.GetString(ConfigProject); proj != "" {
892		if s.traceClient, err = trace.NewClient(ctx, proj); err != nil {
893			return nil, err
894		}
895		sp, err := trace.NewLimitedSampler(s.v.GetFloat64(ConfigTraceSamplerFraction), s.v.GetFloat64(ConfigTraceSamplerMaxQPS))
896		if err != nil {
897			return nil, err
898		}
899		s.traceClient.SetSamplingPolicy(sp)
900
901		// This topic should be created in the cloud console.
902		ps, err := pubsub.NewClient(ctx, proj)
903		if err != nil {
904			return nil, err
905		}
906		s.crawlTopic = ps.Topic(ConfigCrawlPubSubTopic)
907	}
908
909	assets := v.GetString(ConfigAssetsDir)
910	staticServer := httputil.StaticServer{
911		Dir:    assets,
912		MaxAge: time.Hour,
913		MIMETypes: map[string]string{
914			".css": "text/css; charset=utf-8",
915			".js":  "text/javascript; charset=utf-8",
916		},
917	}
918	s.statusPNG = staticServer.FileHandler("status.png")
919	s.statusSVG = staticServer.FileHandler("status.svg")
920
921	apiHandler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
922		return requestCleaner{
923			h: errorHandler{
924				fn:    f,
925				errFn: handleAPIError,
926			},
927			trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
928		}
929	}
930	apiMux := http.NewServeMux()
931	apiMux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
932	apiMux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
933	apiMux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
934	apiMux.Handle("/robots.txt", staticServer.FileHandler("apiRobots.txt"))
935	apiMux.Handle("/search", apiHandler(s.serveAPISearch))
936	apiMux.Handle("/packages", apiHandler(s.serveAPIPackages))
937	apiMux.Handle("/importers/", apiHandler(s.serveAPIImporters))
938	apiMux.Handle("/imports/", apiHandler(s.serveAPIImports))
939	apiMux.Handle("/", apiHandler(serveAPIHome))
940
941	mux := http.NewServeMux()
942	mux.Handle("/-/site.js", staticServer.FilesHandler(
943		"third_party/jquery.timeago.js",
944		"site.js"))
945	mux.Handle("/-/site.css", staticServer.FilesHandler("site.css"))
946	mux.Handle("/-/bootstrap.min.css", staticServer.FilesHandler("bootstrap.min.css"))
947	mux.Handle("/-/bootstrap.min.js", staticServer.FilesHandler("bootstrap.min.js"))
948	mux.Handle("/-/jquery-2.0.3.min.js", staticServer.FilesHandler("jquery-2.0.3.min.js"))
949	if s.v.GetBool(ConfigSidebar) {
950		mux.Handle("/-/sidebar.css", staticServer.FilesHandler("sidebar.css"))
951	}
952	mux.Handle("/-/", http.NotFoundHandler())
953
954	handler := func(f func(http.ResponseWriter, *http.Request) error) http.Handler {
955		return requestCleaner{
956			h: errorHandler{
957				fn:    f,
958				errFn: s.handleError,
959			},
960			trustProxyHeaders: v.GetBool(ConfigTrustProxyHeaders),
961		}
962	}
963
964	mux.Handle("/-/about", handler(pkgGoDevRedirectHandler(s.serveAbout)))
965	mux.Handle("/-/bot", handler(s.serveBot))
966	mux.Handle("/-/go", handler(pkgGoDevRedirectHandler(s.serveGoIndex)))
967	mux.Handle("/-/subrepo", handler(s.serveGoSubrepoIndex))
968	mux.Handle("/-/refresh", handler(s.serveRefresh))
969	mux.Handle("/about", http.RedirectHandler("/-/about", http.StatusMovedPermanently))
970	mux.Handle("/favicon.ico", staticServer.FileHandler("favicon.ico"))
971	mux.Handle("/google3d2f3cd4cc2bb44b.html", staticServer.FileHandler("google3d2f3cd4cc2bb44b.html"))
972	mux.Handle("/humans.txt", staticServer.FileHandler("humans.txt"))
973	mux.Handle("/robots.txt", staticServer.FileHandler("robots.txt"))
974	mux.Handle("/BingSiteAuth.xml", staticServer.FileHandler("BingSiteAuth.xml"))
975	mux.Handle("/C", http.RedirectHandler("http://golang.org/doc/articles/c_go_cgo.html", http.StatusMovedPermanently))
976	mux.Handle("/code.jquery.com/", http.NotFoundHandler())
977	mux.Handle("/", handler(pkgGoDevRedirectHandler(s.serveHome)))
978
979	ahMux := http.NewServeMux()
980	ready := new(health.Handler)
981	ahMux.HandleFunc("/_ah/health", health.HandleLive)
982	ahMux.Handle("/_ah/ready", ready)
983
984	mainMux := http.NewServeMux()
985	mainMux.Handle("/_ah/", ahMux)
986	mainMux.Handle("/", s.traceClient.HTTPHandler(mux))
987
988	s.root = rootHandler{
989		{"api.", httpsRedirectHandler{s.traceClient.HTTPHandler(apiMux)}},
990		{"talks.godoc.org", otherDomainHandler{"https", "go-talks.appspot.com"}},
991		{"", httpsRedirectHandler{mainMux}},
992	}
993
994	cacheBusters := &httputil.CacheBusters{Handler: mux}
995	s.templates, err = parseTemplates(assets, cacheBusters, v)
996	if err != nil {
997		return nil, err
998	}
999	s.db, err = database.New(
1000		v.GetString(ConfigDBServer),
1001		v.GetDuration(ConfigDBIdleTimeout),
1002		v.GetBool(ConfigDBLog),
1003		v.GetString(ConfigGAERemoteAPI),
1004	)
1005	if err != nil {
1006		return nil, fmt.Errorf("open database: %v", err)
1007	}
1008	ready.Add(s.db)
1009	if gceLogName := v.GetString(ConfigGCELogName); gceLogName != "" {
1010		logc, err := logging.NewClient(ctx, v.GetString(ConfigProject))
1011		if err != nil {
1012			return nil, fmt.Errorf("create cloud logging client: %v", err)
1013		}
1014		logger := logc.Logger(gceLogName)
1015		if err := logc.Ping(ctx); err != nil {
1016			return nil, fmt.Errorf("pinging cloud logging: %v", err)
1017		}
1018		s.gceLogger = newGCELogger(logger)
1019	}
1020	return s, nil
1021}
1022
1023const (
1024	pkgGoDevRedirectCookie = "pkggodev-redirect"
1025	pkgGoDevRedirectParam  = "redirect"
1026	pkgGoDevRedirectOn     = "on"
1027	pkgGoDevRedirectOff    = "off"
1028	pkgGoDevHost           = "pkg.go.dev"
1029	teeproxyHost           = "teeproxy-dot-go-discovery.appspot.com"
1030)
1031
1032type responseWriter struct {
1033	http.ResponseWriter
1034	status int
1035}
1036
1037func (rw *responseWriter) WriteHeader(code int) {
1038	rw.status = code
1039	rw.ResponseWriter.WriteHeader(code)
1040}
1041
1042func translateStatus(code int) int {
1043	if code == 0 {
1044		return http.StatusOK
1045	}
1046	return code
1047}
1048
1049func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
1050	start := time.Now()
1051	s.logRequestStart(r)
1052	w2 := &responseWriter{ResponseWriter: w}
1053	s.root.ServeHTTP(w2, r)
1054	latency := time.Since(start)
1055	s.logRequestEnd(r, latency)
1056	if f, ok := w.(http.Flusher); ok {
1057		f.Flush()
1058	}
1059	// Don't tee App Engine requests to pkg.go.dev.
1060	if strings.HasPrefix(r.URL.Path, "/_ah/") {
1061		return
1062	}
1063	if err := makePkgGoDevRequest(r, latency, s.isRobot(r), translateStatus(w2.status)); err != nil {
1064		log.Printf("makePkgGoDevRequest(%q, %d) error: %v", r.URL, latency, err)
1065	}
1066}
1067
1068func makePkgGoDevRequest(r *http.Request, latency time.Duration, isRobot bool, status int) error {
1069	event := newGDDOEvent(r, latency, isRobot, status)
1070	b, err := json.Marshal(event)
1071	if err != nil {
1072		return fmt.Errorf("json.Marshal(%v): %v", event, err)
1073	}
1074
1075	teeproxyURL := url.URL{Scheme: "https", Host: teeproxyHost}
1076	if _, err := http.Post(teeproxyURL.String(), jsonMIMEType, bytes.NewReader(b)); err != nil {
1077		return fmt.Errorf("http.Post(%q, %q, %v): %v", teeproxyURL.String(), jsonMIMEType, event, err)
1078	}
1079	log.Printf("makePkgGoDevRequest: request made to %q for %+v", teeproxyURL.String(), event)
1080	return nil
1081}
1082
1083type gddoEvent struct {
1084	Host        string
1085	Path        string
1086	Status      int
1087	URL         string
1088	Header      http.Header
1089	Latency     time.Duration
1090	IsRobot     bool
1091	UsePkgGoDev bool
1092}
1093
1094func newGDDOEvent(r *http.Request, latency time.Duration, isRobot bool, status int) *gddoEvent {
1095	targetURL := url.URL{
1096		Scheme:   "https",
1097		Host:     r.URL.Host,
1098		Path:     r.URL.Path,
1099		RawQuery: r.URL.RawQuery,
1100	}
1101	if targetURL.Host == "" && r.Host != "" {
1102		targetURL.Host = r.Host
1103	}
1104	return &gddoEvent{
1105		Host:        targetURL.Host,
1106		Path:        r.URL.Path,
1107		Status:      status,
1108		URL:         targetURL.String(),
1109		Header:      r.Header,
1110		Latency:     latency,
1111		IsRobot:     isRobot,
1112		UsePkgGoDev: shouldRedirectToPkgGoDev(r),
1113	}
1114}
1115
1116func (s *server) logRequestStart(req *http.Request) {
1117	if s.gceLogger == nil {
1118		return
1119	}
1120	s.gceLogger.Log(logging.Entry{
1121		HTTPRequest: &logging.HTTPRequest{Request: req},
1122		Payload:     fmt.Sprintf("%s request start", req.Host),
1123		Severity:    logging.Info,
1124	})
1125}
1126
1127func (s *server) logRequestEnd(req *http.Request, latency time.Duration) {
1128	if s.gceLogger == nil {
1129		return
1130	}
1131	s.gceLogger.Log(logging.Entry{
1132		HTTPRequest: &logging.HTTPRequest{
1133			Request: req,
1134			Latency: latency,
1135		},
1136		Payload:  fmt.Sprintf("%s request end", req.Host),
1137		Severity: logging.Info,
1138	})
1139}
1140
1141func userReturningFromPkgGoDev(req *http.Request) bool {
1142	return req.FormValue("utm_source") == "backtogodoc"
1143}
1144
1145func shouldRedirectToPkgGoDev(req *http.Request) bool {
1146	// API requests are not redirected.
1147	if strings.HasPrefix(req.URL.Host, "api") {
1148		return false
1149	}
1150	redirectParam := req.FormValue(pkgGoDevRedirectParam)
1151	if redirectParam == pkgGoDevRedirectOn || redirectParam == pkgGoDevRedirectOff {
1152		return redirectParam == pkgGoDevRedirectOn
1153	}
1154	cookie, err := req.Cookie(pkgGoDevRedirectCookie)
1155	return (err == nil && cookie.Value == pkgGoDevRedirectOn)
1156}
1157
1158// pkgGoDevRedirectHandler redirects requests from godoc.org to pkg.go.dev,
1159// based on whether a cookie is set for pkggodev-redirect. The cookie
1160// can be turned on/off using a query param.
1161func pkgGoDevRedirectHandler(f func(http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) error {
1162	return func(w http.ResponseWriter, r *http.Request) error {
1163		if userReturningFromPkgGoDev(r) {
1164			return f(w, r)
1165		}
1166
1167		redirectParam := r.FormValue(pkgGoDevRedirectParam)
1168
1169		if redirectParam == pkgGoDevRedirectOn {
1170			cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: redirectParam, Path: "/"}
1171			http.SetCookie(w, cookie)
1172		}
1173		if redirectParam == pkgGoDevRedirectOff {
1174			cookie := &http.Cookie{Name: pkgGoDevRedirectCookie, Value: "", MaxAge: -1, Path: "/"}
1175			http.SetCookie(w, cookie)
1176		}
1177
1178		if !shouldRedirectToPkgGoDev(r) {
1179			return f(w, r)
1180		}
1181
1182		http.Redirect(w, r, pkgGoDevURL(r.URL).String(), http.StatusFound)
1183		return nil
1184	}
1185}
1186
1187func pkgGoDevURL(godocURL *url.URL) *url.URL {
1188	u := &url.URL{Scheme: "https", Host: pkgGoDevHost}
1189	q := url.Values{"utm_source": []string{"godoc"}}
1190
1191	if strings.Contains(godocURL.Path, "/vendor/") || strings.HasSuffix(godocURL.Path, "/vendor") {
1192		u.Path = "/"
1193		u.RawQuery = q.Encode()
1194		return u
1195	}
1196
1197	switch godocURL.Path {
1198	case "/-/go":
1199		u.Path = "/std"
1200		q.Add("tab", "packages")
1201	case "/-/about":
1202		u.Path = "/about"
1203	case "/":
1204		if qparam := godocURL.Query().Get("q"); qparam != "" {
1205			u.Path = "/search"
1206			q.Set("q", qparam)
1207		} else {
1208			u.Path = "/"
1209		}
1210	default:
1211		{
1212			u.Path = godocURL.Path
1213			if _, ok := godocURL.Query()["imports"]; ok {
1214				q.Set("tab", "imports")
1215			} else if _, ok := godocURL.Query()["importers"]; ok {
1216				q.Set("tab", "importedby")
1217			} else {
1218				q.Set("tab", "doc")
1219			}
1220		}
1221	}
1222
1223	u.RawQuery = q.Encode()
1224	return u
1225}
1226
1227func main() {
1228	ctx := context.Background()
1229	v, err := loadConfig(ctx, os.Args)
1230	if err != nil {
1231		log.Fatal(ctx, "load config", "error", err.Error())
1232	}
1233	doc.SetDefaultGOOS(v.GetString(ConfigDefaultGOOS))
1234
1235	s, err := newServer(ctx, v)
1236	if err != nil {
1237		log.Fatal("error creating server:", err)
1238	}
1239
1240	go func() {
1241		for range time.Tick(s.v.GetDuration(ConfigCrawlInterval)) {
1242			if err := s.doCrawl(ctx); err != nil {
1243				log.Printf("Task Crawl: %v", err)
1244			}
1245		}
1246	}()
1247	go func() {
1248		for range time.Tick(s.v.GetDuration(ConfigGithubInterval)) {
1249			if err := s.readGitHubUpdates(ctx); err != nil {
1250				log.Printf("Task GitHub updates: %v", err)
1251			}
1252		}
1253	}()
1254	http.Handle("/", s)
1255	log.Fatal(http.ListenAndServe(s.v.GetString(ConfigBindAddress), s))
1256}
1257