1/*
2Copyright 2013 The Perkeep Authors
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8     http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package main
18
19import (
20	"bytes"
21	"context"
22	"encoding/json"
23	"flag"
24	"fmt"
25	"log"
26	"net/http"
27	"os"
28	"os/exec"
29	"strings"
30	"sync"
31	"time"
32
33	"cloud.google.com/go/datastore"
34	"github.com/mailgun/mailgun-go"
35
36	"perkeep.org/internal/osutil"
37)
38
39var (
40	emailNow       = flag.String("email_now", "", "[debug] if non-empty, this commit hash is emailed immediately, without starting the webserver.")
41	mailgunCfgFile = flag.String("mailgun_config", "", "[optional] Mailgun JSON configuration for sending emails on new commits.")
42	emailsTo       = flag.String("email_dest", "", "[optional] The email address for new commit emails.")
43)
44
45type mailgunCfg struct {
46	Domain       string `json:"domain"`
47	APIKey       string `json:"apiKey"`
48	PublicAPIKey string `json:"publicAPIKey"`
49}
50
51// mailgun is for sending the camweb startup e-mail, and the commits e-mails. No
52// e-mails are sent if it is nil. It is set in sendStartingEmail, and it is nil
53// if mailgunCfgFile is not set.
54var mailGun mailgun.Mailgun
55
56func mailgunCfgFromGCS() (*mailgunCfg, error) {
57	var cfg mailgunCfg
58	data, err := fromGCS(*mailgunCfgFile)
59	if err != nil {
60		return nil, err
61	}
62	if err := json.Unmarshal(data, &cfg); err != nil {
63		return nil, fmt.Errorf("could not JSON decode website's mailgun config: %v", err)
64	}
65	return &cfg, nil
66}
67
68func startEmailCommitLoop(errc chan<- error) {
69	if *emailsTo == "" {
70		return
71	}
72	if *emailNow != "" {
73		dir, err := osutil.GoPackagePath(prodDomain)
74		if err != nil {
75			log.Fatal(err)
76		}
77		if err := emailCommit(dir, *emailNow); err != nil {
78			log.Fatal(err)
79		}
80		os.Exit(0)
81	}
82	go func() {
83		errc <- commitEmailLoop()
84	}()
85}
86
87// tokenc holds tokens for the /mailnow handler.
88// Hitting /mailnow (unauthenticated) forces a 'git fetch origin
89// master'.  Because it's unauthenticated, we don't want to allow
90// attackers to force us to hit git. The /mailnow handler tries to
91// take a token from tokenc.
92var tokenc = make(chan bool, 3)
93
94var fetchc = make(chan bool, 1)
95
96var knownCommit = map[string]bool{} // commit -> true
97
98var diffMarker = []byte("diff --git a/")
99
100func emailCommit(dir, hash string) (err error) {
101	if mailGun == nil {
102		return nil
103	}
104
105	var body []byte
106	if err := emailOnTimeout("git show", 2*time.Minute, func() error {
107		cmd := execGit(dir, "show", nil, "show", hash)
108		body, err = cmd.CombinedOutput()
109		if err != nil {
110			return fmt.Errorf("error runnning git show: %v\n%s", err, body)
111		}
112		return nil
113	}); err != nil {
114		return err
115	}
116	if !bytes.Contains(body, diffMarker) {
117		// Boring merge commit. Don't email.
118		return nil
119	}
120
121	var out []byte
122	if err := emailOnTimeout("git show_pretty", 2*time.Minute, func() error {
123		cmd := execGit(dir, "show_pretty", nil, "show", "--pretty=oneline", hash)
124		out, err = cmd.Output()
125		if err != nil {
126			return fmt.Errorf("error runnning git show_pretty: %v\n%s", err, out)
127		}
128		return nil
129	}); err != nil {
130		return err
131	}
132	subj := out[41:] // remove hash and space
133	if i := bytes.IndexByte(subj, '\n'); i != -1 {
134		subj = subj[:i]
135	}
136	if len(subj) > 80 {
137		subj = subj[:80]
138	}
139
140	contents := fmt.Sprintf(`
141
142https://github.com/perkeep/perkeep/commit/%s
143
144%s`, hash, body)
145
146	m := mailGun.NewMessage(
147		"noreply@perkeep.org",
148		string(subj),
149		contents,
150		*emailsTo,
151	)
152	m.SetReplyTo("camlistore-commits@googlegroups.com")
153	if _, _, err := mailGun.Send(m); err != nil {
154		return fmt.Errorf("failed to send e-mail: %v", err)
155	}
156	return nil
157}
158
159var latestHash struct {
160	sync.Mutex
161	s string // hash of the most recent perkeep revision
162}
163
164// dsClient is our datastore client to track which commits we've
165// emailed about. It's only non-nil in production.
166var dsClient *datastore.Client
167
168func commitEmailLoop() error {
169	http.HandleFunc("/mailnow", mailNowHandler)
170
171	var err error
172	dsClient, err = datastore.NewClient(context.Background(), "camlistore-website")
173	log.Printf("datastore = %v, %v", dsClient, err)
174
175	go func() {
176		for {
177			select {
178			case tokenc <- true:
179			default:
180			}
181			time.Sleep(15 * time.Second)
182		}
183	}()
184
185	dir := pkSrcDir()
186
187	http.HandleFunc("/latesthash", latestHashHandler)
188	http.HandleFunc("/debug/email", func(w http.ResponseWriter, r *http.Request) {
189		fmt.Fprintf(w, "ds = %v, %v", dsClient, err)
190	})
191
192	for {
193		pollCommits(dir)
194
195		// Poll every minute or whenever we're forced with the
196		// /mailnow handler.
197		select {
198		case <-time.After(1 * time.Minute):
199		case <-fetchc:
200			log.Printf("Polling git due to explicit trigger.")
201		}
202	}
203}
204
205// emailOnTimeout runs fn in a goroutine. If fn is not done after d,
206// a message about fnName is logged, and an e-mail about it is sent.
207func emailOnTimeout(fnName string, d time.Duration, fn func() error) error {
208	c := make(chan error, 1)
209	go func() {
210		c <- fn()
211	}()
212	select {
213	case <-time.After(d):
214		log.Printf("timeout for %s, sending e-mail about it", fnName)
215		m := mailGun.NewMessage(
216			"noreply@perkeep.org",
217			"timeout for docker on pk-web",
218			"Because "+fnName+" is stuck.",
219			"mathieu.lonjaret@gmail.com",
220		)
221		if _, _, err := mailGun.Send(m); err != nil {
222			return fmt.Errorf("failed to send docker restart e-mail: %v", err)
223		}
224		return nil
225	case err := <-c:
226		return err
227	}
228}
229
230// execGit runs the git command with gitArgs. All the other arguments are only
231// relevant if *gitContainer, in which case we run in a docker container.
232func execGit(workdir string, containerName string, mounts map[string]string, gitArgs ...string) *exec.Cmd {
233	var cmd *exec.Cmd
234	if *gitContainer {
235		removeContainer(containerName)
236		args := []string{
237			"run",
238			"--rm",
239			"--name=" + containerName,
240		}
241		for host, container := range mounts {
242			args = append(args, "-v", host+":"+container+":ro")
243		}
244		args = append(args, []string{
245			"-v", workdir + ":" + workdir,
246			"--workdir=" + workdir,
247			"camlistore/git",
248			"git"}...)
249		args = append(args, gitArgs...)
250		cmd = exec.Command("docker", args...)
251	} else {
252		cmd = exec.Command("git", gitArgs...)
253		cmd.Dir = workdir
254	}
255	return cmd
256}
257
258// GitCommit is a datastore entity to track which commits we've
259// already emailed about.
260type GitCommit struct {
261	Emailed bool
262}
263
264func pollCommits(dir string) {
265	if err := emailOnTimeout("git pull_origin", 5*time.Minute, func() error {
266		cmd := execGit(dir, "pull_origin", nil, "pull", "origin")
267		out, err := cmd.CombinedOutput()
268		if err != nil {
269			return fmt.Errorf("error running git pull origin master in %s: %v\n%s", dir, err, out)
270		}
271		return nil
272	}); err != nil {
273		log.Printf("%v", err)
274		return
275	}
276	log.Printf("Ran git pull.")
277	// TODO: see if .git/refs/remotes/origin/master
278	// changed. (quicker than running recentCommits each time)
279
280	hashes, err := recentCommits(dir)
281	if err != nil {
282		log.Print(err)
283		return
284	}
285	if len(hashes) == 0 {
286		return
287	}
288	latestHash.Lock()
289	latestHash.s = hashes[0]
290	latestHash.Unlock()
291	for _, commit := range hashes {
292		if knownCommit[commit] {
293			continue
294		}
295		if dsClient != nil {
296			ctx := context.Background()
297			key := datastore.NameKey("git_commit", commit, nil)
298			var gc GitCommit
299			if err := dsClient.Get(ctx, key, &gc); err == nil && gc.Emailed {
300				log.Printf("Already emailed about commit %v; skipping", commit)
301				knownCommit[commit] = true
302				continue
303			}
304		}
305		if err := emailCommit(dir, commit); err != nil {
306			log.Printf("Error with commit e-mail: %v", err)
307			continue
308		}
309		log.Printf("Emailed commit %s", commit)
310		knownCommit[commit] = true
311		if dsClient != nil {
312			ctx := context.Background()
313			key := datastore.NameKey("git_commit", commit, nil)
314			_, err := dsClient.Put(ctx, key, &GitCommit{Emailed: true})
315			log.Printf("datastore put of git_commit(%v): %v", commit, err)
316		}
317	}
318}
319
320func recentCommits(dir string) (hashes []string, err error) {
321	var out []byte
322	if err := emailOnTimeout("git log_origin_master", 2*time.Minute, func() error {
323		cmd := execGit(dir, "log_origin_master", nil, "log", "--since=1 month ago", "--pretty=oneline", "origin/master")
324		out, err = cmd.CombinedOutput()
325		if err != nil {
326			return fmt.Errorf("error running git log in %s: %v\n%s", dir, err, out)
327		}
328		return nil
329	}); err != nil {
330		return nil, err
331	}
332	for _, line := range strings.Split(string(out), "\n") {
333		v := strings.SplitN(line, " ", 2)
334		if len(v) > 1 {
335			hashes = append(hashes, v[0])
336		}
337	}
338	return
339}
340
341func mailNowHandler(w http.ResponseWriter, r *http.Request) {
342	select {
343	case <-tokenc:
344		log.Printf("/mailnow got a token")
345	default:
346		// Too many requests. Ignore.
347		log.Printf("Ignoring /mailnow request; too soon.")
348		return
349	}
350	select {
351	case fetchc <- true:
352		log.Printf("/mailnow triggered a git fetch")
353	default:
354	}
355}
356
357func latestHashHandler(w http.ResponseWriter, r *http.Request) {
358	latestHash.Lock()
359	defer latestHash.Unlock()
360	fmt.Fprint(w, latestHash.s)
361}
362