1/* 2Copyright 2013 The Perkeep Authors 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package main 18 19import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "flag" 24 "fmt" 25 "log" 26 "net/http" 27 "os" 28 "os/exec" 29 "strings" 30 "sync" 31 "time" 32 33 "cloud.google.com/go/datastore" 34 "github.com/mailgun/mailgun-go" 35 36 "perkeep.org/internal/osutil" 37) 38 39var ( 40 emailNow = flag.String("email_now", "", "[debug] if non-empty, this commit hash is emailed immediately, without starting the webserver.") 41 mailgunCfgFile = flag.String("mailgun_config", "", "[optional] Mailgun JSON configuration for sending emails on new commits.") 42 emailsTo = flag.String("email_dest", "", "[optional] The email address for new commit emails.") 43) 44 45type mailgunCfg struct { 46 Domain string `json:"domain"` 47 APIKey string `json:"apiKey"` 48 PublicAPIKey string `json:"publicAPIKey"` 49} 50 51// mailgun is for sending the camweb startup e-mail, and the commits e-mails. No 52// e-mails are sent if it is nil. It is set in sendStartingEmail, and it is nil 53// if mailgunCfgFile is not set. 54var mailGun mailgun.Mailgun 55 56func mailgunCfgFromGCS() (*mailgunCfg, error) { 57 var cfg mailgunCfg 58 data, err := fromGCS(*mailgunCfgFile) 59 if err != nil { 60 return nil, err 61 } 62 if err := json.Unmarshal(data, &cfg); err != nil { 63 return nil, fmt.Errorf("could not JSON decode website's mailgun config: %v", err) 64 } 65 return &cfg, nil 66} 67 68func startEmailCommitLoop(errc chan<- error) { 69 if *emailsTo == "" { 70 return 71 } 72 if *emailNow != "" { 73 dir, err := osutil.GoPackagePath(prodDomain) 74 if err != nil { 75 log.Fatal(err) 76 } 77 if err := emailCommit(dir, *emailNow); err != nil { 78 log.Fatal(err) 79 } 80 os.Exit(0) 81 } 82 go func() { 83 errc <- commitEmailLoop() 84 }() 85} 86 87// tokenc holds tokens for the /mailnow handler. 88// Hitting /mailnow (unauthenticated) forces a 'git fetch origin 89// master'. Because it's unauthenticated, we don't want to allow 90// attackers to force us to hit git. The /mailnow handler tries to 91// take a token from tokenc. 92var tokenc = make(chan bool, 3) 93 94var fetchc = make(chan bool, 1) 95 96var knownCommit = map[string]bool{} // commit -> true 97 98var diffMarker = []byte("diff --git a/") 99 100func emailCommit(dir, hash string) (err error) { 101 if mailGun == nil { 102 return nil 103 } 104 105 var body []byte 106 if err := emailOnTimeout("git show", 2*time.Minute, func() error { 107 cmd := execGit(dir, "show", nil, "show", hash) 108 body, err = cmd.CombinedOutput() 109 if err != nil { 110 return fmt.Errorf("error runnning git show: %v\n%s", err, body) 111 } 112 return nil 113 }); err != nil { 114 return err 115 } 116 if !bytes.Contains(body, diffMarker) { 117 // Boring merge commit. Don't email. 118 return nil 119 } 120 121 var out []byte 122 if err := emailOnTimeout("git show_pretty", 2*time.Minute, func() error { 123 cmd := execGit(dir, "show_pretty", nil, "show", "--pretty=oneline", hash) 124 out, err = cmd.Output() 125 if err != nil { 126 return fmt.Errorf("error runnning git show_pretty: %v\n%s", err, out) 127 } 128 return nil 129 }); err != nil { 130 return err 131 } 132 subj := out[41:] // remove hash and space 133 if i := bytes.IndexByte(subj, '\n'); i != -1 { 134 subj = subj[:i] 135 } 136 if len(subj) > 80 { 137 subj = subj[:80] 138 } 139 140 contents := fmt.Sprintf(` 141 142https://github.com/perkeep/perkeep/commit/%s 143 144%s`, hash, body) 145 146 m := mailGun.NewMessage( 147 "noreply@perkeep.org", 148 string(subj), 149 contents, 150 *emailsTo, 151 ) 152 m.SetReplyTo("camlistore-commits@googlegroups.com") 153 if _, _, err := mailGun.Send(m); err != nil { 154 return fmt.Errorf("failed to send e-mail: %v", err) 155 } 156 return nil 157} 158 159var latestHash struct { 160 sync.Mutex 161 s string // hash of the most recent perkeep revision 162} 163 164// dsClient is our datastore client to track which commits we've 165// emailed about. It's only non-nil in production. 166var dsClient *datastore.Client 167 168func commitEmailLoop() error { 169 http.HandleFunc("/mailnow", mailNowHandler) 170 171 var err error 172 dsClient, err = datastore.NewClient(context.Background(), "camlistore-website") 173 log.Printf("datastore = %v, %v", dsClient, err) 174 175 go func() { 176 for { 177 select { 178 case tokenc <- true: 179 default: 180 } 181 time.Sleep(15 * time.Second) 182 } 183 }() 184 185 dir := pkSrcDir() 186 187 http.HandleFunc("/latesthash", latestHashHandler) 188 http.HandleFunc("/debug/email", func(w http.ResponseWriter, r *http.Request) { 189 fmt.Fprintf(w, "ds = %v, %v", dsClient, err) 190 }) 191 192 for { 193 pollCommits(dir) 194 195 // Poll every minute or whenever we're forced with the 196 // /mailnow handler. 197 select { 198 case <-time.After(1 * time.Minute): 199 case <-fetchc: 200 log.Printf("Polling git due to explicit trigger.") 201 } 202 } 203} 204 205// emailOnTimeout runs fn in a goroutine. If fn is not done after d, 206// a message about fnName is logged, and an e-mail about it is sent. 207func emailOnTimeout(fnName string, d time.Duration, fn func() error) error { 208 c := make(chan error, 1) 209 go func() { 210 c <- fn() 211 }() 212 select { 213 case <-time.After(d): 214 log.Printf("timeout for %s, sending e-mail about it", fnName) 215 m := mailGun.NewMessage( 216 "noreply@perkeep.org", 217 "timeout for docker on pk-web", 218 "Because "+fnName+" is stuck.", 219 "mathieu.lonjaret@gmail.com", 220 ) 221 if _, _, err := mailGun.Send(m); err != nil { 222 return fmt.Errorf("failed to send docker restart e-mail: %v", err) 223 } 224 return nil 225 case err := <-c: 226 return err 227 } 228} 229 230// execGit runs the git command with gitArgs. All the other arguments are only 231// relevant if *gitContainer, in which case we run in a docker container. 232func execGit(workdir string, containerName string, mounts map[string]string, gitArgs ...string) *exec.Cmd { 233 var cmd *exec.Cmd 234 if *gitContainer { 235 removeContainer(containerName) 236 args := []string{ 237 "run", 238 "--rm", 239 "--name=" + containerName, 240 } 241 for host, container := range mounts { 242 args = append(args, "-v", host+":"+container+":ro") 243 } 244 args = append(args, []string{ 245 "-v", workdir + ":" + workdir, 246 "--workdir=" + workdir, 247 "camlistore/git", 248 "git"}...) 249 args = append(args, gitArgs...) 250 cmd = exec.Command("docker", args...) 251 } else { 252 cmd = exec.Command("git", gitArgs...) 253 cmd.Dir = workdir 254 } 255 return cmd 256} 257 258// GitCommit is a datastore entity to track which commits we've 259// already emailed about. 260type GitCommit struct { 261 Emailed bool 262} 263 264func pollCommits(dir string) { 265 if err := emailOnTimeout("git pull_origin", 5*time.Minute, func() error { 266 cmd := execGit(dir, "pull_origin", nil, "pull", "origin") 267 out, err := cmd.CombinedOutput() 268 if err != nil { 269 return fmt.Errorf("error running git pull origin master in %s: %v\n%s", dir, err, out) 270 } 271 return nil 272 }); err != nil { 273 log.Printf("%v", err) 274 return 275 } 276 log.Printf("Ran git pull.") 277 // TODO: see if .git/refs/remotes/origin/master 278 // changed. (quicker than running recentCommits each time) 279 280 hashes, err := recentCommits(dir) 281 if err != nil { 282 log.Print(err) 283 return 284 } 285 if len(hashes) == 0 { 286 return 287 } 288 latestHash.Lock() 289 latestHash.s = hashes[0] 290 latestHash.Unlock() 291 for _, commit := range hashes { 292 if knownCommit[commit] { 293 continue 294 } 295 if dsClient != nil { 296 ctx := context.Background() 297 key := datastore.NameKey("git_commit", commit, nil) 298 var gc GitCommit 299 if err := dsClient.Get(ctx, key, &gc); err == nil && gc.Emailed { 300 log.Printf("Already emailed about commit %v; skipping", commit) 301 knownCommit[commit] = true 302 continue 303 } 304 } 305 if err := emailCommit(dir, commit); err != nil { 306 log.Printf("Error with commit e-mail: %v", err) 307 continue 308 } 309 log.Printf("Emailed commit %s", commit) 310 knownCommit[commit] = true 311 if dsClient != nil { 312 ctx := context.Background() 313 key := datastore.NameKey("git_commit", commit, nil) 314 _, err := dsClient.Put(ctx, key, &GitCommit{Emailed: true}) 315 log.Printf("datastore put of git_commit(%v): %v", commit, err) 316 } 317 } 318} 319 320func recentCommits(dir string) (hashes []string, err error) { 321 var out []byte 322 if err := emailOnTimeout("git log_origin_master", 2*time.Minute, func() error { 323 cmd := execGit(dir, "log_origin_master", nil, "log", "--since=1 month ago", "--pretty=oneline", "origin/master") 324 out, err = cmd.CombinedOutput() 325 if err != nil { 326 return fmt.Errorf("error running git log in %s: %v\n%s", dir, err, out) 327 } 328 return nil 329 }); err != nil { 330 return nil, err 331 } 332 for _, line := range strings.Split(string(out), "\n") { 333 v := strings.SplitN(line, " ", 2) 334 if len(v) > 1 { 335 hashes = append(hashes, v[0]) 336 } 337 } 338 return 339} 340 341func mailNowHandler(w http.ResponseWriter, r *http.Request) { 342 select { 343 case <-tokenc: 344 log.Printf("/mailnow got a token") 345 default: 346 // Too many requests. Ignore. 347 log.Printf("Ignoring /mailnow request; too soon.") 348 return 349 } 350 select { 351 case fetchc <- true: 352 log.Printf("/mailnow triggered a git fetch") 353 default: 354 } 355} 356 357func latestHashHandler(w http.ResponseWriter, r *http.Request) { 358 latestHash.Lock() 359 defer latestHash.Unlock() 360 fmt.Fprint(w, latestHash.s) 361} 362