1// +build ignore 2 3/* 4Copyright 2013 The Perkeep Authors 5 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17*/ 18 19// This program builds Perkeep. 20// 21// $ go run make.go 22// 23// See the BUILDING file. 24// 25// The output binaries go into the ./bin/ directory (under the 26// Perkeep root, where make.go is) 27package main 28 29import ( 30 "archive/zip" 31 "bytes" 32 "crypto/sha256" 33 "errors" 34 "flag" 35 "fmt" 36 "io" 37 "io/ioutil" 38 "log" 39 "net/http" 40 "os" 41 "os/exec" 42 pathpkg "path" 43 "path/filepath" 44 "regexp" 45 "runtime" 46 "strconv" 47 "strings" 48 "time" 49) 50 51var haveSQLite = checkHaveSQLite() 52 53var ( 54 embedResources = flag.Bool("embed_static", true, "Whether to embed resources needed by the UI such as images, css, and javascript.") 55 sqlFlag = flag.String("sqlite", "false", "Whether you want SQLite in your build: true, false, or auto.") 56 race = flag.Bool("race", false, "Build race-detector version of binaries (they will run slowly)") 57 verbose = flag.Bool("v", strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "makego"), "Verbose mode") 58 targets = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. '*' builds everything. Empty builds defaults for this platform. Example: perkeep.org/server/perkeepd,perkeep.org/cmd/pk-put") 59 quiet = flag.Bool("quiet", false, "Don't print anything unless there's a failure.") 60 buildARCH = flag.String("arch", runtime.GOARCH, "Architecture to build for.") 61 buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.") 62 buildARM = flag.String("arm", "7", "ARM version to use if building for ARM. Note that this version applies even if the host arch is ARM too (and possibly of a different version).") 63 stampVersion = flag.Bool("stampversion", true, "Stamp version into buildinfo.GitInfo") 64 website = flag.Bool("website", false, "Just build the website.") 65 camnetdns = flag.Bool("camnetdns", false, "Just build perkeep.org/server/camnetdns.") 66 static = flag.Bool("static", false, "Build a static binary, so it can run in an empty container.") 67 buildWebUI = flag.Bool("buildWebUI", false, "Rebuild the JS code of the web UI instead of fetching it from perkeep.org.") 68 offline = flag.Bool("offline", false, "Do not fetch the JS code for the web UI from perkeep.org. If not rebuilding the web UI, just trust the files on disk (if they exist).") 69) 70 71var ( 72 // pkRoot is the Perkeep project root 73 pkRoot string 74 binDir string // $GOBIN or $GOPATH/bin, based on user setting or default Go value. 75 76 // gopherjsGoroot should be specified through the env var 77 // CAMLI_GOPHERJS_GOROOT when the user's using go tip, because gopherjs only 78 // builds with Go 1.12. 79 gopherjsGoroot string 80) 81 82func main() { 83 log.SetFlags(0) 84 flag.Parse() 85 86 if *buildARCH == "386" && *buildOS == "darwin" { 87 if ok, _ := strconv.ParseBool(os.Getenv("CAMLI_FORCE_OSARCH")); !ok { 88 log.Fatalf("You're trying to build a 32-bit binary for a Mac. That is almost always a mistake.\nTo do it anyway, set env CAMLI_FORCE_OSARCH=1 and run again.\n") 89 } 90 } 91 92 if *website && *camnetdns { 93 log.Fatal("-camnetdns and -website are mutually exclusive") 94 } 95 96 failIfCamlistoreOrgDir() 97 verifyGoVersion() 98 verifyPerkeepRoot() 99 version := getVersion() 100 gitRev := getGitVersion() 101 sql := withSQLite() 102 103 if *verbose { 104 log.Printf("Perkeep version = %q, git = %q", version, gitRev) 105 log.Printf("SQLite included: %v", sql) 106 log.Printf("Project source: %s", pkRoot) 107 log.Printf("Output binaries: %s", actualBinDir()) 108 } 109 110 buildAll := false 111 targs := []string{ 112 "perkeep.org/dev/devcam", 113 "perkeep.org/cmd/pk-get", 114 "perkeep.org/cmd/pk-put", 115 "perkeep.org/cmd/pk", 116 "perkeep.org/cmd/pk-deploy", 117 "perkeep.org/server/perkeepd", 118 "perkeep.org/app/hello", 119 "perkeep.org/app/publisher", 120 "perkeep.org/app/scanningcabinet", 121 "perkeep.org/app/scanningcabinet/scancab", 122 } 123 switch *targets { 124 case "*": 125 buildAll = true 126 case "": 127 // Add pk-mount to default build targets on OSes that support FUSE. 128 switch *buildOS { 129 case "linux", "darwin": 130 targs = append(targs, "perkeep.org/cmd/pk-mount") 131 } 132 default: 133 if *website { 134 log.Fatal("-targets and -website are mutually exclusive") 135 } 136 if *camnetdns { 137 log.Fatal("-targets and -camnetdns are mutually exclusive") 138 } 139 if t := strings.Split(*targets, ","); len(t) != 0 { 140 targs = t 141 } 142 } 143 if *website || *camnetdns { 144 buildAll = false 145 if *website { 146 targs = []string{"perkeep.org/website/pk-web"} 147 } else if *camnetdns { 148 targs = []string{"perkeep.org/server/camnetdns"} 149 } 150 } 151 152 withPerkeepd := stringListContains(targs, "perkeep.org/server/perkeepd") 153 withPublisher := stringListContains(targs, "perkeep.org/app/publisher") 154 if err := doUI(withPerkeepd, withPublisher); err != nil { 155 log.Fatal(err) 156 } 157 158 if *embedResources && withPerkeepd { 159 doEmbed() 160 } 161 162 tags := []string{"purego"} // for cznic/zappy 163 if *static { 164 tags = append(tags, "netgo") 165 } 166 if sql { 167 // used by go-sqlite to use system sqlite libraries 168 tags = append(tags, "libsqlite3") 169 // used by perkeep to switch behavior to sqlite for tests 170 // and some underlying libraries 171 tags = append(tags, "with_sqlite") 172 } 173 if *embedResources { 174 tags = append(tags, "with_embed") 175 } 176 baseArgs := []string{"install", "-v"} 177 if *race { 178 baseArgs = append(baseArgs, "-race") 179 } 180 if *verbose { 181 log.Printf("version to stamp is %q, %q", version, gitRev) 182 } 183 var ldFlags string 184 if *static { 185 ldFlags = "-w -d -linkmode internal" 186 } 187 if *stampVersion { 188 if ldFlags != "" { 189 ldFlags += " " 190 } 191 ldFlags += "-X \"perkeep.org/pkg/buildinfo.GitInfo=" + gitRev + "\"" 192 ldFlags += "-X \"perkeep.org/pkg/buildinfo.Version=" + version + "\"" 193 } 194 if ldFlags != "" { 195 baseArgs = append(baseArgs, "--ldflags="+ldFlags) 196 } 197 baseArgs = append(baseArgs, "--tags="+strings.Join(tags, " ")) 198 199 // First install command: build just the final binaries, installed to a GOBIN 200 // under <perkeep_root>/bin: 201 args := append(baseArgs, targs...) 202 203 if buildAll { 204 args = append(args, 205 "perkeep.org/app/...", 206 "perkeep.org/pkg/...", 207 "perkeep.org/server/...", 208 "perkeep.org/internal/...", 209 ) 210 } 211 212 cmd := exec.Command("go", args...) 213 cmd.Env = cleanGoEnv() 214 if *static { 215 cmd.Env = append(cmd.Env, "CGO_ENABLED=0") 216 } 217 218 if *verbose { 219 log.Printf("Running go %q with Env %q", args, cmd.Env) 220 } 221 222 var output bytes.Buffer 223 if *quiet { 224 cmd.Stdout = &output 225 cmd.Stderr = &output 226 } else { 227 cmd.Stdout = os.Stdout 228 cmd.Stderr = os.Stderr 229 } 230 if *verbose { 231 log.Printf("Running go install of main binaries with args %s", cmd.Args) 232 } 233 if err := cmd.Run(); err != nil { 234 log.Fatalf("Error building main binaries: %v\n%s", err, output.String()) 235 } 236 237 if !*quiet { 238 log.Printf("Success. Binaries are in %s", actualBinDir()) 239 } 240} 241 242func actualBinDir() string { 243 cmd := exec.Command("go", "list", "-f", "{{.Target}}", "perkeep.org/cmd/pk") 244 cmd.Env = cleanGoEnv() 245 cmd.Stderr = os.Stderr 246 out, err := cmd.Output() 247 if err != nil { 248 log.Fatalf("Could not run go list to guess install dir: %v, %v", err, out) 249 } 250 return filepath.Dir(strings.TrimSpace(string(out))) 251} 252 253func baseDirName(sql bool) string { 254 buildBaseDir := "build-gopath" 255 if !sql { 256 buildBaseDir += "-nosqlite" 257 } 258 // We don't even consider whether we're cross-compiling. As long as we 259 // build for ARM, we do it in its own versioned dir. 260 if *buildARCH == "arm" { 261 buildBaseDir += "-armv" + *buildARM 262 } 263 return buildBaseDir 264} 265 266const ( 267 publisherJS = "app/publisher/publisher.js" 268 gopherjsUI = "server/perkeepd/ui/goui.js" 269 gopherjsUIURL = "https://storage.googleapis.com/perkeep-release/gopherjs/goui.js" 270 publisherJSURL = "https://storage.googleapis.com/perkeep-release/gopherjs/publisher.js" 271) 272 273func buildGopherjs() error { 274 // if gopherjs binary already exists, record its modtime, so we can reset it later. 275 // See explanation below. 276 outBin := hostExeName(filepath.Join(binDir, "gopherjs")) 277 fi, err := os.Stat(outBin) 278 if err != nil && !os.IsNotExist(err) { 279 return err 280 } 281 modtime := time.Now() 282 var hashBefore string 283 if err == nil { 284 modtime = fi.ModTime() 285 hashBefore = hashsum(outBin) 286 } 287 288 goBin := "go" 289 if gopherjsGoroot != "" { 290 goBin = hostExeName(filepath.Join(gopherjsGoroot, "bin", "go")) 291 } 292 293 src := filepath.Join(pkRoot, filepath.FromSlash("vendor/github.com/gopherjs/gopherjs")) 294 cmd := exec.Command(goBin, "install", "-v") 295 cmd.Dir = src 296 cmd.Env = os.Environ() 297 // forcing GOOS and GOARCH to prevent cross-compiling, as gopherjs will run on the 298 // current (host) platform. 299 cmd.Env = append(cmd.Env, "GOOS="+runtime.GOOS) 300 cmd.Env = append(cmd.Env, "GOARCH="+runtime.GOARCH) 301 if gopherjsGoroot != "" { 302 cmd.Env = append(cmd.Env, "GOROOT="+gopherjsGoroot) 303 } 304 cmd.Env = append(cmd.Env, "GO111MODULE=off") 305 var buf bytes.Buffer 306 cmd.Stderr = &buf 307 if err := cmd.Run(); err != nil { 308 return fmt.Errorf("error while building gopherjs: %v, %v", err, buf.String()) 309 } 310 if *verbose { 311 fmt.Println(buf.String()) 312 } 313 314 hashAfter := hashsum(outBin) 315 if hashAfter != hashBefore { 316 log.Printf("gopherjs rebuilt at %v", outBin) 317 return nil 318 } 319 // even if the source hasn't changed, apparently goinstall still at least bumps 320 // the modtime. Which means, 'gopherjs install' would then always rebuild its 321 // output too, even if no source changed since last time. We want to avoid that 322 // (because then parts of Perkeep get unnecessarily rebuilt too and yada yada), so 323 // we reset the modtime of gopherjs if the binary is the same as the previous time 324 // it was built. 325 return os.Chtimes(outBin, modtime, modtime) 326} 327 328func hashsum(filename string) string { 329 h := sha256.New() 330 f, err := os.Open(filename) 331 if err != nil { 332 log.Fatalf("could not compute SHA256 of %v: %v", filename, err) 333 } 334 defer f.Close() 335 if _, err := io.Copy(h, f); err != nil { 336 log.Fatalf("could not compute SHA256 of %v: %v", filename, err) 337 } 338 return string(h.Sum(nil)) 339} 340 341// genSearchTypes duplicates some of the perkeep.org/pkg/search types into 342// perkeep.org/app/publisher/js/zsearch.go , because it's too costly (in output 343// file size) for now to import the search pkg into gopherjs. 344func genSearchTypes() error { 345 sourceFile := filepath.Join(pkRoot, filepath.FromSlash("pkg/search/describe.go")) 346 outputFile := filepath.Join(pkRoot, filepath.FromSlash("app/publisher/js/zsearch.go")) 347 fi1, err := os.Stat(sourceFile) 348 if err != nil { 349 return err 350 } 351 fi2, err := os.Stat(outputFile) 352 if err != nil && !os.IsNotExist(err) { 353 return err 354 } 355 if err == nil && fi2.ModTime().After(fi1.ModTime()) { 356 return nil 357 } 358 cmd := exec.Command("go", "generate", "-tags=js", "-v", "perkeep.org/app/publisher/js") 359 if out, err := cmd.CombinedOutput(); err != nil { 360 return fmt.Errorf("go generate for publisher js error: %v, %v", err, string(out)) 361 } 362 log.Printf("generated %v", outputFile) 363 return nil 364} 365 366func genPublisherJS() error { 367 if err := genSearchTypes(); err != nil { 368 return err 369 } 370 output := filepath.Join(pkRoot, filepath.FromSlash(publisherJS)) 371 pkg := "perkeep.org/app/publisher/js" 372 return genJS(pkg, output) 373} 374 375func genWebUIJS() error { 376 output := filepath.Join(pkRoot, filepath.FromSlash(gopherjsUI)) 377 pkg := "perkeep.org/server/perkeepd/ui/goui" 378 return genJS(pkg, output) 379} 380 381func goPathBinDir() (string, error) { 382 cmd := exec.Command("go", "env", "GOPATH") 383 out, err := cmd.Output() 384 if err != nil { 385 return "", fmt.Errorf("could not get GOPATH: %v, %s", err, out) 386 } 387 paths := filepath.SplitList(strings.TrimSpace(string(out))) 388 if len(paths) < 1 { 389 return "", errors.New("no GOPATH") 390 } 391 return filepath.Join(paths[0], "bin"), nil 392} 393 394func genJS(pkg, output string) error { 395 // We want to use 'gopherjs install', and not 'gopherjs build', as the former is 396 // smarter and only rebuilds the output if needed. However, 'install' writes the 397 // output to GOPATH/bin, and not GOBIN. (https://github.com/gopherjs/gopherjs/issues/494) 398 // This means we have to be somewhat careful with naming our source pkg since gopherjs 399 // derives its output name from it. 400 // TODO(mpl): maybe rename the source pkg directories mentioned above. 401 402 if err := runGopherJS(pkg); err != nil { 403 return err 404 } 405 406 // TODO(mpl): set GOBIN, and remove all below, once 407 // https://github.com/gopherjs/gopherjs/issues/494 is fixed 408 binDir, err := goPathBinDir() 409 if err != nil { 410 return err 411 } 412 jsout := filepath.Join(binDir, filepath.Base(pkg)+".js") 413 fi1, err1 := os.Stat(output) 414 if err1 != nil && !os.IsNotExist(err1) { 415 return err1 416 } 417 fi2, err2 := os.Stat(jsout) 418 if err2 != nil && !os.IsNotExist(err2) { 419 return err2 420 } 421 if err1 == nil && fi1.ModTime().After(fi2.ModTime()) { 422 // output exists and is already up to date, nothing to do 423 return nil 424 } 425 data, err := ioutil.ReadFile(jsout) 426 if err != nil { 427 return err 428 } 429 return ioutil.WriteFile(output, data, 0600) 430} 431 432func runGopherJS(pkg string) error { 433 gopherjsBin := hostExeName(filepath.Join(binDir, "gopherjs")) 434 args := []string{"install", pkg, "-v", "--tags", "nocgo noReactBundle"} 435 if *embedResources { 436 // when embedding for "production", use -m to minify the javascript output 437 args = append(args, "-m") 438 } 439 cmd := exec.Command(gopherjsBin, args...) 440 cmd.Env = os.Environ() 441 // Pretend we're on linux regardless of the actual host, because recommended 442 // hack to work around https://github.com/gopherjs/gopherjs/issues/511 443 cmd.Env = append(cmd.Env, "GOOS=linux") 444 if gopherjsGoroot != "" { 445 cmd.Env = append(cmd.Env, "GOROOT="+gopherjsGoroot) 446 } 447 cmd.Env = append(cmd.Env, "GO111MODULE=off") 448 var buf bytes.Buffer 449 cmd.Stderr = &buf 450 err := cmd.Run() 451 if err != nil { 452 return fmt.Errorf("gopherjs for %v error: %v, %v", pkg, err, buf.String()) 453 } 454 if *verbose { 455 fmt.Println(buf.String()) 456 } 457 return nil 458} 459 460// genWebUIReact runs go generate on the gopherjs code of the web UI, which 461// invokes reactGen on the Go React components. This generates the boilerplate 462// code, in gen_*_reactGen.go files, required to complete those components. 463func genWebUIReact() error { 464 args := []string{"generate", "-v", "perkeep.org/server/perkeepd/ui/goui/..."} 465 466 path := strings.Join([]string{ 467 binDir, 468 os.Getenv("PATH"), 469 }, string(os.PathListSeparator)) 470 471 cmd := exec.Command("go", args...) 472 cmd.Env = os.Environ() 473 cmd.Env = append(cmd.Env, "PATH="+path) 474 var buf bytes.Buffer 475 cmd.Stderr = &buf 476 cmd.Env = append(os.Environ(), "GO111MODULE=off") 477 err := cmd.Run() 478 if err != nil { 479 return fmt.Errorf("go generate for web UI error: %v, %v", err, buf.String()) 480 } 481 if *verbose { 482 fmt.Println(buf.String()) 483 } 484 return nil 485} 486 487// makeJS builds and runs the gopherjs command on perkeep.org/app/publisher/js 488// and perkeep.org/server/perkeepd/ui/goui 489func makeJS(doWebUI, doPublisher bool) error { 490 if err := buildGopherjs(); err != nil { 491 return fmt.Errorf("error building gopherjs: %v", err) 492 } 493 494 if doPublisher { 495 if err := genPublisherJS(); err != nil { 496 return err 497 } 498 } 499 if doWebUI { 500 if err := genWebUIJS(); err != nil { 501 return err 502 } 503 } 504 return nil 505} 506 507func fetchAllJS(doWebUI, doPublisher bool) error { 508 if doPublisher { 509 if err := fetchJS(publisherJSURL, filepath.FromSlash(publisherJS)); err != nil { 510 return err 511 } 512 } 513 if doWebUI { 514 if err := fetchJS(gopherjsUIURL, filepath.FromSlash(gopherjsUI)); err != nil { 515 return err 516 } 517 } 518 return nil 519} 520 521// fetchJS gets the javascript resource at jsURL and writes it to jsOnDisk. 522// Since said resource can be quite large, it first fetches the hashsum contained 523// in the file at jsURL+".sha256", and if we already have the file on disk, with a 524// matching hashsum, it does not actually fetch jsURL. If it does, it checks that 525// the newly written file does match the hashsum. 526func fetchJS(jsURL, jsOnDisk string) error { 527 var currentSum string 528 _, err := os.Stat(jsOnDisk) 529 if err != nil { 530 if !os.IsNotExist(err) { 531 return err 532 } 533 } else { 534 // If yes, compute its hash 535 h := sha256.New() 536 f, err := os.Open(jsOnDisk) 537 if err != nil { 538 return err 539 } 540 defer f.Close() 541 if _, err := io.Copy(h, f); err != nil { 542 return err 543 } 544 currentSum = fmt.Sprintf("%x", h.Sum(nil)) 545 } 546 547 // fetch the hash of the remote 548 resp, err := http.Get(jsURL + ".sha256") 549 if err != nil { 550 return err 551 } 552 defer resp.Body.Close() 553 data, err := ioutil.ReadAll(resp.Body) 554 if err != nil { 555 return err 556 } 557 upstreamSum := strings.TrimSuffix(string(data), "\n") 558 559 if currentSum != "" && 560 currentSum == upstreamSum { 561 // We already have the latest version 562 return nil 563 } 564 565 resp, err = http.Get(jsURL) 566 if err != nil { 567 return err 568 } 569 defer resp.Body.Close() 570 js := filepath.Join(pkRoot, filepath.FromSlash(jsOnDisk)) 571 f, err := os.Create(js) 572 if err != nil { 573 return err 574 } 575 h := sha256.New() 576 mr := io.MultiWriter(f, h) 577 if _, err := io.Copy(mr, resp.Body); err != nil { 578 f.Close() 579 return err 580 } 581 if err := f.Close(); err != nil { 582 return err 583 } 584 sum := fmt.Sprintf("%x", h.Sum(nil)) 585 586 if upstreamSum != sum { 587 return fmt.Errorf("checksum mismatch for %q: got %q, want %q", jsURL, sum, upstreamSum) 588 } 589 return nil 590} 591 592func doUI(withPerkeepd, withPublisher bool) error { 593 if !withPerkeepd && !withPublisher { 594 return nil 595 } 596 597 if !*buildWebUI { 598 if !*offline { 599 return fetchAllJS(withPerkeepd, withPublisher) 600 } 601 if withPublisher { 602 _, err := os.Stat(filepath.FromSlash(publisherJS)) 603 if os.IsNotExist(err) { 604 return fmt.Errorf("%s on disk is required for offline building. Fetch if first at %s.", publisherJS, publisherJSURL) 605 } 606 if err != nil { 607 return err 608 } 609 } 610 if withPerkeepd { 611 _, err := os.Stat(filepath.FromSlash(gopherjsUI)) 612 if os.IsNotExist(err) { 613 return fmt.Errorf("%s on disk is required for offline building. Fetch if first at %s.", gopherjsUI, gopherjsUIURL) 614 } 615 if err != nil { 616 return err 617 } 618 } 619 return nil 620 } 621 622 if os.Getenv("GO111MODULE") != "off" { 623 fmt.Println("Cannot rebuild web UI with go modules enabled, as it is not supported by GopherJS. Now rebuilding with GO111MODULE=off.") 624 } 625 626 if err := buildReactGen(); err != nil { 627 return err 628 } 629 630 if withPerkeepd { 631 if err := genWebUIReact(); err != nil { 632 return err 633 } 634 } 635 636 // gopherjs has to run before doEmbed since we need all the javascript 637 // to be generated before embedding happens. 638 return makeJS(withPerkeepd, withPublisher) 639} 640 641// Create an environment variable of the form key=value. 642func envPair(key, value string) string { 643 return fmt.Sprintf("%s=%s", key, value) 644} 645 646// TODO(mpl): we probably can get rid of cleanGoEnv now that "last in wins" for 647// duplicates in Env. 648 649// cleanGoEnv returns a copy of the current environment with any variable listed 650// in others removed. Also, when cross-compiling, it removes GOBIN and sets GOOS 651// and GOARCH, and GOARM as needed. 652func cleanGoEnv(others ...string) (clean []string) { 653 excl := make([]string, len(others)) 654 for i, v := range others { 655 excl[i] = v + "=" 656 } 657 658Env: 659 for _, env := range os.Environ() { 660 for _, v := range excl { 661 if strings.HasPrefix(env, v) { 662 continue Env 663 } 664 } 665 // remove GOBIN if we're cross-compiling 666 if strings.HasPrefix(env, "GOBIN=") && 667 (*buildOS != runtime.GOOS || *buildARCH != runtime.GOARCH) { 668 continue 669 } 670 // We skip these two as well, otherwise they'd take precedence over the 671 // ones appended below. 672 if *buildOS != runtime.GOOS && strings.HasPrefix(env, "GOOS=") { 673 continue 674 } 675 if *buildARCH != runtime.GOARCH && strings.HasPrefix(env, "GOARCH=") { 676 continue 677 } 678 // If we're building for ARM (regardless of cross-compiling or not), we reset GOARM 679 if *buildARCH == "arm" && strings.HasPrefix(env, "GOARM=") { 680 continue 681 } 682 683 clean = append(clean, env) 684 } 685 if *buildOS != runtime.GOOS { 686 clean = append(clean, envPair("GOOS", *buildOS)) 687 } 688 if *buildARCH != runtime.GOARCH { 689 clean = append(clean, envPair("GOARCH", *buildARCH)) 690 } 691 // If we're building for ARM (regardless of cross-compiling or not), we reset GOARM 692 if *buildARCH == "arm" { 693 clean = append(clean, envPair("GOARM", *buildARM)) 694 } 695 return 696} 697 698func stringListContains(strs []string, str string) bool { 699 for _, s := range strs { 700 if s == str { 701 return true 702 } 703 } 704 return false 705} 706 707// fullSrcPath returns the full path concatenation 708// of pkRoot with fromSrc. 709func fullSrcPath(fromSrc string) string { 710 return filepath.Join(pkRoot, filepath.FromSlash(fromSrc)) 711} 712 713func genEmbeds() error { 714 cmdName := hostExeName(filepath.Join(binDir, "genfileembed")) 715 for _, embeds := range []string{ 716 "server/perkeepd/ui", 717 "pkg/server", 718 "clients/web/embed/fontawesome", 719 "clients/web/embed/keepy", 720 "clients/web/embed/leaflet", 721 "clients/web/embed/less", 722 "clients/web/embed/opensans", 723 "clients/web/embed/react", 724 "app/publisher", 725 "app/scanningcabinet/ui", 726 } { 727 embeds := fullSrcPath(embeds) 728 args := []string{"-build-tags=with_embed"} 729 args = append(args, embeds) 730 cmd := exec.Command(cmdName, args...) 731 cmd.Stdout = os.Stdout 732 var buf bytes.Buffer 733 cmd.Stderr = &buf 734 735 if *verbose { 736 log.Printf("Running %s %s", cmdName, embeds) 737 } 738 if err := cmd.Run(); err != nil { 739 os.Stderr.Write(buf.Bytes()) 740 return fmt.Errorf("error running %s %s: %v", cmdName, embeds, err) 741 } 742 if *verbose { 743 fmt.Println(buf.String()) 744 } 745 } 746 return nil 747} 748 749func buildGenfileembed() error { 750 return buildBin("perkeep.org/pkg/fileembed/genfileembed", false) 751} 752 753func buildReactGen() error { 754 return buildBin("perkeep.org/vendor/myitcv.io/react/cmd/reactGen", true) 755} 756 757func buildDevcam() error { 758 return buildBin("perkeep.org/dev/devcam", false) 759} 760 761func buildBin(pkg string, forceModulesOff bool) error { 762 pkgBase := pathpkg.Base(pkg) 763 764 args := []string{"install", "-v"} 765 args = append(args, 766 filepath.FromSlash(pkg), 767 ) 768 cmd := exec.Command("go", args...) 769 cmd.Stdout = os.Stdout 770 cmd.Stderr = os.Stderr 771 if *verbose { 772 log.Printf("Running go with args %s", args) 773 } 774 if forceModulesOff { 775 cmd.Env = append(os.Environ(), "GO111MODULE=off") 776 } 777 if err := cmd.Run(); err != nil { 778 return fmt.Errorf("Error building %v: %v", pkgBase, err) 779 } 780 if *verbose { 781 log.Printf("%v installed in %s", pkgBase, actualBinDir()) 782 } 783 return nil 784} 785 786// getVersion returns the version of Perkeep found in a VERSION file at the root. 787func getVersion() string { 788 slurp, err := ioutil.ReadFile(filepath.Join(pkRoot, "VERSION")) 789 v := strings.TrimSpace(string(slurp)) 790 if err != nil && !os.IsNotExist(err) { 791 log.Fatal(err) 792 } 793 if v == "" { 794 return "unknown" 795 } 796 return v 797} 798 799var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{10,10}\b`) 800 801// getGitVersion returns the git version of the git repo at pkRoot as a 802// string of the form "yyyy-mm-dd-xxxxxxx", with an optional trailing 803// '+' if there are any local uncommitted modifications to the tree. 804func getGitVersion() string { 805 if _, err := exec.LookPath("git"); err != nil { 806 return "" 807 } 808 if _, err := os.Stat(filepath.Join(pkRoot, ".git")); os.IsNotExist(err) { 809 return "" 810 } 811 cmd := exec.Command("git", "rev-list", "--max-count=1", "--pretty=format:'%ad-%h'", 812 "--date=short", "--abbrev=10", "HEAD") 813 cmd.Dir = pkRoot 814 out, err := cmd.Output() 815 if err != nil { 816 log.Fatalf("Error running git rev-list in %s: %v", pkRoot, err) 817 } 818 v := strings.TrimSpace(string(out)) 819 if m := gitVersionRx.FindStringSubmatch(v); m != nil { 820 v = m[0] 821 } else { 822 panic("Failed to find git version in " + v) 823 } 824 cmd = exec.Command("git", "diff", "--exit-code") 825 cmd.Dir = pkRoot 826 if err := cmd.Run(); err != nil { 827 v += "+" 828 } 829 return v 830} 831 832// verifyPerkeepRoot sets pkRoot and crashes if dir isn't the Perkeep root directory. 833func verifyPerkeepRoot() { 834 var err error 835 pkRoot, err = os.Getwd() 836 if err != nil { 837 log.Fatalf("Failed to get current directory: %v", err) 838 } 839 testFile := filepath.Join(pkRoot, "pkg", "blob", "ref.go") 840 if _, err := os.Stat(testFile); err != nil { 841 log.Fatalf("make.go must be run from the Perkeep src root directory (where make.go is). Current working directory is %s", pkRoot) 842 } 843 844 // we can't rely on perkeep.org/cmd/pk with modules on as we have no assurance 845 // the current dir is $GOPATH/src/perkeep.org, so we use ./cmd/pk instead. 846 cmd := exec.Command("go", "list", "-f", "{{.Target}}", "./cmd/pk") 847 if os.Getenv("GO111MODULE") == "off" || *buildWebUI { 848 // if we're building the webUI we need to be in "legacy" GOPATH mode, so in 849 // $GOPATH/src/perkeep.org 850 if err := validateDirInGOPATH(pkRoot); err != nil { 851 log.Fatalf("We're running in GO111MODULE=off mode, either because you set it, or because you want to build the Web UI, so we need to be in a GOPATH, but: %v", err) 852 } 853 cmd = exec.Command("go", "list", "-f", "{{.Target}}", "perkeep.org/cmd/pk") 854 } 855 cmd.Stderr = os.Stderr 856 out, err := cmd.Output() 857 if err != nil { 858 log.Fatalf("Could not run go list to find install dir: %v, %s", err, out) 859 } 860 binDir = filepath.Dir(strings.TrimSpace(string(out))) 861} 862 863func validateDirInGOPATH(dir string) error { 864 fi, err := os.Lstat(dir) 865 if err != nil { 866 return err 867 } 868 869 gopathEnv, err := exec.Command("go", "env", "GOPATH").Output() 870 if err != nil { 871 return fmt.Errorf("error finding GOPATH: %v", err) 872 } 873 gopaths := filepath.SplitList(strings.TrimSpace(string(gopathEnv))) 874 if len(gopaths) == 0 { 875 return fmt.Errorf("failed to find your GOPATH: go env GOPATH returned nothing") 876 } 877 var validOpts []string 878 for _, gopath := range gopaths { 879 validDir := filepath.Join(gopath, "src", "perkeep.org") 880 validOpts = append(validOpts, validDir) 881 fi2, err := os.Lstat(validDir) 882 if os.IsNotExist(err) { 883 continue 884 } 885 if err != nil { 886 return err 887 } 888 if os.SameFile(fi, fi2) { 889 // In a valid directory. 890 return nil 891 } 892 } 893 if len(validOpts) == 1 { 894 return fmt.Errorf("make.go cannot be run from %s; it must be in a valid GOPATH. Move the directory containing make.go to %s", dir, validOpts[0]) 895 } else { 896 return fmt.Errorf("make.go cannot be run from %s; it must be in a valid GOPATH. Move the directory containing make.go to one of %q", dir, validOpts) 897 } 898} 899 900const ( 901 goVersionMinor = 15 902 gopherJSGoMinor = 12 903) 904 905var validVersionRx = regexp.MustCompile(`go version go1\.(\d+)`) 906 907// verifyGoVersion runs "go version" and parses the output. If the version is 908// acceptable a check for gopherjs versions are also done. If problems 909// are found a message is logged and we abort. 910func verifyGoVersion() { 911 _, err := exec.LookPath("go") 912 if err != nil { 913 log.Fatalf("Go doesn't appear to be installed ('go' isn't in your PATH). Install Go 1.%d or newer.", goVersionMinor) 914 } 915 out, err := exec.Command("go", "version").Output() 916 if err != nil { 917 log.Fatalf("Error checking Go version with the 'go' command: %v", err) 918 } 919 920 version := string(out) 921 922 // Handle non-versioned binaries 923 // ex: "go version devel +c26fac8 Thu Feb 15 21:41:39 2018 +0000 linux/amd64" 924 if strings.HasPrefix(version, "go version devel ") { 925 verifyGopherjsGoroot(" devel") 926 return 927 } 928 929 m := validVersionRx.FindStringSubmatch(version) 930 if m == nil { 931 log.Fatalf("Unexpected output while checking 'go version': %q", version) 932 } 933 minorVersion, err := strconv.Atoi(m[1]) 934 if err != nil { 935 log.Fatalf("Unexpected error while parsing version string %q: %v", m[1], err) 936 } 937 938 if minorVersion < goVersionMinor { 939 log.Fatalf("Your version of Go (%s) is too old. Perkeep requires Go 1.%d or later.", string(out), goVersionMinor) 940 } 941 942 if *website || *camnetdns { 943 return 944 } 945 946 if minorVersion != gopherJSGoMinor { 947 verifyGopherjsGoroot(fmt.Sprintf("1.%d", minorVersion)) 948 } 949} 950 951func verifyGopherjsGoroot(goFound string) { 952 if !*buildWebUI { 953 return 954 } 955 gopherjsGoroot = os.Getenv("CAMLI_GOPHERJS_GOROOT") 956 goBin := hostExeName(filepath.Join(gopherjsGoroot, "bin", "go")) 957 if gopherjsGoroot == "" { 958 goInHomeDir, err := findGopherJSGoroot() 959 if err != nil { 960 log.Fatalf("Error while looking for a go1.%d dir in %v: %v", gopherJSGoMinor, homeDir(), err) 961 } 962 if goInHomeDir == "" { 963 log.Fatalf("You're using go%s != go1.%d, which GopherJS requires, and it was not found in %v. You need to specify a go1.%d root in CAMLI_GOPHERJS_GOROOT for building GopherJS.", goFound, gopherJSGoMinor, homeDir(), gopherJSGoMinor) 964 } 965 gopherjsGoroot = filepath.Join(homeDir(), goInHomeDir) 966 goBin = hostExeName(filepath.Join(gopherjsGoroot, "bin", "go")) 967 log.Printf("You're using go%s != go1.%d, which GopherJS requires, and CAMLI_GOPHERJS_GOROOT was not provided, so defaulting to %v for building GopherJS instead.", goFound, gopherJSGoMinor, goBin) 968 } 969 if _, err := os.Stat(goBin); err != nil { 970 if !os.IsNotExist(err) { 971 log.Fatal(err) 972 } 973 log.Fatalf("%v not found. You need to specify a go1.%d root in CAMLI_GOPHERJS_GOROOT for building GopherJS", goBin, gopherJSGoMinor) 974 } 975} 976 977// findGopherJSGoroot tries to find a go1.gopherJSGoMinor.* go root in the home 978// directory. It returns the empty string and no error if none was found. 979func findGopherJSGoroot() (string, error) { 980 dir, err := os.Open(homeDir()) 981 if err != nil { 982 return "", err 983 } 984 defer dir.Close() 985 names, err := dir.Readdirnames(-1) 986 if err != nil { 987 return "", err 988 } 989 goVersion := fmt.Sprintf("go1.%d", gopherJSGoMinor) 990 for _, name := range names { 991 if strings.HasPrefix(name, goVersion) { 992 return name, nil 993 } 994 } 995 return "", nil 996} 997 998func withSQLite() bool { 999 cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH 1000 var sql bool 1001 var err error 1002 if *sqlFlag == "auto" { 1003 sql = !cross && haveSQLite 1004 } else { 1005 sql, err = strconv.ParseBool(*sqlFlag) 1006 if err != nil { 1007 log.Fatalf("Bad boolean --sql flag %q", *sqlFlag) 1008 } 1009 } 1010 1011 if cross && sql { 1012 log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.") 1013 } 1014 if sql && !haveSQLite { 1015 // TODO(lindner): fix these docs. 1016 log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite") 1017 switch runtime.GOOS { 1018 case "darwin": 1019 log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/") 1020 case "linux": 1021 log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.") 1022 case "windows": 1023 log.Printf("SQLite is not easy on windows. Please see https://perkeep.org/doc/server-config#windows") 1024 } 1025 os.Exit(2) 1026 } 1027 return sql 1028} 1029 1030func checkHaveSQLite() bool { 1031 if runtime.GOOS == "windows" { 1032 // TODO: Find some other non-pkg-config way to test, like 1033 // just compiling a small Go program that sees whether 1034 // it's available. 1035 // 1036 // For now: 1037 return false 1038 } 1039 _, err := exec.LookPath("pkg-config") 1040 if err != nil { 1041 return false 1042 } 1043 out, err := exec.Command("pkg-config", "--libs", "sqlite3").Output() 1044 if err != nil && err.Error() == "exit status 1" { 1045 // This is sloppy (comparing against a string), but 1046 // doing it correctly requires using multiple *.go 1047 // files to portably get the OS-syscall bits, and I 1048 // want to keep make.go a single file. 1049 return false 1050 } 1051 if err != nil { 1052 log.Fatalf("Can't determine whether sqlite3 is available, and where. pkg-config error was: %v, %s", err, out) 1053 } 1054 return strings.TrimSpace(string(out)) != "" 1055} 1056 1057func doEmbed() { 1058 if *verbose { 1059 log.Printf("Embedding resources...") 1060 } 1061 closureEmbed := fullSrcPath("server/perkeepd/ui/closure/z_data.go") 1062 closureSrcDir := filepath.Join(pkRoot, filepath.FromSlash("clients/web/embed/closure/lib")) 1063 err := embedClosure(closureSrcDir, closureEmbed) 1064 if err != nil { 1065 log.Fatal(err) 1066 } 1067 if err = buildGenfileembed(); err != nil { 1068 log.Fatal(err) 1069 } 1070 if err = genEmbeds(); err != nil { 1071 log.Fatal(err) 1072 } 1073} 1074 1075func embedClosure(closureDir, embedFile string) error { 1076 if _, err := os.Stat(closureDir); err != nil { 1077 return fmt.Errorf("Could not stat %v: %v", closureDir, err) 1078 } 1079 1080 // first collect the files and modTime 1081 var modTime time.Time 1082 type pathAndSuffix struct { 1083 path, suffix string 1084 } 1085 var files []pathAndSuffix 1086 err := filepath.Walk(closureDir, func(path string, fi os.FileInfo, err error) error { 1087 if err != nil { 1088 return err 1089 } 1090 suffix, err := filepath.Rel(closureDir, path) 1091 if err != nil { 1092 return fmt.Errorf("Failed to find Rel(%q, %q): %v", closureDir, path, err) 1093 } 1094 if fi.IsDir() { 1095 return nil 1096 } 1097 if mt := fi.ModTime(); mt.After(modTime) { 1098 modTime = mt 1099 } 1100 files = append(files, pathAndSuffix{path, suffix}) 1101 return nil 1102 }) 1103 if err != nil { 1104 return err 1105 } 1106 // do not regenerate the whole embedFile if it exists and newer than modTime. 1107 if fi, err := os.Stat(embedFile); err == nil && fi.Size() > 0 && fi.ModTime().After(modTime) { 1108 if *verbose { 1109 log.Printf("skipping regeneration of %s", embedFile) 1110 } 1111 return nil 1112 } 1113 1114 // second, zip it 1115 var zipbuf bytes.Buffer 1116 var zipdest io.Writer = &zipbuf 1117 if os.Getenv("CAMLI_WRITE_TMP_ZIP") != "" { 1118 f, _ := os.Create("/tmp/camli-closure.zip") 1119 zipdest = io.MultiWriter(zipdest, f) 1120 defer f.Close() 1121 } 1122 w := zip.NewWriter(zipdest) 1123 for _, elt := range files { 1124 b, err := ioutil.ReadFile(elt.path) 1125 if err != nil { 1126 return err 1127 } 1128 f, err := w.Create(filepath.ToSlash(elt.suffix)) 1129 if err != nil { 1130 return err 1131 } 1132 if _, err = f.Write(b); err != nil { 1133 return err 1134 } 1135 } 1136 if err = w.Close(); err != nil { 1137 return err 1138 } 1139 1140 // then embed it as a quoted string 1141 var qb bytes.Buffer 1142 fmt.Fprint(&qb, "// +build with_embed\n\n") 1143 fmt.Fprint(&qb, "package closure\n\n") 1144 fmt.Fprint(&qb, "import \"time\"\n\n") 1145 fmt.Fprint(&qb, "func init() {\n") 1146 fmt.Fprintf(&qb, "\tZipModTime = time.Unix(%d, 0)\n", modTime.Unix()) 1147 fmt.Fprint(&qb, "\tZipData = ") 1148 quote(&qb, zipbuf.Bytes()) 1149 fmt.Fprint(&qb, "\n}\n") 1150 1151 // and write to a .go file 1152 if err := writeFileIfDifferent(embedFile, qb.Bytes()); err != nil { 1153 return err 1154 } 1155 return nil 1156 1157} 1158 1159func writeFileIfDifferent(filename string, contents []byte) error { 1160 fi, err := os.Stat(filename) 1161 if err == nil && fi.Size() == int64(len(contents)) && contentsEqual(filename, contents) { 1162 return nil 1163 } 1164 return ioutil.WriteFile(filename, contents, 0644) 1165} 1166 1167func contentsEqual(filename string, contents []byte) bool { 1168 got, err := ioutil.ReadFile(filename) 1169 if os.IsNotExist(err) { 1170 return false 1171 } 1172 if err != nil { 1173 log.Fatalf("Error reading %v: %v", filename, err) 1174 } 1175 return bytes.Equal(got, contents) 1176} 1177 1178// quote escapes and quotes the bytes from bs and writes 1179// them to dest. 1180func quote(dest *bytes.Buffer, bs []byte) { 1181 dest.WriteByte('"') 1182 for _, b := range bs { 1183 if b == '\n' { 1184 dest.WriteString(`\n`) 1185 continue 1186 } 1187 if b == '\\' { 1188 dest.WriteString(`\\`) 1189 continue 1190 } 1191 if b == '"' { 1192 dest.WriteString(`\"`) 1193 continue 1194 } 1195 if (b >= 32 && b <= 126) || b == '\t' { 1196 dest.WriteByte(b) 1197 continue 1198 } 1199 fmt.Fprintf(dest, "\\x%02x", b) 1200 } 1201 dest.WriteByte('"') 1202} 1203 1204// hostExeName returns the executable name 1205// for s on the currently running host OS. 1206func hostExeName(s string) string { 1207 if runtime.GOOS == "windows" { 1208 return s + ".exe" 1209 } 1210 return s 1211} 1212 1213// copied from pkg/osutil/paths.go 1214func homeDir() string { 1215 if runtime.GOOS == "windows" { 1216 return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") 1217 } 1218 return os.Getenv("HOME") 1219} 1220 1221func failIfCamlistoreOrgDir() { 1222 dir, _ := os.Getwd() 1223 if strings.HasSuffix(dir, "camlistore.org") { 1224 log.Fatalf(`Camlistore was renamed to Perkeep. Your current directory (%s) looks like a camlistore.org directory. 1225 1226We're expecting you to be in a perkeep.org directory now. 1227 1228See https://github.com/perkeep/perkeep/issues/981#issuecomment-354690313 for details. 1229 1230You need to rename your "camlistore.org" parent directory to "perkeep.org" 1231 1232`, dir) 1233 } 1234} 1235