1#
2#
3#         Maintenance program for Nim
4#        (c) Copyright 2017 Andreas Rumpf
5#
6#    See the file "copying.txt", included in this
7#    distribution, for details about the copyright.
8#
9#    See doc/koch.txt for documentation.
10#
11
12const
13  NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master
14  # examples of possible values: #head, #ea82b54, 1.2.3
15  FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014"
16  HeadHash = "#head"
17when not defined(windows):
18  const
19    Z3StableCommit = "65de3f748a6812eecd7db7c478d5fc54424d368b" # the version of Z3 that DrNim uses
20
21when defined(gcc) and defined(windows):
22  when defined(x86):
23    {.link: "icons/koch.res".}
24  else:
25    {.link: "icons/koch_icon.o".}
26
27when defined(amd64) and defined(windows) and defined(vcc):
28  {.link: "icons/koch-amd64-windows-vcc.res".}
29when defined(i386) and defined(windows) and defined(vcc):
30  {.link: "icons/koch-i386-windows-vcc.res".}
31
32import std/[os, strutils, parseopt, osproc]
33  # Using `std/os` instead of `os` to fail early if config isn't set up properly.
34  # If this fails with: `Error: cannot open file: std/os`, see
35  # https://github.com/nim-lang/Nim/pull/14291 for explanation + how to fix.
36
37import tools / kochdocs
38import tools / deps
39
40const VersionAsString = system.NimVersion
41
42const
43  HelpText = """
44+-----------------------------------------------------------------+
45|         Maintenance program for Nim                             |
46|             Version $1|
47|             (c) 2017 Andreas Rumpf                              |
48+-----------------------------------------------------------------+
49Build time: $2, $3
50
51Usage:
52  koch [options] command [options for command]
53Options:
54  --help, -h               shows this help and quits
55  --latest                 bundle the installers with bleeding edge versions of
56                           external components.
57  --stable                 bundle the installers with stable versions of
58                           external components (default).
59  --nim:path               use specified path for nim binary
60  --localdocs[:path]       only build local documentations. If a path is not
61                           specified (or empty), the default is used.
62Possible Commands:
63  boot [options]           bootstraps with given command line options
64  distrohelper [bindir]    helper for distro packagers
65  tools                    builds Nim related tools
66  toolsNoExternal          builds Nim related tools (except external tools,
67                           e.g. nimble)
68                           doesn't require network connectivity
69  nimble                   builds the Nimble tool
70  fusion                   installs fusion via Nimble
71
72Boot options:
73  -d:release               produce a release version of the compiler
74  -d:nimUseLinenoise       use the linenoise library for interactive mode
75                           `nim secret` (not needed on Windows)
76  -d:leanCompiler          produce a compiler without JS codegen or
77                           documentation generator in order to use less RAM
78                           for bootstrapping
79
80Commands for core developers:
81  runCI                    runs continuous integration (CI), e.g. from travis
82  docs [options]           generates the full documentation
83  csource -d:danger        builds the C sources for installation
84  pdf                      builds the PDF documentation
85  zip                      builds the installation zip package
86  xz                       builds the installation tar.xz package
87  testinstall              test tar.xz package; Unix only!
88  installdeps [options]    installs external dependency (e.g. tinyc) to dist/
89  tests [options]          run the testsuite (run a subset of tests by
90                           specifying a category, e.g. `tests cat async`)
91  temp options             creates a temporary compiler for testing
92Web options:
93  --googleAnalytics:UA-... add the given google analytics code to the docs. To
94                           build the official docs, use UA-48159761-1
95"""
96
97let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed e.g.: `nim c -o:koch_debug koch.nim`
98               else: getAppDir() / "koch".exe # works for winrelease
99
100proc kochExec*(cmd: string) =
101  exec kochExe.quoteShell & " " & cmd
102
103proc kochExecFold*(desc, cmd: string) =
104  execFold(desc, kochExe.quoteShell & " " & cmd)
105
106template withDir(dir, body) =
107  let old = getCurrentDir()
108  try:
109    setCurrentDir(dir)
110    body
111  finally:
112    setCurrentDir(old)
113
114let origDir = getCurrentDir()
115setCurrentDir(getAppDir())
116
117proc tryExec(cmd: string): bool =
118  echo(cmd)
119  result = execShellCmd(cmd) == 0
120
121proc safeRemove(filename: string) =
122  if fileExists(filename): removeFile(filename)
123
124proc overwriteFile(source, dest: string) =
125  safeRemove(dest)
126  moveFile(source, dest)
127
128proc copyExe(source, dest: string) =
129  safeRemove(dest)
130  copyFile(dest=dest, source=source)
131  inclFilePermissions(dest, {fpUserExec, fpGroupExec, fpOthersExec})
132
133const
134  compileNimInst = "tools/niminst/niminst"
135  distDir = "dist"
136
137proc csource(args: string) =
138  nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " &
139           "--main:compiler/nim.nim compiler/installer.ini $1") %
140       [args, VersionAsString, compileNimInst])
141
142#proc bundleC2nim(args: string) =
143#  cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git")
144#  nimCompile("dist/c2nim/c2nim",
145#             options = "--noNimblePath --path:. " & args)
146
147#proc bundleNimbleExe(latest: bool, args: string) =
148#  let commit = if latest: "HEAD" else: NimbleStableCommit
149#  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
150#                  commit = commit, allowBundled = true)
151#  # installer.ini expects it under $nim/bin
152#  nimCompile("dist/nimble/src/nimble.nim",
153#             options = "-d:release --noNimblePath " & args)
154
155proc bundleNimsuggest(args: string) =
156  nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
157                 options = "-d:danger " & args)
158
159proc buildVccTool(args: string) =
160  let input = "tools/vccexe/vccexe.nim"
161  if contains(args, "--cc:vcc"):
162    nimCompileFold("Compile Vcc", input, "build", options = args)
163    let fileName = input.splitFile.name
164    moveFile(exe("build" / fileName), exe("bin" / fileName))
165  else:
166    nimCompileFold("Compile Vcc", input, options = args)
167
168proc bundleNimpretty(args: string) =
169  nimCompileFold("Compile nimpretty", "nimpretty/nimpretty.nim",
170                 options = "-d:release " & args)
171
172proc bundleWinTools(args: string) =
173  nimCompile("tools/finish.nim", outputDir = "", options = args)
174
175  buildVccTool(args)
176  nimCompile("tools/nimgrab.nim", options = "-d:ssl " & args)
177  nimCompile("tools/nimgrep.nim", options = args)
178  nimCompile("testament/testament.nim", options = args)
179  when false:
180    # not yet a tool worth including
181    nimCompile(r"tools\downloader.nim",
182               options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui " & args)
183
184proc zip(latest: bool; args: string) =
185  #bundleNimbleExe(latest, args)
186  bundleNimsuggest(args)
187  bundleNimpretty(args)
188  bundleWinTools(args)
189  nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
190       [VersionAsString, compileNimInst])
191  exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" %
192       ["tools/niminst/niminst".exe, VersionAsString])
193
194#proc ensureCleanGit() =
195#  let (outp, status) = osproc.execCmdEx("git diff")
196#  if outp.len != 0:
197#    quit "Not a clean git repository; 'git diff' not empty!"
198#  if status != 0:
199#    quit "Not a clean git repository; 'git diff' returned non-zero!"
200
201proc xz(latest: bool; args: string) =
202  #ensureCleanGit()
203  nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
204       [VersionAsString, compileNimInst])
205  exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" %
206       ["tools" / "niminst" / "niminst".exe, VersionAsString])
207
208proc buildTool(toolname, args: string) =
209  nimexec("cc $# $#" % [args, toolname])
210  copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe)
211
212proc buildTools(args: string = "") =
213  bundleNimsuggest(args)
214  nimCompileFold("Compile nimgrep", "tools/nimgrep.nim",
215                 options = "-d:release " & args)
216  when defined(windows): buildVccTool(args)
217  bundleNimpretty(args)
218  nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args)
219
220  # pre-packages a debug version of nim which can help in many cases investigate issuses
221  # withouth having to rebuild compiler.
222  # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler
223  # `-d:debug` should be changed to a flag that doesn't require re-compiling nim
224  # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces
225  nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options =
226      "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args,
227      outputName = "nim_dbg")
228
229  nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args,
230      outputName = "atlas")
231
232
233proc nsis(latest: bool; args: string) =
234  #bundleNimbleExe(latest, args)
235  bundleNimsuggest(args)
236  bundleWinTools(args)
237  # make sure we have generated the niminst executables:
238  buildTool("tools/niminst/niminst", args)
239  #buildTool("tools/nimgrep", args)
240  # produce 'nim_debug.exe':
241  #exec "nim c compiler" / "nim.nim"
242  #copyExe("compiler/nim".exe, "bin/nim_debug".exe)
243  exec(("tools" / "niminst" / "niminst --var:version=$# --var:mingw=mingw$#" &
244        " nsis compiler/installer.ini") % [VersionAsString, $(sizeof(pointer)*8)])
245
246proc geninstall(args="") =
247  nimexec("cc -r $# --var:version=$# --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini $#" %
248       [compileNimInst, VersionAsString, args])
249
250proc install(args: string) =
251  geninstall()
252  exec("sh ./install.sh $#" % args)
253
254when false:
255  proc web(args: string) =
256    nimexec("js tools/dochack/dochack.nim")
257    nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" %
258        [args, VersionAsString])
259
260  proc website(args: string) =
261    nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" %
262        [args, VersionAsString])
263
264  proc pdf(args="") =
265    exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" %
266        [findNim().quoteShell(), args, VersionAsString], additionalPATH=findNim().splitFile.dir)
267
268# -------------- boot ---------------------------------------------------------
269
270proc findStartNim: string =
271  # we try several things before giving up:
272  # * nimExe
273  # * bin/nim
274  # * $PATH/nim
275  # If these fail, we try to build nim with the "build.(sh|bat)" script.
276  let (nim, ok) = findNimImpl()
277  if ok: return nim
278  when defined(posix):
279    const buildScript = "build.sh"
280    if fileExists(buildScript):
281      if tryExec("./" & buildScript): return "bin" / nim
282  else:
283    const buildScript = "build.bat"
284    if fileExists(buildScript):
285      if tryExec(buildScript): return "bin" / nim
286
287  echo("Found no nim compiler and every attempt to build one failed!")
288  quit("FAILURE")
289
290proc thVersion(i: int): string =
291  result = ("compiler" / "nim" & $i).exe
292
293template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true"
294
295proc boot(args: string) =
296  ## bootstrapping is a process that involves 3 steps:
297  ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but
298  ## rock solid for building a Nim compiler. It shouldn't be used for anything else.
299  ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need.
300  ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps.
301  ## This step ensures a minimum amount of quality. We know that nim2.exe can be used
302  ## for Nim compiler development.
303  var output = "compiler" / "nim".exe
304  var finalDest = "bin" / "nim".exe
305  # default to use the 'c' command:
306  let useCpp = doUseCpp()
307  let smartNimcache = (if "release" in args or "danger" in args: "nimcache/r_" else: "nimcache/d_") &
308                      hostOS & "_" & hostCPU
309
310  let nimStart = findStartNim().quoteShell()
311  for i in 0..2:
312    # Nim versions < (1, 1) expect Nim's exception type to have a 'raiseId' field for
313    # C++ interop. Later Nim versions do this differently and removed the 'raiseId' field.
314    # Thus we always bootstrap the first iteration with "c" and not with "cpp" as
315    # a workaround.
316    let defaultCommand = if useCpp and i > 0: "cpp" else: "c"
317    let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: ""
318    echo "iteration: ", i+1
319    var extraOption = ""
320    var nimi = i.thVersion
321    if i == 0:
322      nimi = nimStart
323      extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap"
324        # The configs are skipped for bootstrap
325        # (1st iteration) to prevent newer flags from breaking bootstrap phase.
326      let ret = execCmdEx(nimStart & " --version")
327      doAssert ret.exitCode == 0
328      let version = ret.output.splitLines[0]
329      if version.startsWith "Nim Compiler Version 0.20.0":
330        extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291
331
332    # in order to use less memory, we split the build into two steps:
333    # --compileOnly produces a $project.json file and does not run GCC/Clang.
334    # jsonbuild then uses the $project.json file to build the Nim binary.
335    exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" %
336      [nimi, bootOptions, extraOption, smartNimcache, args]
337    exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" %
338      [nimi, smartNimcache, args]
339
340    if sameFileContent(output, i.thVersion):
341      copyExe(output, finalDest)
342      echo "executables are equal: SUCCESS!"
343      return
344    copyExe(output, (i+1).thVersion)
345  copyExe(output, finalDest)
346  when not defined(windows): echo "[Warning] executables are still not equal"
347
348# -------------- clean --------------------------------------------------------
349
350const
351  cleanExt = [
352    ".ppu", ".o", ".obj", ".dcu", ".~pas", ".~inc", ".~dsk", ".~dpr",
353    ".map", ".tds", ".err", ".bak", ".pyc", ".exe", ".rod", ".pdb", ".idb",
354    ".idx", ".ilk"
355  ]
356  ignore = [
357    ".bzrignore", "nim", "nim.exe", "koch", "koch.exe", ".gitignore"
358  ]
359
360proc cleanAux(dir: string) =
361  for kind, path in walkDir(dir):
362    case kind
363    of pcFile:
364      var (_, name, ext) = splitFile(path)
365      if ext == "" or cleanExt.contains(ext):
366        if not ignore.contains(name):
367          echo "removing: ", path
368          removeFile(path)
369    of pcDir:
370      case splitPath(path).tail
371      of "nimcache":
372        echo "removing dir: ", path
373        removeDir(path)
374      of "dist", ".git", "icons": discard
375      else: cleanAux(path)
376    else: discard
377
378proc removePattern(pattern: string) =
379  for f in walkFiles(pattern):
380    echo "removing: ", f
381    removeFile(f)
382
383proc clean(args: string) =
384  removePattern("web/*.html")
385  removePattern("doc/*.html")
386  cleanAux(getCurrentDir())
387  for kind, path in walkDir(getCurrentDir() / "build"):
388    if kind == pcDir:
389      echo "removing dir: ", path
390      removeDir(path)
391
392# -------------- builds a release ---------------------------------------------
393
394proc winReleaseArch(arch: string) =
395  doAssert arch in ["32", "64"]
396  let cpu = if arch == "32": "i386" else: "amd64"
397
398  template withMingw(path, body) =
399    let prevPath = getEnv("PATH")
400    putEnv("PATH", (if path.len > 0: path & PathSep else: "") & prevPath)
401    try:
402      body
403    finally:
404      putEnv("PATH", prevPath)
405
406  withMingw r"..\mingw" & arch & r"\bin":
407    # Rebuilding koch is necessary because it uses its pointer size to
408    # determine which mingw link to put in the NSIS installer.
409    inFold "winrelease koch":
410      nimexec "c --cpu:$# koch" % cpu
411    kochExecFold("winrelease boot", "boot -d:release --cpu:$#" % cpu)
412    kochExecFold("winrelease zip", "zip -d:release")
413    overwriteFile r"build\nim-$#.zip" % VersionAsString,
414             r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch]
415
416proc winRelease*() =
417  # Now used from "tools/winrelease" and not directly supported by koch
418  # anymore!
419  # Build -docs file:
420  when true:
421    inFold "winrelease buildDocs":
422      buildDocs(gaCode)
423    withDir "web/upload/" & VersionAsString:
424      inFold "winrelease zipdocs":
425        exec "7z a -tzip docs-$#.zip *.html" % VersionAsString
426    overwriteFile "web/upload/$1/docs-$1.zip" % VersionAsString,
427                  "web/upload/download/docs-$1.zip" % VersionAsString
428  when true:
429    inFold "winrelease csource":
430      csource("-d:danger")
431  when sizeof(pointer) == 4:
432    winReleaseArch "32"
433  when sizeof(pointer) == 8:
434    winReleaseArch "64"
435
436# -------------- tests --------------------------------------------------------
437
438template `|`(a, b): string = (if a.len > 0: a else: b)
439
440proc tests(args: string) =
441  nimexec "cc --opt:speed testament/testament"
442  var testCmd = quoteShell(getCurrentDir() / "testament/testament".exe)
443  testCmd.add " " & quoteShell("--nim:" & findNim())
444  testCmd.add " " & (args|"all")
445  let success = tryExec testCmd
446  if not success:
447    quit("tests failed", QuitFailure)
448
449proc temp(args: string) =
450  proc splitArgs(a: string): (string, string) =
451    # every --options before the command (indicated by starting
452    # with not a dash) is part of the bootArgs, the rest is part
453    # of the programArgs:
454    let args = os.parseCmdLine a
455    result = ("", "")
456    var i = 0
457    while i < args.len and args[i][0] == '-':
458      result[0].add " " & quoteShell(args[i])
459      inc i
460    while i < args.len:
461      result[1].add " " & quoteShell(args[i])
462      inc i
463
464  let d = getAppDir()
465  let output = d / "compiler" / "nim".exe
466  let finalDest = d / "bin" / "nim_temp".exe
467  # 125 is the magic number to tell git bisect to skip the current commit.
468  var (bootArgs, programArgs) = splitArgs(args)
469  if "doc" notin programArgs and
470      "threads" notin programArgs and
471      "js" notin programArgs and "rst2html" notin programArgs:
472    bootArgs = " -d:leanCompiler" & bootArgs
473  let nimexec = findNim().quoteShell()
474  exec(nimexec & " c -d:debug --debugger:native -d:nimBetterRun " & bootArgs & " " & (d / "compiler" / "nim"), 125)
475  copyExe(output, finalDest)
476  setCurrentDir(origDir)
477  if programArgs.len > 0: exec(finalDest & " " & programArgs)
478
479proc xtemp(cmd: string) =
480  let d = getAppDir()
481  copyExe(d / "bin" / "nim".exe, d / "bin" / "nim_backup".exe)
482  try:
483    withDir(d):
484      temp""
485    copyExe(d / "bin" / "nim_temp".exe, d / "bin" / "nim".exe)
486    exec(cmd)
487  finally:
488    copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
489
490proc icTest(args: string) =
491  temp("")
492  let inp = os.parseCmdLine(args)[0]
493  let content = readFile(inp)
494  let nimExe = getAppDir() / "bin" / "nim_temp".exe
495  var i = 0
496  for fragment in content.split("#!EDIT!#"):
497    let file = inp.replace(".nim", "_temp.nim")
498    writeFile(file, fragment)
499    var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd "
500    if i == 0:
501      cmd.add "-f "
502    cmd.add quoteShell(file)
503    exec(cmd)
504    inc i
505
506#proc buildDrNim(args: string) =
507#  if not dirExists("dist/nimz3"):
508#    exec("git clone https://github.com/zevv/nimz3.git dist/nimz3")
509#  when defined(windows):
510#    if not dirExists("dist/dlls"):
511#      exec("git clone -q https://github.com/nim-lang/dlls.git dist/dlls")
512#    copyExe("dist/dlls/libz3.dll", "bin/libz3.dll")
513#    execFold("build drnim", "nim c -o:$1 $2 drnim/drnim" % ["bin/drnim".exe, args])
514#  else:
515#    if not dirExists("dist/z3"):
516#      exec("git clone -q https://github.com/Z3Prover/z3.git dist/z3")
517#      withDir("dist/z3"):
518#        exec("git fetch")
519#        exec("git checkout " & Z3StableCommit)
520#        createDir("build")
521#        withDir("build"):
522#          exec("""cmake -DZ3_BUILD_LIBZ3_SHARED=FALSE -G "Unix Makefiles" ../""")
523#          exec("make -j4")
524#    execFold("build drnim", "nim cpp --dynlibOverride=libz3 -o:$1 $2 drnim/drnim" % ["bin/drnim".exe, args])
525#  # always run the tests for now:
526#  exec("testament/testament".exe & " --nim:" & "drnim".exe & " pat drnim/tests")
527
528
529proc hostInfo(): string =
530  "hostOS: $1, hostCPU: $2, int: $3, float: $4, cpuEndian: $5, cwd: $6" %
531    [hostOS, hostCPU, $int.sizeof, $float.sizeof, $cpuEndian, getCurrentDir()]
532
533proc installDeps(dep: string, commit = "") =
534  # the hashes/urls are version controlled here, so can be changed seamlessly
535  # and tied to a nim release (mimicking git submodules)
536  var commit = commit
537  case dep
538  #of "tinyc":
539  #  if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3"
540  #  cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit)
541  else: doAssert false, "unsupported: " & dep
542  # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206
543
544proc runCI(cmd: string) =
545  doAssert cmd.len == 0, cmd # avoid silently ignoring
546  echo "runCI: ", cmd
547  echo hostInfo()
548  # boot without -d:nimHasLibFFI to make sure this still works
549  # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in
550  # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd).
551  kochExecFold("Boot in release mode", "boot -d:release -d:nimStrictMode --lib:lib")
552
553  when false: # debugging: when you need to run only 1 test in CI, use something like this:
554    execFold("debugging test", "nim r tests/stdlib/tosproc.nim")
555    doAssert false, "debugging only"
556
557  ## build nimble early on to enable remainder to depend on it if needed
558  kochExecFold("Build Nimble", "nimble")
559
560  let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_")
561  if getEnv("NIM_TEST_PACKAGES", "0") == "1":
562    execFold("Test selected Nimble packages", "nim r testament/testament $# pcat nimble-packages" % batchParam)
563  else:
564    buildTools()
565
566    for a in "zip opengl sdl1 jester@#head".split:
567      let buildDeps = "build"/"deps" # xxx factor pending https://github.com/timotheecour/Nim/issues/616
568      # if this gives `Additional info: "build/deps" [OSError]`, make sure nimble is >= v0.12.0,
569      # otherwise `absolutePath` is needed, refs https://github.com/nim-lang/nimble/issues/901
570      execFold("", "nimble install -y --nimbleDir:$# $#" % [buildDeps.quoteShell, a])
571
572    ## run tests
573    execFold("Test nimscript", "nim e tests/test_nimscript.nims")
574    when defined(windows):
575      execFold("Compile tester", "nim c --usenimcache -d:nimCoroutines --os:genode -d:posix --compileOnly testament/testament")
576
577    # main bottleneck here
578    # xxx: even though this is the main bottleneck, we could speedup the rest via batching with `--batch`.
579    # BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch, pending bug #14343
580    execFold("Run tester", "nim c -r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimStrictMode testament/testament $# all -d:nimCoroutines" % batchParam)
581
582    block: # nimHasLibFFI:
583      when defined(posix): # windows can be handled in future PR's
584        execFold("nimble install -y libffi", "nimble install -y libffi")
585        const nimFFI = "bin/nim.ctffi"
586        # no need to bootstrap with koch boot (would be slower)
587        let backend = if doUseCpp(): "cpp" else: "c"
588        execFold("build with -d:nimHasLibFFI", "nim $1 -d:release -d:nimHasLibFFI -o:$2 compiler/nim.nim" % [backend, nimFFI])
589        execFold("test with -d:nimHasLibFFI", "$1 $2 -r testament/testament --nim:$1 r tests/misc/trunner.nim -d:nimTrunnerFfi" % [nimFFI, backend])
590
591    execFold("Run nimdoc tests", "nim r nimdoc/tester")
592    execFold("Run rst2html tests", "nim r nimdoc/rsttester")
593    execFold("Run nimpretty tests", "nim r nimpretty/tester.nim")
594    when defined(posix):
595      # refs #18385, build with -d:release instead of -d:danger for testing
596      # We could also skip building nimsuggest in buildTools, or build it with -d:release
597      # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage
598      # of rebuilding is this won't affect bin/nimsuggest when running runCI locally
599      execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest")
600      execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
601
602    #execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls")
603
604  when not defined(bsd):
605    if not doUseCpp:
606      # the BSDs are overwhelmed already, so only run this test on the other machines:
607      kochExecFold("Boot Nim ORC", "boot -d:release --mm:orc --lib:lib")
608
609proc testUnixInstall(cmdLineRest: string) =
610  csource("-d:danger" & cmdLineRest)
611  xz(false, cmdLineRest)
612  let oldCurrentDir = getCurrentDir()
613  try:
614    let destDir = getTempDir()
615    copyFile("build/nim-$1.tar.xz" % VersionAsString,
616             destDir / "nim-$1.tar.xz" % VersionAsString)
617    setCurrentDir(destDir)
618    execCleanPath("tar -xJf nim-$1.tar.xz" % VersionAsString)
619    setCurrentDir("nim-$1" % VersionAsString)
620    execCleanPath("sh build.sh")
621    # first test: try if './bin/nim --version' outputs something sane:
622    let output = execProcess("./bin/nim --version").splitLines
623    if output.len > 0 and output[0].contains(VersionAsString):
624      echo "Version check: success"
625      execCleanPath("./bin/nim c koch.nim")
626      execCleanPath("./koch boot -d:release", destDir / "bin")
627      # check the docs build:
628      execCleanPath("./koch docs", destDir / "bin")
629      # check nimble builds:
630      execCleanPath("./koch tools")
631      # check the tests work:
632      putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH")
633      execCleanPath("./koch tests --nim:bin/nim cat megatest", destDir / "bin")
634    else:
635      echo "Version check: failure"
636  finally:
637    setCurrentDir oldCurrentDir
638
639proc valgrind(cmd: string) =
640  # somewhat hacky: '=' sign means "pass to valgrind" else "pass to Nim"
641  let args = parseCmdLine(cmd)
642  var nimcmd = ""
643  var valcmd = ""
644  for i, a in args:
645    if i == args.len-1:
646      # last element is the filename:
647      valcmd.add ' '
648      valcmd.add changeFileExt(a, ExeExt)
649      nimcmd.add ' '
650      nimcmd.add a
651    elif '=' in a:
652      valcmd.add ' '
653      valcmd.add a
654    else:
655      nimcmd.add ' '
656      nimcmd.add a
657  exec("nim c" & nimcmd)
658  let supp = getAppDir() / "tools" / "nimgrind.supp"
659  exec("valgrind --suppressions=" & supp & valcmd)
660
661proc showHelp(success: bool) =
662  quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)),
663                   CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure)
664
665#proc branchDone() =
666#  let thisBranch = execProcess("git symbolic-ref --short HEAD").strip()
667#  if thisBranch != "devel" and thisBranch != "":
668#    exec("git checkout devel")
669#    exec("git branch -D " & thisBranch)
670#    exec("git pull --rebase")
671
672when isMainModule:
673  var op = initOptParser()
674  var
675    latest = false
676    localDocsOnly = false
677    localDocsOut = ""
678  while true:
679    op.next()
680    case op.kind
681    of cmdLongOption, cmdShortOption:
682      case normalize(op.key)
683      of "help", "h": showHelp(success = true)
684      of "latest": latest = true
685      of "stable": latest = false
686      of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir
687      of "localdocs":
688        localDocsOnly = true
689        if op.val.len > 0:
690          localDocsOut = op.val.absolutePath
691      else: showHelp(success = false)
692    of cmdArgument:
693      case normalize(op.key)
694      of "boot": boot(op.cmdLineRest)
695      of "clean": clean(op.cmdLineRest)
696      of "doc", "docs": buildDocs(op.cmdLineRest, localDocsOnly, localDocsOut)
697      of "doc0", "docs0":
698        # undocumented command for Araq-the-merciful:
699        buildDocs(op.cmdLineRest & gaCode)
700      of "pdf": buildPdfDoc(op.cmdLineRest, "doc/pdf")
701      of "csource", "csources": csource(op.cmdLineRest)
702      of "zip": zip(latest, op.cmdLineRest)
703      of "xz": xz(latest, op.cmdLineRest)
704      of "nsis": nsis(latest, op.cmdLineRest)
705      of "geninstall": geninstall(op.cmdLineRest)
706      of "distrohelper": geninstall()
707      of "install": install(op.cmdLineRest)
708      of "testinstall": testUnixInstall(op.cmdLineRest)
709      of "installdeps": installDeps(op.cmdLineRest)
710      of "runci": runCI(op.cmdLineRest)
711      of "test", "tests": tests(op.cmdLineRest)
712      of "temp": temp(op.cmdLineRest)
713      of "xtemp": xtemp(op.cmdLineRest)
714      of "wintools": bundleWinTools(op.cmdLineRest)
715      #of "nimble": bundleNimbleExe(latest, op.cmdLineRest)
716      of "nimsuggest": bundleNimsuggest(op.cmdLineRest)
717      # toolsNoNimble is kept for backward compatibility with build scripts
718      of "toolsnonimble", "toolsnoexternal":
719        buildTools(op.cmdLineRest)
720      of "tools":
721        buildTools(op.cmdLineRest)
722        #bundleNimbleExe(latest, op.cmdLineRest)
723      #of "pushcsource":
724      #  quit "use this instead: https://github.com/nim-lang/csources_v1/blob/master/push_c_code.nim"
725      of "valgrind": valgrind(op.cmdLineRest)
726      #of "c2nim": bundleC2nim(op.cmdLineRest)
727      #of "drnim": buildDrNim(op.cmdLineRest)
728      of "fusion":
729        let suffix = if latest: HeadHash else: FusionStableHash
730        exec("nimble install -y fusion@$#" % suffix)
731      of "ic": icTest(op.cmdLineRest)
732      #of "branchdone": branchDone()
733      else: showHelp(success = false)
734      break
735    of cmdEnd:
736      showHelp(success = false)
737