1 <#
2 .NOTES
3 Author: @jhowardmsft
4
5 Summary: Windows native build script. This is similar to functionality provided
6 by hack\make.sh, but uses native Windows PowerShell semantics. It does
7 not support the full set of options provided by the Linux counterpart.
8 For example:
9
10 - You can't cross-build Linux docker binaries on Windows
11 - Hashes aren't generated on binaries
12 - 'Releasing' isn't supported.
13 - Integration tests. This is because they currently cannot run inside a container,
14 and require significant external setup.
15
16 It does however provided the minimum necessary to support parts of local Windows
17 development and Windows to Windows CI.
18
19 Usage Examples (run from repo root):
20 "hack\make.ps1 -Client" to build docker.exe client 64-bit binary (remote repo)
21 "hack\make.ps1 -TestUnit" to run unit tests
22 "hack\make.ps1 -Daemon -TestUnit" to build the daemon and run unit tests
23 "hack\make.ps1 -All" to run everything this script knows about that can run in a container
24 "hack\make.ps1" to build the daemon binary (same as -Daemon)
25 "hack\make.ps1 -Binary" shortcut to -Client and -Daemon
26
27 .PARAMETER Client
28 Builds the client binaries.
29
30 .PARAMETER Daemon
31 Builds the daemon binary.
32
33 .PARAMETER Binary
34 Builds the client and daemon binaries. A convenient shortcut to `make.ps1 -Client -Daemon`.
35
36 .PARAMETER Race
37 Use -race in go build and go test.
38
39 .PARAMETER Noisy
40 Use -v in go build.
41
42 .PARAMETER ForceBuildAll
43 Use -a in go build.
44
45 .PARAMETER NoOpt
46 Use -gcflags -N -l in go build to disable optimisation (can aide debugging).
47
48 .PARAMETER CommitSuffix
49 Adds a custom string to be appended to the commit ID (spaces are stripped).
50
51 .PARAMETER DCO
52 Runs the DCO (Developer Certificate Of Origin) test (must be run outside a container).
53
54 .PARAMETER PkgImports
55 Runs the pkg\ directory imports test (must be run outside a container).
56
57 .PARAMETER GoFormat
58 Runs the Go formatting test (must be run outside a container).
59
60 .PARAMETER TestUnit
61 Runs unit tests.
62
63 .PARAMETER All
64 Runs everything this script knows about that can run in a container.
65
66
67 TODO
68 - Unify the head commit
69 - Add golint and other checks (swagger maybe?)
70
71 #>
72
73
74 param(
75 [Parameter(Mandatory=$False)][switch]$Client,
76 [Parameter(Mandatory=$False)][switch]$Daemon,
77 [Parameter(Mandatory=$False)][switch]$Binary,
78 [Parameter(Mandatory=$False)][switch]$Race,
79 [Parameter(Mandatory=$False)][switch]$Noisy,
80 [Parameter(Mandatory=$False)][switch]$ForceBuildAll,
81 [Parameter(Mandatory=$False)][switch]$NoOpt,
82 [Parameter(Mandatory=$False)][string]$CommitSuffix="",
83 [Parameter(Mandatory=$False)][switch]$DCO,
84 [Parameter(Mandatory=$False)][switch]$PkgImports,
85 [Parameter(Mandatory=$False)][switch]$GoFormat,
86 [Parameter(Mandatory=$False)][switch]$TestUnit,
87 [Parameter(Mandatory=$False)][switch]$All
88 )
89
90 $ErrorActionPreference = "Stop"
91 $ProgressPreference = "SilentlyContinue"
92 $pushed=$False # To restore the directory if we have temporarily pushed to one.
93
94 # Utility function to get the commit ID of the repository
Get-GitCommit()95 Function Get-GitCommit() {
96 if (-not (Test-Path ".\.git")) {
97 # If we don't have a .git directory, but we do have the environment
98 # variable DOCKER_GITCOMMIT set, that can override it.
99 if ($env:DOCKER_GITCOMMIT.Length -eq 0) {
100 Throw ".git directory missing and DOCKER_GITCOMMIT environment variable not specified."
101 }
102 Write-Host "INFO: Git commit ($env:DOCKER_GITCOMMIT) assumed from DOCKER_GITCOMMIT environment variable"
103 return $env:DOCKER_GITCOMMIT
104 }
105 $gitCommit=$(git rev-parse --short HEAD)
106 if ($(git status --porcelain --untracked-files=no).Length -ne 0) {
107 $gitCommit="$gitCommit-unsupported"
108 Write-Host ""
109 Write-Warning "This version is unsupported because there are uncommitted file(s)."
110 Write-Warning "Either commit these changes, or add them to .gitignore."
111 git status --porcelain --untracked-files=no | Write-Warning
112 Write-Host ""
113 }
114 return $gitCommit
115 }
116
117 # Utility function to determine if we are running in a container or not.
118 # In Windows, we get this through an environment variable set in `Dockerfile.Windows`
Check-InContainer()119 Function Check-InContainer() {
120 if ($env:FROM_DOCKERFILE.Length -eq 0) {
121 Write-Host ""
122 Write-Warning "Not running in a container. The result might be an incorrect build."
123 Write-Host ""
124 return $False
125 }
126 return $True
127 }
128
129 # Utility function to warn if the version of go is correct. Used for local builds
130 # outside of a container where it may be out of date with master.
Verify-GoVersion()131 Function Verify-GoVersion() {
132 Try {
133 $goVersionDockerfile=(Get-Content ".\Dockerfile" | Select-String "ENV GO_VERSION").ToString().Split(" ")[2]
134 $goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2)
135 }
136 Catch [Exception] {
137 Throw "Failed to validate go version correctness: $_"
138 }
139 if (-not($goVersionInstalled -eq $goVersionDockerfile)) {
140 Write-Host ""
141 Write-Warning "Building with golang version $goVersionInstalled. You should update to $goVersionDockerfile"
142 Write-Host ""
143 }
144 }
145
146 # Utility function to get the commit for HEAD
Get-HeadCommit()147 Function Get-HeadCommit() {
148 $head = Invoke-Expression "git rev-parse --verify HEAD"
149 if ($LASTEXITCODE -ne 0) { Throw "Failed getting HEAD commit" }
150
151 return $head
152 }
153
154 # Utility function to get the commit for upstream
Get-UpstreamCommit()155 Function Get-UpstreamCommit() {
156 Invoke-Expression "git fetch -q https://github.com/docker/docker.git refs/heads/master"
157 if ($LASTEXITCODE -ne 0) { Throw "Failed fetching" }
158
159 $upstream = Invoke-Expression "git rev-parse --verify FETCH_HEAD"
160 if ($LASTEXITCODE -ne 0) { Throw "Failed getting upstream commit" }
161
162 return $upstream
163 }
164
165 # Build a binary (client or daemon)
Execute-Build($type, $additionalBuildTags, $directory)166 Function Execute-Build($type, $additionalBuildTags, $directory) {
167 # Generate the build flags
168 $buildTags = "autogen"
169 if ($Noisy) { $verboseParm=" -v" }
170 if ($Race) { Write-Warning "Using race detector"; $raceParm=" -race"}
171 if ($ForceBuildAll) { $allParm=" -a" }
172 if ($NoOpt) { $optParm=" -gcflags "+""""+"-N -l"+"""" }
173 if ($additionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) }
174
175 # Do the go build in the appropriate directory
176 # Note -linkmode=internal is required to be able to debug on Windows.
177 # https://github.com/golang/go/issues/14319#issuecomment-189576638
178 Write-Host "INFO: Building $type..."
179 Push-Location $root\cmd\$directory; $global:pushed=$True
180 $buildCommand = "go build" + `
181 $raceParm + `
182 $verboseParm + `
183 $allParm + `
184 $optParm + `
185 " -tags """ + $buildTags + """" + `
186 " -ldflags """ + "-linkmode=internal" + """" + `
187 " -o $root\bundles\"+$directory+".exe"
188 Invoke-Expression $buildCommand
189 if ($LASTEXITCODE -ne 0) { Throw "Failed to compile $type" }
190 Pop-Location; $global:pushed=$False
191 }
192
193
194 # Validates the DCO marker is present on each commit
Validate-DCO($headCommit, $upstreamCommit)195 Function Validate-DCO($headCommit, $upstreamCommit) {
196 Write-Host "INFO: Validating Developer Certificate of Origin..."
197 # Username may only contain alphanumeric characters or dashes and cannot begin with a dash
198 $usernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+'
199
200 $dcoPrefix="Signed-off-by:"
201 $dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \(github: ($usernameRegex)\))?$"
202
203 $counts = Invoke-Expression "git diff --numstat $upstreamCommit...$headCommit"
204 if ($LASTEXITCODE -ne 0) { Throw "Failed git diff --numstat" }
205
206 # Counts of adds and deletes after removing multiple white spaces. AWK anyone? :(
207 $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{
208 $a=$_.Split(" ");
209 if ($a[0] -ne "-") { $adds+=[int]$a[0] }
210 if ($a[1] -ne "-") { $dels+=[int]$a[1] }
211 }
212 if (($adds -eq 0) -and ($dels -eq 0)) {
213 Write-Warning "DCO validation - nothing to validate!"
214 return
215 }
216
217 $commits = Invoke-Expression "git log $upstreamCommit..$headCommit --format=format:%H%n"
218 if ($LASTEXITCODE -ne 0) { Throw "Failed git log --format" }
219 $commits = $($commits -split '\s+' -match '\S')
220 $badCommits=@()
221 $commits | %{
222 # Skip commits with no content such as merge commits etc
223 if ($(git log -1 --format=format: --name-status $_).Length -gt 0) {
224 # Ignore exit code on next call - always process regardless
225 $commitMessage = Invoke-Expression "git log -1 --format=format:%B --name-status $_"
226 if (($commitMessage -match $dcoRegex).Length -eq 0) { $badCommits+=$_ }
227 }
228 }
229 if ($badCommits.Length -eq 0) {
230 Write-Host "Congratulations! All commits are properly signed with the DCO!"
231 } else {
232 $e = "`nThese commits do not have a proper '$dcoPrefix' marker:`n"
233 $badCommits | %{ $e+=" - $_`n"}
234 $e += "`nPlease amend each commit to include a properly formatted DCO marker.`n`n"
235 $e += "Visit the following URL for information about the Docker DCO:`n"
236 $e += "https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work`n"
237 Throw $e
238 }
239 }
240
241 # Validates that .\pkg\... is safely isolated from internal code
Validate-PkgImports($headCommit, $upstreamCommit)242 Function Validate-PkgImports($headCommit, $upstreamCommit) {
243 Write-Host "INFO: Validating pkg import isolation..."
244
245 # Get a list of go source-code files which have changed under pkg\. Ignore exit code on next call - always process regardless
246 $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'pkg\*.go`'"
247 $badFiles=@(); $files | %{
248 $file=$_
249 # For the current changed file, get its list of dependencies, sorted and uniqued.
250 $imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file"
251 if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" }
252 $imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique
253 # Filter out what we are looking for
254 $imports = $imports -NotMatch "^github.com/docker/docker/pkg/" `
255 -NotMatch "^github.com/docker/docker/vendor" `
256 -Match "^github.com/docker/docker" `
257 -Replace "`n", ""
258 $imports | % { $badFiles+="$file imports $_`n" }
259 }
260 if ($badFiles.Length -eq 0) {
261 Write-Host 'Congratulations! ".\pkg\*.go" is safely isolated from internal code.'
262 } else {
263 $e = "`nThese files import internal code: (either directly or indirectly)`n"
264 $badFiles | %{ $e+=" - $_"}
265 Throw $e
266 }
267 }
268
269 # Validates that changed files are correctly go-formatted
Validate-GoFormat($headCommit, $upstreamCommit)270 Function Validate-GoFormat($headCommit, $upstreamCommit) {
271 Write-Host "INFO: Validating go formatting on changed files..."
272
273 # Verify gofmt is installed
274 if ($(Get-Command gofmt -ErrorAction SilentlyContinue) -eq $nil) { Throw "gofmt does not appear to be installed" }
275
276 # Get a list of all go source-code files which have changed. Ignore exit code on next call - always process regardless
277 $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'"
278 $files = $files | Select-String -NotMatch "^vendor/"
279 $badFiles=@(); $files | %{
280 # Deliberately ignore error on next line - treat as failed
281 $content=Invoke-Expression "git show $headCommit`:$_"
282
283 # Next set of hoops are to ensure we have LF not CRLF semantics as otherwise gofmt on Windows will not succeed.
284 # Also note that gofmt on Windows does not appear to support stdin piping correctly. Hence go through a temporary file.
285 $content=$content -join "`n"
286 $content+="`n"
287 $outputFile=[System.IO.Path]::GetTempFileName()
288 if (Test-Path $outputFile) { Remove-Item $outputFile }
289 [System.IO.File]::WriteAllText($outputFile, $content, (New-Object System.Text.UTF8Encoding($False)))
290 $currentFile = $_ -Replace("/","\")
291 Write-Host Checking $currentFile
292 Invoke-Expression "gofmt -s -l $outputFile"
293 if ($LASTEXITCODE -ne 0) { $badFiles+=$currentFile }
294 if (Test-Path $outputFile) { Remove-Item $outputFile }
295 }
296 if ($badFiles.Length -eq 0) {
297 Write-Host 'Congratulations! All Go source files are properly formatted.'
298 } else {
299 $e = "`nThese files are not properly gofmt`'d:`n"
300 $badFiles | %{ $e+=" - $_`n"}
301 $e+= "`nPlease reformat the above files using `"gofmt -s -w`" and commit the result."
302 Throw $e
303 }
304 }
305
306 # Run the unit tests
Run-UnitTests()307 Function Run-UnitTests() {
308 Write-Host "INFO: Running unit tests..."
309 $testPath="./..."
310 $goListCommand = "go list -e -f '{{if ne .Name """ + '\"github.com/docker/docker\"' + """}}{{.ImportPath}}{{end}}' $testPath"
311 $pkgList = $(Invoke-Expression $goListCommand)
312 if ($LASTEXITCODE -ne 0) { Throw "go list for unit tests failed" }
313 $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker"
314 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor"
315 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
316 $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
317 $pkgList = $pkgList -replace "`r`n", " "
318 $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
319 Invoke-Expression $goTestCommand
320 if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
321 }
322
323 # Start of main code.
324 Try {
325 Write-Host -ForegroundColor Cyan "INFO: make.ps1 starting at $(Get-Date)"
326
327 # Get to the root of the repo
328 $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
329 Push-Location $root
330
331 # Handle the "-All" shortcut to turn on all things we can handle.
332 # Note we expressly only include the items which can run in a container - the validations tests cannot
333 # as they require the .git directory which is excluded from the image by .dockerignore
334 if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True }
335
336 # Handle the "-Binary" shortcut to build both client and daemon.
337 if ($Binary) { $Client = $True; $Daemon = $True }
338
339 # Default to building the daemon if not asked for anything explicitly.
340 if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit)) { $Daemon=$True }
341
342 # Verify git is installed
343 if ($(Get-Command git -ErrorAction SilentlyContinue) -eq $nil) { Throw "Git does not appear to be installed" }
344
345 # Verify go is installed
346 if ($(Get-Command go -ErrorAction SilentlyContinue) -eq $nil) { Throw "GoLang does not appear to be installed" }
347
348 # Get the git commit. This will also verify if we are in a repo or not. Then add a custom string if supplied.
349 $gitCommit=Get-GitCommit
350 if ($CommitSuffix -ne "") { $gitCommit += "-"+$CommitSuffix -Replace ' ', '' }
351
352 # Get the version of docker (eg 17.04.0-dev)
353 $dockerVersion="0.0.0-dev"
354
355 # Give a warning if we are not running in a container and are building binaries or running unit tests.
356 # Not relevant for validation tests as these are fine to run outside of a container.
357 if ($Client -or $Daemon -or $TestUnit) { $inContainer=Check-InContainer }
358
359 # If we are not in a container, validate the version of GO that is installed.
360 if (-not $inContainer) { Verify-GoVersion }
361
362 # Verify GOPATH is set
363 if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" }
364
365 # Run autogen if building binaries or running unit tests.
366 if ($Client -or $Daemon -or $TestUnit) {
367 Write-Host "INFO: Invoking autogen..."
368 Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion }
369 Catch [Exception] { Throw $_ }
370 }
371
372 # DCO, Package import and Go formatting tests.
373 if ($DCO -or $PkgImports -or $GoFormat) {
374 # We need the head and upstream commits for these
375 $headCommit=Get-HeadCommit
376 $upstreamCommit=Get-UpstreamCommit
377
378 # Run DCO validation
379 if ($DCO) { Validate-DCO $headCommit $upstreamCommit }
380
381 # Run `gofmt` validation
382 if ($GoFormat) { Validate-GoFormat $headCommit $upstreamCommit }
383
384 # Run pkg isolation validation
385 if ($PkgImports) { Validate-PkgImports $headCommit $upstreamCommit }
386 }
387
388 # Build the binaries
389 if ($Client -or $Daemon) {
390 # Create the bundles directory if it doesn't exist
391 if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
392
393 # Perform the actual build
394 if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }
395 if ($Client) {
396 # Get the Docker channel and version from the environment, or use the defaults.
397 if (-not ($channel = $env:DOCKERCLI_CHANNEL)) { $channel = "edge" }
398 if (-not ($version = $env:DOCKERCLI_VERSION)) { $version = "17.06.0-ce" }
399
400 # Download the zip file and extract the client executable.
401 Write-Host "INFO: Downloading docker/cli version $version from $channel..."
402 $url = "https://download.docker.com/win/static/$channel/x86_64/docker-$version.zip"
403 Invoke-WebRequest $url -OutFile "docker.zip"
404 Try {
405 Add-Type -AssemblyName System.IO.Compression.FileSystem
406 $zip = [System.IO.Compression.ZipFile]::OpenRead("$PWD\docker.zip")
407 Try {
408 if (-not ($entry = $zip.Entries | Where-Object { $_.Name -eq "docker.exe" })) {
409 Throw "Cannot find docker.exe in $url"
410 }
411 [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, "$PWD\bundles\docker.exe", $true)
412 }
413 Finally {
414 $zip.Dispose()
415 }
416 }
417 Finally {
418 Remove-Item -Force "docker.zip"
419 }
420 }
421 }
422
423 # Run unit tests
424 if ($TestUnit) { Run-UnitTests }
425
426 # Gratuitous ASCII art.
427 if ($Daemon -or $Client) {
428 Write-Host
429 Write-Host -ForegroundColor Green " ________ ____ __."
430 Write-Host -ForegroundColor Green " \_____ \ `| `|/ _`|"
431 Write-Host -ForegroundColor Green " / `| \`| `<"
432 Write-Host -ForegroundColor Green " / `| \ `| \"
433 Write-Host -ForegroundColor Green " \_______ /____`|__ \"
434 Write-Host -ForegroundColor Green " \/ \/"
435 Write-Host
436 }
437 }
438 Catch [Exception] {
439 Write-Host -ForegroundColor Red ("`nERROR: make.ps1 failed:`n$_")
440
441 # More gratuitous ASCII art.
442 Write-Host
443 Write-Host -ForegroundColor Red "___________ .__.__ .___"
444 Write-Host -ForegroundColor Red "\_ _____/____ `|__`| `| ____ __`| _/"
445 Write-Host -ForegroundColor Red " `| __) \__ \ `| `| `| _/ __ \ / __ `| "
446 Write-Host -ForegroundColor Red " `| \ / __ \`| `| `|_\ ___// /_/ `| "
447 Write-Host -ForegroundColor Red " \___ / (____ /__`|____/\___ `>____ `| "
448 Write-Host -ForegroundColor Red " \/ \/ \/ \/ "
449 Write-Host
450
451 Throw $_
452 }
453 Finally {
454 Pop-Location # As we pushed to the root of the repo as the very first thing
455 if ($global:pushed) { Pop-Location }
456 Write-Host -ForegroundColor Cyan "INFO: make.ps1 ended at $(Get-Date)"
457 }
458