1package dockerfile // import "github.com/docker/docker/builder/dockerfile" 2 3import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "sort" 10 "strings" 11 12 "github.com/containerd/containerd/platforms" 13 "github.com/docker/docker/api/types" 14 "github.com/docker/docker/api/types/backend" 15 "github.com/docker/docker/api/types/container" 16 "github.com/docker/docker/builder" 17 "github.com/docker/docker/builder/remotecontext" 18 "github.com/docker/docker/errdefs" 19 "github.com/docker/docker/pkg/idtools" 20 "github.com/docker/docker/pkg/streamformatter" 21 "github.com/docker/docker/pkg/stringid" 22 "github.com/docker/docker/pkg/system" 23 "github.com/moby/buildkit/frontend/dockerfile/instructions" 24 "github.com/moby/buildkit/frontend/dockerfile/parser" 25 "github.com/moby/buildkit/frontend/dockerfile/shell" 26 specs "github.com/opencontainers/image-spec/specs-go/v1" 27 "github.com/pkg/errors" 28 "github.com/sirupsen/logrus" 29 "golang.org/x/sync/syncmap" 30) 31 32var validCommitCommands = map[string]bool{ 33 "cmd": true, 34 "entrypoint": true, 35 "healthcheck": true, 36 "env": true, 37 "expose": true, 38 "label": true, 39 "onbuild": true, 40 "user": true, 41 "volume": true, 42 "workdir": true, 43} 44 45const ( 46 stepFormat = "Step %d/%d : %v" 47) 48 49// BuildManager is shared across all Builder objects 50type BuildManager struct { 51 idMapping *idtools.IdentityMapping 52 backend builder.Backend 53 pathCache pathCache // TODO: make this persistent 54} 55 56// NewBuildManager creates a BuildManager 57func NewBuildManager(b builder.Backend, identityMapping *idtools.IdentityMapping) (*BuildManager, error) { 58 bm := &BuildManager{ 59 backend: b, 60 pathCache: &syncmap.Map{}, 61 idMapping: identityMapping, 62 } 63 return bm, nil 64} 65 66// Build starts a new build from a BuildConfig 67func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (*builder.Result, error) { 68 buildsTriggered.Inc() 69 if config.Options.Dockerfile == "" { 70 config.Options.Dockerfile = builder.DefaultDockerfileName 71 } 72 73 source, dockerfile, err := remotecontext.Detect(config) 74 if err != nil { 75 return nil, err 76 } 77 defer func() { 78 if source != nil { 79 if err := source.Close(); err != nil { 80 logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err) 81 } 82 } 83 }() 84 85 ctx, cancel := context.WithCancel(ctx) 86 defer cancel() 87 88 builderOptions := builderOptions{ 89 Options: config.Options, 90 ProgressWriter: config.ProgressWriter, 91 Backend: bm.backend, 92 PathCache: bm.pathCache, 93 IDMapping: bm.idMapping, 94 } 95 b, err := newBuilder(ctx, builderOptions) 96 if err != nil { 97 return nil, err 98 } 99 return b.build(source, dockerfile) 100} 101 102// builderOptions are the dependencies required by the builder 103type builderOptions struct { 104 Options *types.ImageBuildOptions 105 Backend builder.Backend 106 ProgressWriter backend.ProgressWriter 107 PathCache pathCache 108 IDMapping *idtools.IdentityMapping 109} 110 111// Builder is a Dockerfile builder 112// It implements the builder.Backend interface. 113type Builder struct { 114 options *types.ImageBuildOptions 115 116 Stdout io.Writer 117 Stderr io.Writer 118 Aux *streamformatter.AuxFormatter 119 Output io.Writer 120 121 docker builder.Backend 122 clientCtx context.Context 123 124 idMapping *idtools.IdentityMapping 125 disableCommit bool 126 imageSources *imageSources 127 pathCache pathCache 128 containerManager *containerManager 129 imageProber ImageProber 130 platform *specs.Platform 131} 132 133// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options. 134func newBuilder(clientCtx context.Context, options builderOptions) (*Builder, error) { 135 config := options.Options 136 if config == nil { 137 config = new(types.ImageBuildOptions) 138 } 139 140 b := &Builder{ 141 clientCtx: clientCtx, 142 options: config, 143 Stdout: options.ProgressWriter.StdoutFormatter, 144 Stderr: options.ProgressWriter.StderrFormatter, 145 Aux: options.ProgressWriter.AuxFormatter, 146 Output: options.ProgressWriter.Output, 147 docker: options.Backend, 148 idMapping: options.IDMapping, 149 imageSources: newImageSources(clientCtx, options), 150 pathCache: options.PathCache, 151 imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache), 152 containerManager: newContainerManager(options.Backend), 153 } 154 155 // same as in Builder.Build in builder/builder-next/builder.go 156 // TODO: remove once config.Platform is of type specs.Platform 157 if config.Platform != "" { 158 sp, err := platforms.Parse(config.Platform) 159 if err != nil { 160 return nil, err 161 } 162 if err := system.ValidatePlatform(sp); err != nil { 163 return nil, err 164 } 165 b.platform = &sp 166 } 167 168 return b, nil 169} 170 171// Build 'LABEL' command(s) from '--label' options and add to the last stage 172func buildLabelOptions(labels map[string]string, stages []instructions.Stage) { 173 keys := []string{} 174 for key := range labels { 175 keys = append(keys, key) 176 } 177 178 // Sort the label to have a repeatable order 179 sort.Strings(keys) 180 for _, key := range keys { 181 value := labels[key] 182 stages[len(stages)-1].AddCommand(instructions.NewLabelCommand(key, value, true)) 183 } 184} 185 186// Build runs the Dockerfile builder by parsing the Dockerfile and executing 187// the instructions from the file. 188func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) { 189 defer b.imageSources.Unmount() 190 191 stages, metaArgs, err := instructions.Parse(dockerfile.AST) 192 if err != nil { 193 var uiErr *instructions.UnknownInstruction 194 if errors.As(err, &uiErr) { 195 buildsFailed.WithValues(metricsUnknownInstructionError).Inc() 196 } 197 return nil, errdefs.InvalidParameter(err) 198 } 199 if b.options.Target != "" { 200 targetIx, found := instructions.HasStage(stages, b.options.Target) 201 if !found { 202 buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc() 203 return nil, errdefs.InvalidParameter(errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)) 204 } 205 stages = stages[:targetIx+1] 206 } 207 208 // Add 'LABEL' command specified by '--label' option to the last stage 209 buildLabelOptions(b.options.Labels, stages) 210 211 dockerfile.PrintWarnings(b.Stderr) 212 dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source) 213 if err != nil { 214 return nil, err 215 } 216 if dispatchState.imageID == "" { 217 buildsFailed.WithValues(metricsDockerfileEmptyError).Inc() 218 return nil, errors.New("No image was generated. Is your Dockerfile empty?") 219 } 220 return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil 221} 222 223func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error { 224 if aux == nil || state.imageID == "" { 225 return nil 226 } 227 return aux.Emit("", types.BuildResult{ID: state.imageID}) 228} 229 230func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *BuildArgs) error { 231 // shell.Lex currently only support the concatenated string format 232 envs := convertMapToEnvList(args.GetAllAllowed()) 233 if err := meta.Expand(func(word string) (string, error) { 234 return shlex.ProcessWord(word, envs) 235 }); err != nil { 236 return err 237 } 238 for _, arg := range meta.Args { 239 args.AddArg(arg.Key, arg.Value) 240 args.AddMetaArg(arg.Key, arg.Value) 241 } 242 return nil 243} 244 245func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int { 246 fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd) 247 fmt.Fprintln(out) 248 return currentCommandIndex + 1 249} 250 251func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) { 252 dispatchRequest := dispatchRequest{} 253 buildArgs := NewBuildArgs(b.options.BuildArgs) 254 totalCommands := len(metaArgs) + len(parseResult) 255 currentCommandIndex := 1 256 for _, stage := range parseResult { 257 totalCommands += len(stage.Commands) 258 } 259 shlex := shell.NewLex(escapeToken) 260 for _, meta := range metaArgs { 261 currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta) 262 263 err := processMetaArg(meta, shlex, buildArgs) 264 if err != nil { 265 return nil, err 266 } 267 } 268 269 stagesResults := newStagesBuildResults() 270 271 for _, stage := range parseResult { 272 if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil { 273 return nil, err 274 } 275 dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults) 276 277 currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode) 278 if err := initializeStage(dispatchRequest, &stage); err != nil { 279 return nil, err 280 } 281 dispatchRequest.state.updateRunConfig() 282 fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID)) 283 for _, cmd := range stage.Commands { 284 select { 285 case <-b.clientCtx.Done(): 286 logrus.Debug("Builder: build cancelled!") 287 fmt.Fprint(b.Stdout, "Build cancelled\n") 288 buildsFailed.WithValues(metricsBuildCanceled).Inc() 289 return nil, errors.New("Build cancelled") 290 default: 291 // Not cancelled yet, keep going... 292 } 293 294 currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd) 295 296 if err := dispatch(dispatchRequest, cmd); err != nil { 297 return nil, err 298 } 299 dispatchRequest.state.updateRunConfig() 300 fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID)) 301 302 } 303 if err := emitImageID(b.Aux, dispatchRequest.state); err != nil { 304 return nil, err 305 } 306 buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs) 307 if err := commitStage(dispatchRequest.state, stagesResults); err != nil { 308 return nil, err 309 } 310 } 311 buildArgs.WarnOnUnusedBuildArgs(b.Stdout) 312 return dispatchRequest.state, nil 313} 314 315// BuildFromConfig builds directly from `changes`, treating it as if it were the contents of a Dockerfile 316// It will: 317// - Call parse.Parse() to get an AST root for the concatenated Dockerfile entries. 318// - Do build by calling builder.dispatch() to call all entries' handling routines 319// 320// BuildFromConfig is used by the /commit endpoint, with the changes 321// coming from the query parameter of the same name. 322// 323// TODO: Remove? 324func BuildFromConfig(config *container.Config, changes []string, os string) (*container.Config, error) { 325 if !system.IsOSSupported(os) { 326 return nil, errdefs.InvalidParameter(system.ErrNotSupportedOperatingSystem) 327 } 328 if len(changes) == 0 { 329 return config, nil 330 } 331 332 dockerfile, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n"))) 333 if err != nil { 334 return nil, errdefs.InvalidParameter(err) 335 } 336 337 b, err := newBuilder(context.Background(), builderOptions{ 338 Options: &types.ImageBuildOptions{NoCache: true}, 339 }) 340 if err != nil { 341 return nil, err 342 } 343 344 // ensure that the commands are valid 345 for _, n := range dockerfile.AST.Children { 346 if !validCommitCommands[n.Value] { 347 return nil, errdefs.InvalidParameter(errors.Errorf("%s is not a valid change command", n.Value)) 348 } 349 } 350 351 b.Stdout = ioutil.Discard 352 b.Stderr = ioutil.Discard 353 b.disableCommit = true 354 355 var commands []instructions.Command 356 for _, n := range dockerfile.AST.Children { 357 cmd, err := instructions.ParseCommand(n) 358 if err != nil { 359 return nil, errdefs.InvalidParameter(err) 360 } 361 commands = append(commands, cmd) 362 } 363 364 dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, NewBuildArgs(b.options.BuildArgs), newStagesBuildResults()) 365 // We make mutations to the configuration, ensure we have a copy 366 dispatchRequest.state.runConfig = copyRunConfig(config) 367 dispatchRequest.state.imageID = config.Image 368 dispatchRequest.state.operatingSystem = os 369 for _, cmd := range commands { 370 err := dispatch(dispatchRequest, cmd) 371 if err != nil { 372 return nil, errdefs.InvalidParameter(err) 373 } 374 dispatchRequest.state.updateRunConfig() 375 } 376 377 return dispatchRequest.state.runConfig, nil 378} 379 380func convertMapToEnvList(m map[string]string) []string { 381 result := []string{} 382 for k, v := range m { 383 result = append(result, k+"="+v) 384 } 385 return result 386} 387