From 54240f8da9992880e20a1508e9a6f0e59f2adef1 Mon Sep 17 00:00:00 2001 From: Madhav Puri Date: Fri, 14 Nov 2014 10:59:14 -0800 Subject: [PATCH] Support for passing build-time variables in build context - The build-time variables are passed as environment-context for command(s) run as part of the RUN primitve. These variables are not persisted in environment of intermediate and final images when passed as context for RUN. The build environment is prepended to the intermediate continer's command string for aiding cache lookups. It also helps with build traceability. But this also makes the feature less secure from point of view of passing build time secrets. - The build-time variables also get used to expand the symbols used in certain Dockerfile primitves like ADD, COPY, USER etc, without an explicit prior definiton using a ENV primitive. These variables get persisted in the intermediate and final images whenever they are expanded. - The build-time variables are only expanded or passed to the RUN primtive if they are defined in Dockerfile using the ARG primitive or belong to list of built-in variables. HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy, FTP_PROXY and NO_PROXY are built-in variables that needn't be explicitly defined in Dockerfile to use this feature. Signed-off-by: Madhav Puri --- api/client/build.go | 11 + api/server/image.go | 9 + builder/command/command.go | 2 + builder/dispatchers.go | 107 +++- builder/evaluator.go | 52 +- builder/job.go | 55 +- builder/parser/line_parsers.go | 69 ++- builder/parser/parser.go | 1 + builder/parser/parser_test.go | 37 ++ docs/reference/api/docker_remote_api.md | 2 +- docs/reference/api/docker_remote_api_v1.21.md | 5 + docs/reference/builder.md | 121 +++++ docs/reference/commandline/build.md | 20 + integration-cli/docker_cli_build_test.go | 504 ++++++++++++++++++ integration-cli/docker_utils.go | 21 +- man/Dockerfile.5.md | 120 +++++ man/docker-build.1.md | 19 + runconfig/parse.go | 8 +- 18 files changed, 1106 insertions(+), 57 deletions(-) diff --git a/api/client/build.go b/api/client/build.go index fe4b8ccf88..965fbe696c 100644 --- a/api/client/build.go +++ b/api/client/build.go @@ -35,6 +35,7 @@ import ( "github.com/docker/docker/pkg/units" "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" + "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" ) @@ -64,6 +65,8 @@ func (cli *DockerCli) CmdBuild(args ...string) error { flCPUSetCpus := cmd.String([]string{"-cpuset-cpus"}, "", "CPUs in which to allow execution (0-3, 0,1)") flCPUSetMems := cmd.String([]string{"-cpuset-mems"}, "", "MEMs in which to allow execution (0-3, 0,1)") flCgroupParent := cmd.String([]string{"-cgroup-parent"}, "", "Optional parent cgroup for the container") + flBuildArg := opts.NewListOpts(opts.ValidateEnv) + cmd.Var(&flBuildArg, []string{"-build-arg"}, "Set build-time variables") ulimits := make(map[string]*ulimit.Ulimit) flUlimits := opts.NewUlimitOpt(&ulimits) @@ -257,6 +260,14 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } v.Set("ulimits", string(ulimitsJSON)) + // collect all the build-time environment variables for the container + buildArgs := runconfig.ConvertKVStringsToMap(flBuildArg.GetAll()) + buildArgsJSON, err := json.Marshal(buildArgs) + if err != nil { + return err + } + v.Set("buildargs", string(buildArgsJSON)) + headers := http.Header(make(map[string][]string)) buf, err := json.Marshal(cli.configFile.AuthConfigs) if err != nil { diff --git a/api/server/image.go b/api/server/image.go index 384f770049..9a5718b07f 100644 --- a/api/server/image.go +++ b/api/server/image.go @@ -323,6 +323,15 @@ func (s *Server) postBuild(ctx context.Context, w http.ResponseWriter, r *http.R buildConfig.Ulimits = buildUlimits } + var buildArgs = map[string]string{} + buildArgsJSON := r.FormValue("buildargs") + if buildArgsJSON != "" { + if err := json.NewDecoder(strings.NewReader(buildArgsJSON)).Decode(&buildArgs); err != nil { + return err + } + } + buildConfig.BuildArgs = buildArgs + // Job cancellation. Note: not all job types support this. if closeNotifier, ok := w.(http.CloseNotifier); ok { finished := make(chan struct{}) diff --git a/builder/command/command.go b/builder/command/command.go index 968dba1f8d..9e1b799dcf 100644 --- a/builder/command/command.go +++ b/builder/command/command.go @@ -18,6 +18,7 @@ const ( Volume = "volume" User = "user" StopSignal = "stopsignal" + Arg = "arg" ) // Commands is list of all Dockerfile commands @@ -37,4 +38,5 @@ var Commands = map[string]struct{}{ Volume: {}, User: {}, StopSignal: {}, + Arg: {}, } diff --git a/builder/dispatchers.go b/builder/dispatchers.go index c164b190ec..f2db48dec9 100644 --- a/builder/dispatchers.go +++ b/builder/dispatchers.go @@ -327,15 +327,59 @@ func run(b *builder, args []string, attributes map[string]bool, original string) return err } + // stash the cmd cmd := b.Config.Cmd - // set Cmd manually, this is special case only for Dockerfiles - b.Config.Cmd = config.Cmd runconfig.Merge(b.Config, config) + // stash the config environment + env := b.Config.Env defer func(cmd *stringutils.StrSlice) { b.Config.Cmd = cmd }(cmd) + defer func(env []string) { b.Config.Env = env }(env) - logrus.Debugf("[BUILDER] Command to be executed: %v", b.Config.Cmd) + // derive the net build-time environment for this run. We let config + // environment override the build time environment. + // This means that we take the b.buildArgs list of env vars and remove + // any of those variables that are defined as part of the container. In other + // words, anything in b.Config.Env. What's left is the list of build-time env + // vars that we need to add to each RUN command - note the list could be empty. + // + // We don't persist the build time environment with container's config + // environment, but just sort and prepend it to the command string at time + // of commit. + // This helps with tracing back the image's actual environment at the time + // of RUN, without leaking it to the final image. It also aids cache + // lookup for same image built with same build time environment. + cmdBuildEnv := []string{} + configEnv := runconfig.ConvertKVStringsToMap(b.Config.Env) + for key, val := range b.buildArgs { + if !b.isBuildArgAllowed(key) { + // skip build-args that are not in allowed list, meaning they have + // not been defined by an "ARG" Dockerfile command yet. + // This is an error condition but only if there is no "ARG" in the entire + // Dockerfile, so we'll generate any necessary errors after we parsed + // the entire file (see 'leftoverArgs' processing in evaluator.go ) + continue + } + if _, ok := configEnv[key]; !ok { + cmdBuildEnv = append(cmdBuildEnv, fmt.Sprintf("%s=%s", key, val)) + } + } + // derive the command to use for probeCache() and to commit in this container. + // Note that we only do this if there are any build-time env vars. Also, we + // use the special argument "|#" at the start of the args array. This will + // avoid conflicts with any RUN command since commands can not + // start with | (vertical bar). The "#" (number of build envs) is there to + // help ensure proper cache matches. We don't want a RUN command + // that starts with "foo=abc" to be considered part of a build-time env var. + saveCmd := config.Cmd + if len(cmdBuildEnv) > 0 { + sort.Strings(cmdBuildEnv) + tmpEnv := append([]string{fmt.Sprintf("|%d", len(cmdBuildEnv))}, cmdBuildEnv...) + saveCmd = stringutils.NewStrSlice(append(tmpEnv, saveCmd.Slice()...)...) + } + + b.Config.Cmd = saveCmd hit, err := b.probeCache() if err != nil { return err @@ -344,6 +388,13 @@ func run(b *builder, args []string, attributes map[string]bool, original string) return nil } + // set Cmd manually, this is special case only for Dockerfiles + b.Config.Cmd = config.Cmd + // set build-time environment for 'run'. + b.Config.Env = append(b.Config.Env, cmdBuildEnv...) + + logrus.Debugf("[BUILDER] Command to be executed: %v", b.Config.Cmd) + c, err := b.create() if err != nil { return err @@ -358,6 +409,12 @@ func run(b *builder, args []string, attributes map[string]bool, original string) if err != nil { return err } + + // revert to original config environment and set the command string to + // have the build-time env vars in it (if any) so that future cache look-ups + // properly match it. + b.Config.Env = env + b.Config.Cmd = saveCmd if err := b.commit(c.ID, cmd, "run"); err != nil { return err } @@ -557,3 +614,47 @@ func stopSignal(b *builder, args []string, attributes map[string]bool, original b.Config.StopSignal = sig return b.commit("", b.Config.Cmd, fmt.Sprintf("STOPSIGNAL %v", args)) } + +// ARG name[=value] +// +// Adds the variable foo to the trusted list of variables that can be passed +// to builder using the --build-arg flag for expansion/subsitution or passing to 'run'. +// Dockerfile author may optionally set a default value of this variable. +func arg(b *builder, args []string, attributes map[string]bool, original string) error { + if len(args) != 1 { + return fmt.Errorf("ARG requires exactly one argument definition") + } + + var ( + name string + value string + hasDefault bool + ) + + arg := args[0] + // 'arg' can just be a name or name-value pair. Note that this is different + // from 'env' that handles the split of name and value at the parser level. + // The reason for doing it differently for 'arg' is that we support just + // defining an arg and not assign it a value (while 'env' always expects a + // name-value pair). If possible, it will be good to harmonize the two. + if strings.Contains(arg, "=") { + parts := strings.SplitN(arg, "=", 2) + name = parts[0] + value = parts[1] + hasDefault = true + } else { + name = arg + hasDefault = false + } + // add the arg to allowed list of build-time args from this step on. + b.allowedBuildArgs[name] = true + + // If there is a default value associated with this arg then add it to the + // b.buildArgs if one is not already passed to the builder. The args passed + // to builder override the defaut value of 'arg'. + if _, ok := b.buildArgs[name]; !ok && hasDefault { + b.buildArgs[name] = value + } + + return b.commit("", b.Config.Cmd, fmt.Sprintf("ARG %s", arg)) +} diff --git a/builder/evaluator.go b/builder/evaluator.go index 1cf7bb727e..5bab3373fd 100644 --- a/builder/evaluator.go +++ b/builder/evaluator.go @@ -54,6 +54,7 @@ var replaceEnvAllowed = map[string]struct{}{ command.Volume: {}, command.User: {}, command.StopSignal: {}, + command.Arg: {}, } var evaluateTable map[string]func(*builder, []string, map[string]bool, string) error @@ -75,6 +76,7 @@ func init() { command.Volume: volume, command.User: user, command.StopSignal: stopSignal, + command.Arg: arg, } } @@ -111,6 +113,9 @@ type builder struct { Config *runconfig.Config // runconfig for cmd, run, entrypoint etc. + buildArgs map[string]string // build-time args received in build context for expansion/substitution and commands in 'run'. + allowedBuildArgs map[string]bool // list of build-time args that are allowed for expansion/substitution and passing to commands in 'run'. + // both of these are controlled by the Remove and ForceRemove options in BuildOpts TmpContainers map[string]struct{} // a map of containers used for removes @@ -194,6 +199,18 @@ func (b *builder) Run(context io.Reader) (string, error) { } } + // check if there are any leftover build-args that were passed but not + // consumed during build. Return an error, if there are any. + leftoverArgs := []string{} + for arg := range b.buildArgs { + if !b.isBuildArgAllowed(arg) { + leftoverArgs = append(leftoverArgs, arg) + } + } + if len(leftoverArgs) > 0 { + return "", fmt.Errorf("One or more build-args %v were not consumed, failing build.", leftoverArgs) + } + if b.image == "" { return "", fmt.Errorf("No image was generated. Is your Dockerfile empty?") } @@ -268,6 +285,18 @@ func (b *builder) readDockerfile() error { return nil } +// determine if build arg is part of built-in args or user +// defined args in Dockerfile at any point in time. +func (b *builder) isBuildArgAllowed(arg string) bool { + if _, ok := BuiltinAllowedBuildArgs[arg]; ok { + return true + } + if _, ok := b.allowedBuildArgs[arg]; ok { + return true + } + return false +} + // This method is the entrypoint to all statement handling routines. // // Almost all nodes will have this structure: @@ -330,13 +359,34 @@ func (b *builder) dispatch(stepN int, ast *parser.Node) error { msgList := make([]string, n) var i int + // Append the build-time args to config-environment. + // This allows builder config to override the variables, making the behavior similar to + // a shell script i.e. `ENV foo bar` overrides value of `foo` passed in build + // context. But `ENV foo $foo` will use the value from build context if one + // isn't already been defined by a previous ENV primitive. + // Note, we get this behavior because we know that ProcessWord() will + // stop on the first occurrence of a variable name and not notice + // a subsequent one. So, putting the buildArgs list after the Config.Env + // list, in 'envs', is safe. + envs := b.Config.Env + for key, val := range b.buildArgs { + if !b.isBuildArgAllowed(key) { + // skip build-args that are not in allowed list, meaning they have + // not been defined by an "ARG" Dockerfile command yet. + // This is an error condition but only if there is no "ARG" in the entire + // Dockerfile, so we'll generate any necessary errors after we parsed + // the entire file (see 'leftoverArgs' processing in evaluator.go ) + continue + } + envs = append(envs, fmt.Sprintf("%s=%s", key, val)) + } for ast.Next != nil { ast = ast.Next var str string str = ast.Value if _, ok := replaceEnvAllowed[cmd]; ok { var err error - str, err = ProcessWord(ast.Value, b.Config.Env) + str, err = ProcessWord(ast.Value, envs) if err != nil { return err } diff --git a/builder/job.go b/builder/job.go index 43d7fd5470..de62666608 100644 --- a/builder/job.go +++ b/builder/job.go @@ -46,6 +46,18 @@ var validCommitCommands = map[string]bool{ "workdir": true, } +// BuiltinAllowedBuildArgs is list of built-in allowed build args +var BuiltinAllowedBuildArgs = map[string]bool{ + "HTTP_PROXY": true, + "http_proxy": true, + "HTTPS_PROXY": true, + "https_proxy": true, + "FTP_PROXY": true, + "ftp_proxy": true, + "NO_PROXY": true, + "no_proxy": true, +} + // Config contains all configs for a build job type Config struct { DockerfileName string @@ -66,6 +78,7 @@ type Config struct { CgroupParent string Ulimits []*ulimit.Ulimit AuthConfigs map[string]cliconfig.AuthConfig + BuildArgs map[string]string Stdout io.Writer Context io.ReadCloser @@ -191,26 +204,28 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { Writer: buildConfig.Stdout, StreamFormatter: sf, }, - Verbose: !buildConfig.SuppressOutput, - UtilizeCache: !buildConfig.NoCache, - Remove: buildConfig.Remove, - ForceRemove: buildConfig.ForceRemove, - Pull: buildConfig.Pull, - OutOld: buildConfig.Stdout, - StreamFormatter: sf, - AuthConfigs: buildConfig.AuthConfigs, - dockerfileName: buildConfig.DockerfileName, - cpuShares: buildConfig.CPUShares, - cpuPeriod: buildConfig.CPUPeriod, - cpuQuota: buildConfig.CPUQuota, - cpuSetCpus: buildConfig.CPUSetCpus, - cpuSetMems: buildConfig.CPUSetMems, - cgroupParent: buildConfig.CgroupParent, - memory: buildConfig.Memory, - memorySwap: buildConfig.MemorySwap, - ulimits: buildConfig.Ulimits, - cancelled: buildConfig.WaitCancelled(), - id: stringid.GenerateRandomID(), + Verbose: !buildConfig.SuppressOutput, + UtilizeCache: !buildConfig.NoCache, + Remove: buildConfig.Remove, + ForceRemove: buildConfig.ForceRemove, + Pull: buildConfig.Pull, + OutOld: buildConfig.Stdout, + StreamFormatter: sf, + AuthConfigs: buildConfig.AuthConfigs, + dockerfileName: buildConfig.DockerfileName, + cpuShares: buildConfig.CPUShares, + cpuPeriod: buildConfig.CPUPeriod, + cpuQuota: buildConfig.CPUQuota, + cpuSetCpus: buildConfig.CPUSetCpus, + cpuSetMems: buildConfig.CPUSetMems, + cgroupParent: buildConfig.CgroupParent, + memory: buildConfig.Memory, + memorySwap: buildConfig.MemorySwap, + ulimits: buildConfig.Ulimits, + cancelled: buildConfig.WaitCancelled(), + id: stringid.GenerateRandomID(), + buildArgs: buildConfig.BuildArgs, + allowedBuildArgs: make(map[string]bool), } defer func() { diff --git a/builder/parser/line_parsers.go b/builder/parser/line_parsers.go index b28693666c..b8792708d5 100644 --- a/builder/parser/line_parsers.go +++ b/builder/parser/line_parsers.go @@ -42,15 +42,10 @@ func parseSubCommand(rest string) (*Node, map[string]bool, error) { return &Node{Children: []*Node{child}}, nil, nil } -// parse environment like statements. Note that this does *not* handle -// variable interpolation, which will be handled in the evaluator. -func parseNameVal(rest string, key string) (*Node, map[string]bool, error) { - // This is kind of tricky because we need to support the old - // variant: KEY name value - // as well as the new one: KEY name=value ... - // The trigger to know which one is being used will be whether we hit - // a space or = first. space ==> old, "=" ==> new - +// helper to parse words (i.e space delimited or quoted strings) in a statement. +// The quotes are preserved as part of this function and they are stripped later +// as part of processWords(). +func parseWords(rest string) []string { const ( inSpaces = iota // looking for start of a word inWord @@ -89,15 +84,6 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) { phase = inSpaces if blankOK || len(word) > 0 { words = append(words, word) - - // Look for = and if not there assume - // we're doing the old stuff and - // just read the rest of the line - if !strings.Contains(word, "=") { - word = strings.TrimSpace(rest[pos:]) - words = append(words, word) - break - } } word = "" blankOK = false @@ -141,13 +127,26 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) { } } + return words +} + +// parse environment like statements. Note that this does *not* handle +// variable interpolation, which will be handled in the evaluator. +func parseNameVal(rest string, key string) (*Node, map[string]bool, error) { + // This is kind of tricky because we need to support the old + // variant: KEY name value + // as well as the new one: KEY name=value ... + // The trigger to know which one is being used will be whether we hit + // a space or = first. space ==> old, "=" ==> new + + words := parseWords(rest) if len(words) == 0 { return nil, nil, nil } - // Old format (KEY name value) var rootnode *Node + // Old format (KEY name value) if !strings.Contains(words[0], "=") { node := &Node{} rootnode = node @@ -195,6 +194,38 @@ func parseLabel(rest string) (*Node, map[string]bool, error) { return parseNameVal(rest, "LABEL") } +// parses a statement containing one or more keyword definition(s) and/or +// value assignments, like `name1 name2= name3="" name4=value`. +// Note that this is a stricter format than the old format of assignment, +// allowed by parseNameVal(), in a way that this only allows assignment of the +// form `keyword=[]` like `name2=`, `name3=""`, and `name4=value` above. +// In addition, a keyword definition alone is of the form `keyword` like `name1` +// above. And the assignments `name2=` and `name3=""` are equivalent and +// assign an empty value to the respective keywords. +func parseNameOrNameVal(rest string) (*Node, map[string]bool, error) { + words := parseWords(rest) + if len(words) == 0 { + return nil, nil, nil + } + + var ( + rootnode *Node + prevNode *Node + ) + for i, word := range words { + node := &Node{} + node.Value = word + if i == 0 { + rootnode = node + } else { + prevNode.Next = node + } + prevNode = node + } + + return rootnode, nil, nil +} + // parses a whitespace-delimited set of arguments. The result is effectively a // linked list of string arguments. func parseStringsWhitespaceDelimited(rest string) (*Node, map[string]bool, error) { diff --git a/builder/parser/parser.go b/builder/parser/parser.go index 37548401d3..6490fea1c7 100644 --- a/builder/parser/parser.go +++ b/builder/parser/parser.go @@ -62,6 +62,7 @@ func init() { command.Expose: parseStringsWhitespaceDelimited, command.Volume: parseMaybeJSONToList, command.StopSignal: parseString, + command.Arg: parseNameOrNameVal, } } diff --git a/builder/parser/parser_test.go b/builder/parser/parser_test.go index 6b55a611ec..9734676469 100644 --- a/builder/parser/parser_test.go +++ b/builder/parser/parser_test.go @@ -73,3 +73,40 @@ func TestTestData(t *testing.T) { } } } + +func TestParseWords(t *testing.T) { + tests := []map[string][]string{ + { + "input": {"foo"}, + "expect": {"foo"}, + }, + { + "input": {"foo bar"}, + "expect": {"foo", "bar"}, + }, + { + "input": {"foo=bar"}, + "expect": {"foo=bar"}, + }, + { + "input": {"foo bar 'abc xyz'"}, + "expect": {"foo", "bar", "'abc xyz'"}, + }, + { + "input": {`foo bar "abc xyz"`}, + "expect": {"foo", "bar", `"abc xyz"`}, + }, + } + + for _, test := range tests { + words := parseWords(test["input"][0]) + if len(words) != len(test["expect"]) { + t.Fatalf("length check failed. input: %v, expect: %v, output: %v", test["input"][0], test["expect"], words) + } + for i, word := range words { + if word != test["expect"][i] { + t.Fatalf("word check failed for word: %q. input: %v, expect: %v, output: %v", word, test["input"][0], test["expect"], words) + } + } + } +} diff --git a/docs/reference/api/docker_remote_api.md b/docs/reference/api/docker_remote_api.md index 716728bd95..2b6b4c9503 100644 --- a/docs/reference/api/docker_remote_api.md +++ b/docs/reference/api/docker_remote_api.md @@ -86,7 +86,7 @@ This section lists each version from latest to oldest. Each listing includes a * `GET /containers/(id)/stats` will return networking information respectively for each interface. * The `hostConfig` option now accepts the field `DnsOptions`, which specifies a list of DNS options to be used in the container. - +* `POST /build` now optionally takes a serialized map of build-time variables. ### v1.20 API changes diff --git a/docs/reference/api/docker_remote_api_v1.21.md b/docs/reference/api/docker_remote_api_v1.21.md index 885b061cba..1fa18f38f4 100644 --- a/docs/reference/api/docker_remote_api_v1.21.md +++ b/docs/reference/api/docker_remote_api_v1.21.md @@ -1367,6 +1367,11 @@ Query Parameters: - **memswap** - Total memory (memory + swap), `-1` to disable swap. - **cpushares** - CPU shares (relative weight). - **cpusetcpus** - CPUs in which to allow execution (e.g., `0-3`, `0,1`). +- **buildargs** – JSON map of string pairs for build-time variables. Users can + set these values at build-time and they are used as environment context + for the command(s) run as part of the Dockerfile's `RUN` instruction or + for variable expansion in other Dockerfile instructions. Read more about + the `ARG` instruction [here](/reference/builder/#arg) Request Headers: diff --git a/docs/reference/builder.md b/docs/reference/builder.md index 537cc98ece..848d69717c 100644 --- a/docs/reference/builder.md +++ b/docs/reference/builder.md @@ -966,6 +966,127 @@ For example: The output of the final `pwd` command in this `Dockerfile` would be `/path/$DIRNAME` +## ARG + + ARG [=] + +The `ARG` instruction defines a variable that users can pass at build-time to +the builder with the `docker build` command using the `--build-arg +=` flag. If a user specifies a build argument that was not +defined in the Dockerfile, the build outputs an error. + +``` +One or more build-args were not consumed, failing build. +``` + +The Dockerfile author can define a single variable by specifying `ARG` once or many +variables by specifying `ARG` more than once. For example, a valid Dockerfile: + +``` +FROM busybox +ARG user1 +ARG buildno +... +``` + +A Dockerfile author may optionally specify a default value for an `ARG` instruction: + +``` +FROM busybox +ARG user1=someuser +ARG buildno=1 +... +``` + +If an `ARG` value has a default and if there is no value passed at build-time, the +builder uses the default. + +An `ARG` variable definition comes into effect from the line on which it is +defined in the `Dockerfile` not from the argument's use on the command-line or +elsewhere. For example, consider this Dockerfile: + +``` +1 FROM busybox +2 USER ${user:-some_user} +3 ARG user +4 USER $user +... +``` +A user builds this file by calling: + +``` +$ docker build --build-arg user=what_user Dockerfile +``` + +The `USER` at line 2 evaluates to `some_user` as the `user` variable is defined on the +subsequent line 3. The `USER` at line 4 evaluates to `what_user` as `user` is +defined and the `what_user` value was passed on the command line. Prior to its definition by an +`ARG` instruction, any use of a variable results in an empty string. + +> **Note:** It is not recommended to use build-time variables for +> passing secrets like github keys, user credentials etc. + +You can use an `ARG` or an `ENV` instruction to specify variables that are +available to the `RUN` instruction. Environment variables defined using the +`ENV` instruction always override an `ARG` instruction of the same name. Consider +this Dockerfile with an `ENV` and `ARG` instruction. + +``` +1 FROM ubuntu +2 ARG CONT_IMG_VER +3 ENV CONT_IMG_VER v1.0.0 +4 RUN echo $CONT_IMG_VER +``` +Then, assume this image is built with this command: + +``` +$ docker build --build-arg CONT_IMG_VER=v2.0.1 Dockerfile +``` + +In this case, the `RUN` instruction uses `v1.0.0` instead of the `ARG` setting +passed by the user:`v2.0.1` This behavior is similar to a shell +script where a locally scoped variable overrides the variables passed as +arguments or inherited from environment, from its point of definition. + +Using the example above but a different `ENV` specification you can create more +useful interactions between `ARG` and `ENV` instructions: + +``` +1 FROM ubuntu +2 ARG CONT_IMG_VER +3 ENV CONT_IMG_VER ${CONT_IMG_VER:-v1.0.0} +4 RUN echo $CONT_IMG_VER +``` + +The command line passes the `--build-arg` and sets the `v2.0.1` value. And the `ARG +CONT_IMG_VER` is defined on line 2 of the Dockerfile. On line 3, the `ENV` +instruction of the same name resolves to `v2.0.1` as the build-time variable +was passed from the command line and expanded here. + +The variable expansion technique in this example allows you to pass arguments +from the command line and persist them in the final image by leveraging the `ENV` +instruction. Variable expansion is only supported for the `Dockerfile` instructions +described [here](#environment-replacement). + +Unlike an `ARG` instruction, `ENV` values are always persisted in the built image. If +`docker build` were run without setting the `--build-arg` flag, then +`CONT_IMG_VER` is still persisted in the image but its value would be `v1.0.0`. + +Docker has a set of predefined `ARG` variables that you can use without a +corresponding `ARG` instruction in the Dockerfile. + +* `HTTP_PROXY` +* `http_proxy` +* `HTTPS_PROXY` +* `https_proxy` +* `FTP_PROXY` +* `ftp_proxy` +* `NO_PROXY` +* `no_proxy` + +To use these, simply pass them on the command line using the `--build-arg +=` flag. + ## ONBUILD ONBUILD [INSTRUCTION] diff --git a/docs/reference/commandline/build.md b/docs/reference/commandline/build.md index 633d1132de..8cb6b13107 100644 --- a/docs/reference/commandline/build.md +++ b/docs/reference/commandline/build.md @@ -17,6 +17,7 @@ weight=1 -f, --file="" Name of the Dockerfile (Default is 'PATH/Dockerfile') --force-rm=false Always remove intermediate containers + --build-arg=[] Set build-time variables --no-cache=false Do not use cache when building the image --pull=false Always attempt to pull a newer version of the image -q, --quiet=false Suppress the verbose output generated by the containers @@ -251,3 +252,22 @@ flag](/reference/run/#specifying-custom-cgroups). Using the `--ulimit` option with `docker build` will cause each build step's container to be started using those [`--ulimit` flag values](/reference/run/#setting-ulimits-in-a-container). + +You can use `ENV` instructions in a Dockerfile to define variable +values. These values persist in the built image. However, often +persistence is not what you want. Users want to specify variables differently +depending on which host they build an image on. + +A good example is `http_proxy` or source versions for pulling intermediate +files. The `ARG` instruction lets Dockerfile authors define values that users +can set at build-time using the `---build-arg` flag: + + $ docker build --build-arg HTTP_PROXY=http://10.20.30.2:1234 . + +This flag allows you to pass the build-time variables that are +accessed like regular environment variables in the `RUN` instruction of the +Dockerfile. Also, these values don't persist in the intermediate or final images +like `ENV` values do. + +For detailed information on using `ARG` and `ENV` instructions, see the +[Dockerfile reference](/reference/builder). diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 9340d6005c..6e1933a088 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -5676,3 +5676,507 @@ func (s *DockerSuite) TestBuildStopSignal(c *check.C) { c.Fatalf("Signal %s, expected SIGKILL", res) } } + +func (s *DockerSuite) TestBuildBuildTimeArg(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + RUN echo $%s + CMD echo $%s`, envKey, envKey, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || !strings.Contains(out, envVal) { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envVal) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); out != "\n" { + c.Fatalf("run produced invalid output: %q, expected empty string", out) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgHistory(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + envDef := "bar1" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s=%s`, envKey, envDef) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || !strings.Contains(out, envVal) { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envVal) + } + + out, _ := dockerCmd(c, "history", "--no-trunc", imgName) + outputTabs := strings.Split(out, "\n")[1] + if !strings.Contains(outputTabs, envDef) { + c.Fatalf("failed to find arg default in image history output: %q expected: %q", outputTabs, envDef) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgCacheHit(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + RUN echo $%s`, envKey, envKey) + + origImgID := "" + var err error + if origImgID, err = buildImage(imgName, dockerfile, true, args...); err != nil { + c.Fatal(err) + } + + imgNameCache := "bldargtestcachehit" + if newImgID, err := buildImage(imgNameCache, dockerfile, true, args...); err != nil || newImgID != origImgID { + if err != nil { + c.Fatal(err) + } + c.Fatalf("build didn't use cache! expected image id: %q built image id: %q", origImgID, newImgID) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgCacheMissExtraArg(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + extraEnvKey := "foo1" + extraEnvVal := "bar1" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + ARG %s + RUN echo $%s`, envKey, extraEnvKey, envKey) + + origImgID := "" + var err error + if origImgID, err = buildImage(imgName, dockerfile, true, args...); err != nil { + c.Fatal(err) + } + + imgNameCache := "bldargtestcachemiss" + args = append(args, "--build-arg", fmt.Sprintf("%s=%s", extraEnvKey, extraEnvVal)) + if newImgID, err := buildImage(imgNameCache, dockerfile, true, args...); err != nil || newImgID == origImgID { + if err != nil { + c.Fatal(err) + } + c.Fatalf("build used cache, expected a miss!") + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgCacheMissSameArgDiffVal(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + newEnvVal := "bar1" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + RUN echo $%s`, envKey, envKey) + + origImgID := "" + var err error + if origImgID, err = buildImage(imgName, dockerfile, true, args...); err != nil { + c.Fatal(err) + } + + imgNameCache := "bldargtestcachemiss" + args = []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, newEnvVal), + } + if newImgID, err := buildImage(imgNameCache, dockerfile, true, args...); err != nil || newImgID == origImgID { + if err != nil { + c.Fatal(err) + } + c.Fatalf("build used cache, expected a miss!") + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgOverrideArgDefinedBeforeEnv(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + envValOveride := "barOverride" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + ENV %s %s + RUN echo $%s + CMD echo $%s + `, envKey, envKey, envValOveride, envKey, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || strings.Count(out, envValOveride) != 2 { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envValOveride) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgOverrideEnvDefinedBeforeArg(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + envValOveride := "barOverride" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + ENV %s %s + ARG %s + RUN echo $%s + CMD echo $%s + `, envKey, envValOveride, envKey, envKey, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || strings.Count(out, envValOveride) != 2 { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envValOveride) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) { + imgName := "bldvarstest" + + wdVar := "WDIR" + wdVal := "/tmp/" + addVar := "AFILE" + addVal := "addFile" + copyVar := "CFILE" + copyVal := "copyFile" + envVar := "foo" + envVal := "bar" + exposeVar := "EPORT" + exposeVal := "9999" + userVar := "USER" + userVal := "testUser" + volVar := "VOL" + volVal := "/testVol/" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", wdVar, wdVal), + "--build-arg", fmt.Sprintf("%s=%s", addVar, addVal), + "--build-arg", fmt.Sprintf("%s=%s", copyVar, copyVal), + "--build-arg", fmt.Sprintf("%s=%s", envVar, envVal), + "--build-arg", fmt.Sprintf("%s=%s", exposeVar, exposeVal), + "--build-arg", fmt.Sprintf("%s=%s", userVar, userVal), + "--build-arg", fmt.Sprintf("%s=%s", volVar, volVal), + } + ctx, err := fakeContext(fmt.Sprintf(`FROM busybox + ARG %s + WORKDIR ${%s} + ARG %s + ADD ${%s} testDir/ + ARG %s + COPY $%s testDir/ + ARG %s + ENV %s=${%s} + ARG %s + EXPOSE $%s + ARG %s + USER $%s + ARG %s + VOLUME ${%s}`, + wdVar, wdVar, addVar, addVar, copyVar, copyVar, envVar, envVar, + envVar, exposeVar, exposeVar, userVar, userVar, volVar, volVar), + map[string]string{ + addVal: "some stuff", + copyVal: "some stuff", + }) + if err != nil { + c.Fatal(err) + } + defer ctx.Close() + + if _, err := buildImageFromContext(imgName, ctx, true, args...); err != nil { + c.Fatal(err) + } + + var resMap map[string]interface{} + var resArr []string + res := "" + res, err = inspectField(imgName, "Config.WorkingDir") + if err != nil { + c.Fatal(err) + } + if res != wdVal { + c.Fatalf("Config.WorkingDir value mismatch. Expected: %s, got: %s", wdVal, res) + } + + err = inspectFieldAndMarshall(imgName, "Config.Env", &resArr) + if err != nil { + c.Fatal(err) + } + + found := false + for _, v := range resArr { + if fmt.Sprintf("%s=%s", envVar, envVal) == v { + found = true + break + } + } + if !found { + c.Fatalf("Config.Env value mismatch. Expected to exist: %s=%s, got: %v", + envVar, envVal, resArr) + } + + err = inspectFieldAndMarshall(imgName, "Config.ExposedPorts", &resMap) + if err != nil { + c.Fatal(err) + } + if _, ok := resMap[fmt.Sprintf("%s/tcp", exposeVal)]; !ok { + c.Fatalf("Config.ExposedPorts value mismatch. Expected exposed port: %s/tcp, got: %v", exposeVal, resMap) + } + + res, err = inspectField(imgName, "Config.User") + if err != nil { + c.Fatal(err) + } + if res != userVal { + c.Fatalf("Config.User value mismatch. Expected: %s, got: %s", userVal, res) + } + + err = inspectFieldAndMarshall(imgName, "Config.Volumes", &resMap) + if err != nil { + c.Fatal(err) + } + if _, ok := resMap[volVal]; !ok { + c.Fatalf("Config.Volumes value mismatch. Expected volume: %s, got: %v", volVal, resMap) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgExpansionOverride(c *check.C) { + imgName := "bldvarstest" + envKey := "foo" + envVal := "bar" + envKey1 := "foo1" + envValOveride := "barOverride" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + ENV %s %s + ENV %s ${%s} + RUN echo $%s + CMD echo $%s`, envKey, envKey, envValOveride, envKey1, envKey, envKey1, envKey1) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || strings.Count(out, envValOveride) != 2 { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envValOveride) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgUntrustedDefinedAfterUse(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + RUN echo $%s + ARG %s + CMD echo $%s`, envKey, envKey, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || strings.Contains(out, envVal) { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("able to access environment variable in output: %q expected to be missing", out) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); out != "\n" { + c.Fatalf("run produced invalid output: %q, expected empty string", out) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgBuiltinArg(c *check.C) { + imgName := "bldargtest" + envKey := "HTTP_PROXY" + envVal := "bar" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + RUN echo $%s + CMD echo $%s`, envKey, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || !strings.Contains(out, envVal) { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envVal) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); out != "\n" { + c.Fatalf("run produced invalid output: %q, expected empty string", out) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgDefaultOverride(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + envValOveride := "barOverride" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envValOveride), + } + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s=%s + ENV %s $%s + RUN echo $%s + CMD echo $%s`, envKey, envVal, envKey, envKey, envKey, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || strings.Count(out, envValOveride) != 1 { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("failed to access environment variable in output: %q expected: %q", out, envValOveride) + } + + containerName := "bldargCont" + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgMultiArgsSameLine(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envKey1 := "foo1" + args := []string{} + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s %s`, envKey, envKey1) + + errStr := "ARG requires exactly one argument definition" + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err == nil { + c.Fatalf("build succeeded, expected to fail. Output: %v", out) + } else if !strings.Contains(out, errStr) { + c.Fatalf("Unexpected error. output: %q, expected error: %q", out, errStr) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgUnconsumedArg(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envVal := "bar" + args := []string{ + "--build-arg", fmt.Sprintf("%s=%s", envKey, envVal), + } + dockerfile := fmt.Sprintf(`FROM busybox + RUN echo $%s + CMD echo $%s`, envKey, envKey) + + errStr := "One or more build-args" + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err == nil { + c.Fatalf("build succeeded, expected to fail. Output: %v", out) + } else if !strings.Contains(out, errStr) { + c.Fatalf("Unexpected error. output: %q, expected error: %q", out, errStr) + } + +} + +func (s *DockerSuite) TestBuildBuildTimeArgQuotedValVariants(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envKey1 := "foo1" + envKey2 := "foo2" + envKey3 := "foo3" + args := []string{} + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s="" + ARG %s='' + ARG %s="''" + ARG %s='""' + RUN [ "$%s" != "$%s" ] + RUN [ "$%s" != "$%s" ] + RUN [ "$%s" != "$%s" ] + RUN [ "$%s" != "$%s" ] + RUN [ "$%s" != "$%s" ]`, envKey, envKey1, envKey2, envKey3, + envKey, envKey2, envKey, envKey3, envKey1, envKey2, envKey1, envKey3, + envKey2, envKey3) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgEmptyValVariants(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + envKey1 := "foo1" + envKey2 := "foo2" + args := []string{} + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s= + ARG %s="" + ARG %s='' + RUN [ "$%s" == "$%s" ] + RUN [ "$%s" == "$%s" ] + RUN [ "$%s" == "$%s" ]`, envKey, envKey1, envKey2, envKey, envKey1, envKey1, envKey2, envKey, envKey2) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } +} + +func (s *DockerSuite) TestBuildBuildTimeArgDefintionWithNoEnvInjection(c *check.C) { + imgName := "bldargtest" + envKey := "foo" + args := []string{} + dockerfile := fmt.Sprintf(`FROM busybox + ARG %s + RUN env`, envKey) + + if _, out, err := buildImageWithOut(imgName, dockerfile, true, args...); err != nil || strings.Count(out, envKey) != 1 { + if err != nil { + c.Fatalf("build failed to complete: %q %q", out, err) + } + c.Fatalf("unexpected number of occurrences of the arg in output: %q expected: 1", out) + } +} diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index 6b4700d426..b569976294 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -1025,11 +1025,12 @@ func getContainerState(c *check.C, id string) (int, bool, error) { return exitStatus, running, nil } -func buildImageCmd(name, dockerfile string, useCache bool) *exec.Cmd { +func buildImageCmd(name, dockerfile string, useCache bool, buildFlags ...string) *exec.Cmd { args := []string{"-D", "build", "-t", name} if !useCache { args = append(args, "--no-cache") } + args = append(args, buildFlags...) args = append(args, "-") buildCmd := exec.Command(dockerBinary, args...) buildCmd.Stdin = strings.NewReader(dockerfile) @@ -1037,8 +1038,8 @@ func buildImageCmd(name, dockerfile string, useCache bool) *exec.Cmd { } -func buildImageWithOut(name, dockerfile string, useCache bool) (string, string, error) { - buildCmd := buildImageCmd(name, dockerfile, useCache) +func buildImageWithOut(name, dockerfile string, useCache bool, buildFlags ...string) (string, string, error) { + buildCmd := buildImageCmd(name, dockerfile, useCache, buildFlags...) out, exitCode, err := runCommandWithOutput(buildCmd) if err != nil || exitCode != 0 { return "", out, fmt.Errorf("failed to build the image: %s", out) @@ -1050,8 +1051,8 @@ func buildImageWithOut(name, dockerfile string, useCache bool) (string, string, return id, out, nil } -func buildImageWithStdoutStderr(name, dockerfile string, useCache bool) (string, string, string, error) { - buildCmd := buildImageCmd(name, dockerfile, useCache) +func buildImageWithStdoutStderr(name, dockerfile string, useCache bool, buildFlags ...string) (string, string, string, error) { + buildCmd := buildImageCmd(name, dockerfile, useCache, buildFlags...) stdout, stderr, exitCode, err := runCommandWithStdoutStderr(buildCmd) if err != nil || exitCode != 0 { return "", stdout, stderr, fmt.Errorf("failed to build the image: %s", stdout) @@ -1063,16 +1064,17 @@ func buildImageWithStdoutStderr(name, dockerfile string, useCache bool) (string, return id, stdout, stderr, nil } -func buildImage(name, dockerfile string, useCache bool) (string, error) { - id, _, err := buildImageWithOut(name, dockerfile, useCache) +func buildImage(name, dockerfile string, useCache bool, buildFlags ...string) (string, error) { + id, _, err := buildImageWithOut(name, dockerfile, useCache, buildFlags...) return id, err } -func buildImageFromContext(name string, ctx *FakeContext, useCache bool) (string, error) { +func buildImageFromContext(name string, ctx *FakeContext, useCache bool, buildFlags ...string) (string, error) { args := []string{"build", "-t", name} if !useCache { args = append(args, "--no-cache") } + args = append(args, buildFlags...) args = append(args, ".") buildCmd := exec.Command(dockerBinary, args...) buildCmd.Dir = ctx.Dir @@ -1083,11 +1085,12 @@ func buildImageFromContext(name string, ctx *FakeContext, useCache bool) (string return getIDByName(name) } -func buildImageFromPath(name, path string, useCache bool) (string, error) { +func buildImageFromPath(name, path string, useCache bool, buildFlags ...string) (string, error) { args := []string{"build", "-t", name} if !useCache { args = append(args, "--no-cache") } + args = append(args, buildFlags...) args = append(args, path) buildCmd := exec.Command(dockerBinary, args...) out, exitCode, err := runCommandWithOutput(buildCmd) diff --git a/man/Dockerfile.5.md b/man/Dockerfile.5.md index 0b188ac330..b29745e2e7 100644 --- a/man/Dockerfile.5.md +++ b/man/Dockerfile.5.md @@ -317,6 +317,126 @@ A Dockerfile is similar to a Makefile. In the above example, the output of the **pwd** command is **a/b/c**. +**ARG** + -- ARG [=] + + The `ARG` instruction defines a variable that users can pass at build-time to + the builder with the `docker build` command using the `--build-arg + =` flag. If a user specifies a build argument that was not + defined in the Dockerfile, the build outputs an error. + + ``` + One or more build-args were not consumed, failing build. + ``` + + The Dockerfile author can define a single variable by specifying `ARG` once or many + variables by specifying `ARG` more than once. For example, a valid Dockerfile: + + ``` + FROM busybox + ARG user1 + ARG buildno + ... + ``` + + A Dockerfile author may optionally specify a default value for an `ARG` instruction: + + ``` + FROM busybox + ARG user1=someuser + ARG buildno=1 + ... + ``` + + If an `ARG` value has a default and if there is no value passed at build-time, the + builder uses the default. + + An `ARG` variable definition comes into effect from the line on which it is + defined in the `Dockerfile` not from the argument's use on the command-line or + elsewhere. For example, consider this Dockerfile: + + ``` + 1 FROM busybox + 2 USER ${user:-some_user} + 3 ARG user + 4 USER $user + ... + ``` + A user builds this file by calling: + + ``` + $ docker build --build-arg user=what_user Dockerfile + ``` + + The `USER` at line 2 evaluates to `some_user` as the `user` variable is defined on the + subsequent line 3. The `USER` at line 4 evaluates to `what_user` as `user` is + defined and the `what_user` value was passed on the command line. Prior to its definition by an + `ARG` instruction, any use of a variable results in an empty string. + + > **Note:** It is not recommended to use build-time variables for + > passing secrets like github keys, user credentials etc. + + You can use an `ARG` or an `ENV` instruction to specify variables that are + available to the `RUN` instruction. Environment variables defined using the + `ENV` instruction always override an `ARG` instruction of the same name. Consider + this Dockerfile with an `ENV` and `ARG` instruction. + + ``` + 1 FROM ubuntu + 2 ARG CONT_IMG_VER + 3 ENV CONT_IMG_VER v1.0.0 + 4 RUN echo $CONT_IMG_VER + ``` + Then, assume this image is built with this command: + + ``` + $ docker build --build-arg CONT_IMG_VER=v2.0.1 Dockerfile + ``` + + In this case, the `RUN` instruction uses `v1.0.0` instead of the `ARG` setting + passed by the user:`v2.0.1` This behavior is similar to a shell + script where a locally scoped variable overrides the variables passed as + arguments or inherited from environment, from its point of definition. + + Using the example above but a different `ENV` specification you can create more + useful interactions between `ARG` and `ENV` instructions: + + ``` + 1 FROM ubuntu + 2 ARG CONT_IMG_VER + 3 ENV CONT_IMG_VER ${CONT_IMG_VER:-v1.0.0} + 4 RUN echo $CONT_IMG_VER + ``` + + The command line passes the `--build-arg` and sets the `v2.0.1` value. And the `ARG + CONT_IMG_VER` is defined on line 2 of the Dockerfile. On line 3, the `ENV` + instruction of the same name resolves to `v2.0.1` as the build-time variable + was passed from the command line and expanded here. + + The variable expansion technique in this example allows you to pass arguments + from the command line and persist them in the final image by leveraging the `ENV` + instruction. Variable expansion is only supported for the `Dockerfile` instructions + described [here](#environment-replacement). + + Unlike an `ARG` instruction, `ENV` values are always persisted in the built image. If + `docker build` were run without setting the `--build-arg` flag, then + `CONT_IMG_VER` is still persisted in the image but its value would be `v1.0.0`. + + Docker has a set of predefined `ARG` variables that you can use without a + corresponding `ARG` instruction in the Dockerfile. + + * `HTTP_PROXY` + * `http_proxy` + * `HTTPS_PROXY` + * `https_proxy` + * `FTP_PROXY` + * `ftp_proxy` + * `NO_PROXY` + * `no_proxy` + + To use these, simply pass them on the command line using the `--build-arg + =` flag. + **ONBUILD** -- `ONBUILD [INSTRUCTION]` The **ONBUILD** instruction adds a trigger instruction to an image. The diff --git a/man/docker-build.1.md b/man/docker-build.1.md index a8714b775a..f725f83aa8 100644 --- a/man/docker-build.1.md +++ b/man/docker-build.1.md @@ -8,6 +8,7 @@ docker-build - Build a new image from the source code at PATH **docker build** [**--help**] [**-f**|**--file**[=*PATH/Dockerfile*]] +[**--build-arg**[=*[]*]] [**--force-rm**[=*false*]] [**--no-cache**[=*false*]] [**--pull**[=*false*]] @@ -51,6 +52,24 @@ cloned locally and then sent as the context. the remote context. In all cases, the file must be within the build context. The default is *Dockerfile*. +**--build-arg**=*variable* + Set value for build-time variable. This option allows you to specify +values of the variables that are available for expansion/substitution in the +Dockerfile instructions like ADD, COPY etc, without an explicit prior definition by +the ENV instruction. The build-time variables are also passed as environment +context for the command(s) that will be executed as part of RUN instruction +of Dockerfile, if there is no explicit prior definition by the ENV instruction. +Normally, these variables are not persisted in the resulting Docker image. This gives +the flexibility to build an image by passing host specific environment variables (like +http_proxy) that will be used on the RUN commands without affecting portability +of the generated image. +However, as with any variable, they can be persisted in the final image if they are used in an +ENV instruction (e.g. ENV myName=$myName will save myName in the image). + +Only the build-time variables that are defined using the ARG instruction of Dockerfile +are allowed to be expanded or passed as environment to the RUN command. Read more about +ARG instruction in Dockerfile reference. + **--force-rm**=*true*|*false* Always remove intermediate containers, even after unsuccessful builds. The default is *false*. diff --git a/runconfig/parse.go b/runconfig/parse.go index 3c92aa9dd7..e3c17bca70 100644 --- a/runconfig/parse.go +++ b/runconfig/parse.go @@ -324,7 +324,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe MacAddress: *flMacAddress, Entrypoint: entrypoint, WorkingDir: *flWorkingDir, - Labels: convertKVStringsToMap(labels), + Labels: ConvertKVStringsToMap(labels), StopSignal: *flStopSignal, } @@ -394,8 +394,8 @@ func readKVStrings(files []string, override []string) ([]string, error) { return envVariables, nil } -// converts ["key=value"] to {"key":"value"} -func convertKVStringsToMap(values []string) map[string]string { +// ConvertKVStringsToMap converts ["key=value"] to {"key":"value"} +func ConvertKVStringsToMap(values []string) map[string]string { result := make(map[string]string, len(values)) for _, value := range values { kv := strings.SplitN(value, "=", 2) @@ -410,7 +410,7 @@ func convertKVStringsToMap(values []string) map[string]string { } func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) { - loggingOptsMap := convertKVStringsToMap(loggingOpts) + loggingOptsMap := ConvertKVStringsToMap(loggingOpts) if loggingDriver == "none" && len(loggingOpts) > 0 { return map[string]string{}, fmt.Errorf("Invalid logging opts for driver %s", loggingDriver) }