Merge pull request #9123 from rhatdan/commit-change

Patch to commit-change patch to add docker import support

api/client/commands.go

@@ -1156,6 +1156,8 @@ func (cli *DockerCli) CmdKill(args ...string) error {
 
 func (cli *DockerCli) CmdImport(args ...string) error {
 	cmd := cli.Subcmd("import", "URL|- [REPOSITORY[:TAG]]", "Create an empty filesystem image and import the contents of the\ntarball (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) into it, then\noptionally tag it.", true)
+	flChanges := opts.NewListOpts(nil)
+	cmd.Var(&flChanges, []string{"c", "-change"}, "Apply Dockerfile instruction to the created image.")
 	cmd.Require(flag.Min, 1)
 
 	utils.ParseFlags(cmd, args, true)
@@ -1168,7 +1170,9 @@ func (cli *DockerCli) CmdImport(args ...string) error {
 
 	v.Set("fromSrc", src)
 	v.Set("repo", repository)
-
+	for _, change := range flChanges.GetAll() {
+		v.Add("changes", change)
+	}
 	if cmd.NArg() == 3 {
 		fmt.Fprintf(cli.err, "[DEPRECATED] The format 'URL|- [REPOSITORY [TAG]]' has been deprecated. Please use URL|- [REPOSITORY[:TAG]]\n")
 		v.Set("tag", cmd.Arg(2))
@@ -1702,6 +1706,8 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 	flPause := cmd.Bool([]string{"p", "-pause"}, true, "Pause container during commit")
 	flComment := cmd.String([]string{"m", "-message"}, "", "Commit message")
 	flAuthor := cmd.String([]string{"a", "#author", "-author"}, "", "Author (e.g., \"John Hannibal Smith <hannibal@a-team.com>\")")
+	flChanges := opts.NewListOpts(nil)
+	cmd.Var(&flChanges, []string{"c", "-change"}, "Apply Dockerfile instruction to the created image.")
 	// FIXME: --run is deprecated, it will be replaced with inline Dockerfile commands.
 	flConfig := cmd.String([]string{"#run", "#-run"}, "", "This option is deprecated and will be removed in a future version in favor of inline Dockerfile-compatible commands")
 	cmd.Require(flag.Max, 2)
@@ -1726,6 +1732,9 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 	v.Set("tag", tag)
 	v.Set("comment", *flComment)
 	v.Set("author", *flAuthor)
+	for _, change := range flChanges.GetAll() {
+		v.Add("changes", change)
+	}
 
 	if *flPause != true {
 		v.Set("pause", "0")

api/server/server.go

@@ -518,6 +518,7 @@ func postCommit(eng *engine.Engine, version version.Version, w http.ResponseWrit
 	job.Setenv("tag", r.Form.Get("tag"))
 	job.Setenv("author", r.Form.Get("author"))
 	job.Setenv("comment", r.Form.Get("comment"))
+	job.SetenvList("changes", r.Form["changes"])
 	job.SetenvSubEnv("config", &config)
 
 	job.Stdout.Add(stdoutBuffer)
@@ -570,6 +571,7 @@ func postImagesCreate(eng *engine.Engine, version version.Version, w http.Respon
 		}
 		job = eng.Job("import", r.Form.Get("fromSrc"), repo, tag)
 		job.Stdin.Add(r.Body)
+		job.SetenvList("changes", r.Form["changes"])
 	}
 
 	if version.GreaterThan("1.0") {

builder/evaluator.go

@@ -96,6 +96,11 @@ type Builder struct {
 	ForceRemove bool
 	Pull        bool
 
+	// set this to true if we want the builder to not commit between steps.
+	// This is useful when we only want to use the evaluator table to generate
+	// the final configs of the Dockerfile but dont want the layers
+	disableCommit bool
+
 	AuthConfig     *registry.AuthConfig
 	AuthConfigFile *registry.ConfigFile
 

builder/internals.go

@@ -60,6 +60,9 @@ func (b *Builder) readContext(context io.Reader) error {
 }
 
 func (b *Builder) commit(id string, autoCmd []string, comment string) error {
+	if b.disableCommit {
+		return nil
+	}
 	if b.image == "" && !b.noBaseImage {
 		return fmt.Errorf("Please provide a source image with `from` prior to commit")
 	}

builder/job.go

@@ -1,12 +1,16 @@
 package builder
 
 import (
+	"bytes"
+	"encoding/json"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/docker/docker/api"
+	"github.com/docker/docker/builder/parser"
 	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/graph"
@@ -14,9 +18,22 @@ import (
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/docker/docker/registry"
+	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/utils"
 )
 
+// whitelist of commands allowed for a commit/import
+var validCommitCommands = map[string]bool{
+	"entrypoint": true,
+	"cmd":        true,
+	"user":       true,
+	"workdir":    true,
+	"env":        true,
+	"volume":     true,
+	"expose":     true,
+	"onbuild":    true,
+}
+
 type BuilderJob struct {
 	Engine *engine.Engine
 	Daemon *daemon.Daemon
@@ -24,6 +41,7 @@ type BuilderJob struct {
 
 func (b *BuilderJob) Install() {
 	b.Engine.Register("build", b.CmdBuild)
+	b.Engine.Register("build_config", b.CmdBuildConfig)
 }
 
 func (b *BuilderJob) CmdBuild(job *engine.Job) engine.Status {
@@ -138,3 +156,50 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) engine.Status {
 	}
 	return engine.StatusOK
 }
+
+func (b *BuilderJob) CmdBuildConfig(job *engine.Job) engine.Status {
+	if len(job.Args) != 0 {
+		return job.Errorf("Usage: %s\n", job.Name)
+	}
+
+	var (
+		changes   = job.GetenvList("changes")
+		newConfig runconfig.Config
+	)
+
+	if err := job.GetenvJson("config", &newConfig); err != nil {
+		return job.Error(err)
+	}
+
+	ast, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
+	if err != nil {
+		return job.Error(err)
+	}
+
+	// ensure that the commands are valid
+	for _, n := range ast.Children {
+		if !validCommitCommands[n.Value] {
+			return job.Errorf("%s is not a valid change command", n.Value)
+		}
+	}
+
+	builder := &Builder{
+		Daemon:        b.Daemon,
+		Engine:        b.Engine,
+		Config:        &newConfig,
+		OutStream:     ioutil.Discard,
+		ErrStream:     ioutil.Discard,
+		disableCommit: true,
+	}
+
+	for i, n := range ast.Children {
+		if err := builder.dispatch(i, n); err != nil {
+			return job.Error(err)
+		}
+	}
+
+	if err := json.NewEncoder(job.Stdout).Encode(builder.Config); err != nil {
+		return job.Error(err)
+	}
+	return engine.StatusOK
+}

daemon/commit.go

@@ -1,6 +1,9 @@
 package daemon
 
 import (
+	"bytes"
+	"encoding/json"
+
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/runconfig"
@@ -18,11 +21,21 @@ func (daemon *Daemon) ContainerCommit(job *engine.Job) engine.Status {
 	}
 
 	var (
-		config    = container.Config
-		newConfig runconfig.Config
+		config       = container.Config
+		stdoutBuffer = bytes.NewBuffer(nil)
+		newConfig    runconfig.Config
 	)
 
-	if err := job.GetenvJson("config", &newConfig); err != nil {
+	buildConfigJob := daemon.eng.Job("build_config")
+	buildConfigJob.Stdout.Add(stdoutBuffer)
+	buildConfigJob.Setenv("changes", job.Getenv("changes"))
+	// FIXME this should be remove when we remove deprecated config param
+	buildConfigJob.Setenv("config", job.Getenv("config"))
+
+	if err := buildConfigJob.Run(); err != nil {
+		return job.Error(err)
+	}
+	if err := json.NewDecoder(stdoutBuffer).Decode(&newConfig); err != nil {
 		return job.Error(err)
 	}
 

docs/man/docker-commit.1.md

@@ -8,6 +8,7 @@ docker-commit - Create a new image from a container's changes
 **docker commit**
 [**-a**|**--author**[=*AUTHOR*]]
 [**--help**]
+[**-c**|**--change**[= []**]]
 [**-m**|**--message**[=*MESSAGE*]]
 [**-p**|**--pause**[=*true*]]
 CONTAINER [REPOSITORY[:TAG]]
@@ -19,6 +20,10 @@ Using an existing container's name or ID you can create a new image.
 **-a**, **--author**=""
    Author (e.g., "John Hannibal Smith <hannibal@a-team.com>")
 
+**-c** , **--change**=[]
+   Apply specified Dockerfile instructions while committing the image
+   Supported Dockerfile instructions: CMD, ENTRYPOINT, ENV, EXPOSE, ONBUILD, USER, VOLUME, WORKDIR
+
 **--help**
   Print usage statement
 
@@ -38,8 +43,17 @@ create a new image run docker ps to find the container's ID and then run:
     # docker commit -m="Added Apache to Fedora base image" \
       -a="A D Ministrator" 98bd7fc99854 fedora/fedora_httpd:20
 
+## Apply specified Dockerfile instructions while committing the image
+If an existing container was created without the DEBUG environment
+variable set to "true", you can create a new image based on that
+container by first getting the container's ID with docker ps and
+then running:
+
+    # docker commit -c="ENV DEBUG true" 98bd7fc99854 debug-image
+
 # HISTORY
 April 2014, Originally compiled by William Henry (whenry at redhat dot com)
 based on docker.com source material and in
 June 2014, updated by Sven Dowideit <SvenDowideit@home.org.au>
 July 2014, updated by Sven Dowideit <SvenDowideit@home.org.au>
+Oct 2014, updated by Daniel, Dao Quang Minh <daniel at nitrous dot io>

docs/man/docker-import.1.md

@@ -6,9 +6,15 @@ docker-import - Create an empty filesystem image and import the contents of the
 
 # SYNOPSIS
 **docker import**
+[**-c**|**--change**[= []**]]
 [**--help**]
 URL|- [REPOSITORY[:TAG]]
 
+# OPTIONS
+**-c**, **--change**=[]
+   Apply specified Dockerfile instructions while importing the image
+   Supported Dockerfile instructions: CMD, ENTRYPOINT, ENV, EXPOSE, ONBUILD, USER, VOLUME, WORKDIR
+
 # DESCRIPTION
 Create a new filesystem image from the contents of a tarball (`.tar`,
 `.tar.gz`, `.tgz`, `.bzip`, `.tar.xz`, `.txz`) into it, then optionally tag it.
@@ -39,6 +45,11 @@ Import to docker via pipe and stdin:
 
     # tar -c . | docker import - exampleimagedir
 
+## Apply specified Dockerfile instructions while importing the image
+This example sets the docker image ENV variable DEBUG to true by default.
+
+    # tar -c . | docker import -c="ENV DEBUG true" - exampleimagedir
+
 # HISTORY
 April 2014, Originally compiled by William Henry (whenry at redhat dot com)
 based on docker.com source material and internal work.

docs/sources/reference/commandline/cli.md

@@ -694,6 +694,7 @@ you refer to it on the command line.
     Create a new image from a container's changes
 
       -a, --author=""     Author (e.g., "John Hannibal Smith <hannibal@a-team.com>")
+      -c, --change=[]     Apply specified Dockerfile instructions while committing the image
       -m, --message=""    Commit message
       -p, --pause=true    Pause container during commit
 
@@ -708,7 +709,12 @@ while the image is committed. This reduces the likelihood of
 encountering data corruption during the process of creating the commit.
 If this behavior is undesired, set the 'p' option to false.
 
-#### Commit an existing container
+The `--change` option will apply `Dockerfile` instructions to the image
+that is created.
+Supported `Dockerfile` instructions: `CMD`, `ENTRYPOINT`, `ENV`, `EXPOSE`,
+`ONBUILD`, `USER`, `VOLUME`, `WORKDIR`
+
+#### Commit a container
 
     $ sudo docker ps
     ID                  IMAGE               COMMAND             CREATED             STATUS              PORTS
@@ -720,6 +726,19 @@ If this behavior is undesired, set the 'p' option to false.
     REPOSITORY                        TAG                 ID                  CREATED             VIRTUAL SIZE
     SvenDowideit/testimage            version3            f5283438590d        16 seconds ago      335.7 MB
 
+#### Commit a container with new configurations
+
+    $ sudo docker ps
+    ID                  IMAGE               COMMAND             CREATED             STATUS              PORTS
+    c3f279d17e0a        ubuntu:12.04        /bin/bash           7 days ago          Up 25 hours
+    197387f1b436        ubuntu:12.04        /bin/bash           7 days ago          Up 25 hours
+    $ sudo docker inspect -f "{{ .Config.Env }}" c3f279d17e0a
+    [HOME=/ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin]
+    $ sudo docker commit --change "ENV DEBUG true" c3f279d17e0a  SvenDowideit/testimage:version3
+    f5283438590d
+    $ sudo docker inspect -f "{{ .Config.Env }}" f5283438590d
+    [HOME=/ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin DEBUG=true]
+
 ## cp
 
 Copy files/folders from a container's filesystem to the host
@@ -1137,11 +1156,18 @@ NOTE: Docker will warn you if any containers exist that are using these untagged
 	tarball (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) into it, then
 	optionally tag it.
 
+      -c, --change=[]     Apply specified Dockerfile instructions while importing the image
+
 URLs must start with `http` and point to a single file archive (.tar,
 .tar.gz, .tgz, .bzip, .tar.xz, or .txz) containing a root filesystem. If
 you would like to import from a local directory or archive, you can use
 the `-` parameter to take the data from `STDIN`.
 
+The `--change` option will apply `Dockerfile` instructions to the image
+that is created.
+Supported `Dockerfile` instructions: `CMD`, `ENTRYPOINT`, `ENV`, `EXPOSE`,
+`ONBUILD`, `USER`, `VOLUME`, `WORKDIR`
+
 #### Examples
 
 **Import from a remote location:**
@@ -1160,6 +1186,10 @@ Import to docker via pipe and `STDIN`.
 
     $ sudo tar -c . | sudo docker import - exampleimagedir
 
+**Import from a local directory with new configurations:**
+
+    $ sudo tar -c . | sudo docker import --change "ENV DEBUG true" - exampleimagedir
+
 Note the `sudo` in this example – you must preserve
 the ownership of the files (especially root ownership) during the
 archiving with tar. If you are not root (or the sudo command) when you

graph/import.go

@@ -1,12 +1,15 @@
 package graph
 
 import (
+	"bytes"
+	"encoding/json"
 	"net/http"
 	"net/url"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/utils"
 )
 
@@ -15,12 +18,14 @@ func (s *TagStore) CmdImport(job *engine.Job) engine.Status {
 		return job.Errorf("Usage: %s SRC REPO [TAG]", job.Name)
 	}
 	var (
-		src     = job.Args[0]
-		repo    = job.Args[1]
-		tag     string
-		sf      = utils.NewStreamFormatter(job.GetenvBool("json"))
-		archive archive.ArchiveReader
-		resp    *http.Response
+		src          = job.Args[0]
+		repo         = job.Args[1]
+		tag          string
+		sf           = utils.NewStreamFormatter(job.GetenvBool("json"))
+		archive      archive.ArchiveReader
+		resp         *http.Response
+		stdoutBuffer = bytes.NewBuffer(nil)
+		newConfig    runconfig.Config
 	)
 	if len(job.Args) > 2 {
 		tag = job.Args[2]
@@ -47,7 +52,21 @@ func (s *TagStore) CmdImport(job *engine.Job) engine.Status {
 		defer progressReader.Close()
 		archive = progressReader
 	}
-	img, err := s.graph.Create(archive, "", "", "Imported from "+src, "", nil, nil)
+
+	buildConfigJob := job.Eng.Job("build_config")
+	buildConfigJob.Stdout.Add(stdoutBuffer)
+	buildConfigJob.Setenv("changes", job.Getenv("changes"))
+	// FIXME this should be remove when we remove deprecated config param
+	buildConfigJob.Setenv("config", job.Getenv("config"))
+
+	if err := buildConfigJob.Run(); err != nil {
+		return job.Error(err)
+	}
+	if err := json.NewDecoder(stdoutBuffer).Decode(&newConfig); err != nil {
+		return job.Error(err)
+	}
+
+	img, err := s.graph.Create(archive, "", "", "Imported from "+src, "", nil, &newConfig)
 	if err != nil {
 		return job.Error(err)
 	}

integration-cli/docker_cli_commit_test.go

@@ -240,3 +240,41 @@ func TestCommitWithHostBindMount(t *testing.T) {
 
 	logDone("commit - commit bind mounted file")
 }
+
+func TestCommitChange(t *testing.T) {
+	defer deleteAllContainers()
+
+	cmd := exec.Command(dockerBinary, "run", "--name", "test", "busybox", "true")
+	if _, err := runCommand(cmd); err != nil {
+		t.Fatal(err)
+	}
+
+	cmd = exec.Command(dockerBinary, "commit",
+		"--change", "EXPOSE 8080",
+		"--change", "ENV DEBUG true",
+		"--change", "ENV test 1",
+		"test", "test-commit")
+	imageId, _, err := runCommandWithOutput(cmd)
+	if err != nil {
+		t.Fatal(imageId, err)
+	}
+	imageId = strings.Trim(imageId, "\r\n")
+	defer deleteImages(imageId)
+
+	expected := map[string]string{
+		"Config.ExposedPorts": "map[8080/tcp:map[]]",
+		"Config.Env":          "[DEBUG=true test=1]",
+	}
+
+	for conf, value := range expected {
+		res, err := inspectField(imageId, conf)
+		if err != nil {
+			t.Errorf("failed to get value %s, error: %s", conf, err)
+		}
+		if res != value {
+			t.Errorf("%s('%s'), expected %s", conf, res, value)
+		}
+	}
+
+	logDone("commit - commit --change")
+}

integration/api_test.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/docker/docker/api"
 	"github.com/docker/docker/api/server"
+	"github.com/docker/docker/builder"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
@@ -158,6 +159,8 @@ func TestGetContainersTop(t *testing.T) {
 
 func TestPostCommit(t *testing.T) {
 	eng := NewTestEngine(t)
+	b := &builder.BuilderJob{Engine: eng}
+	b.Install()
 	defer mkDaemonFromEngine(eng, t).Nuke()
 
 	// Create a container and remove a file

integration/server_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/builder"
 	"github.com/docker/docker/engine"
 )
 
@@ -22,6 +23,8 @@ func TestCreateNumberHostname(t *testing.T) {
 
 func TestCommit(t *testing.T) {
 	eng := NewTestEngine(t)
+	b := &builder.BuilderJob{Engine: eng}
+	b.Install()
 	defer mkDaemonFromEngine(eng, t).Nuke()
 
 	config, _, _, err := parseRun([]string{unitTestImageID, "/bin/cat"})
@@ -42,6 +45,8 @@ func TestCommit(t *testing.T) {
 
 func TestMergeConfigOnCommit(t *testing.T) {
 	eng := NewTestEngine(t)
+	b := &builder.BuilderJob{Engine: eng}
+	b.Install()
 	runtime := mkDaemonFromEngine(eng, t)
 	defer runtime.Nuke()