Merge pull request #6089 from unclejack/docker_build_copy

add support for COPY to docker build

docs/sources/reference/builder.md

@@ -290,6 +290,46 @@ The copy obeys the following rules:
 - If `<dest>` doesn't exist, it is created along with all missing directories
   in its path.
 
+## COPY
+
+    COPY <src> <dest>
+
+The `COPY` instruction will copy new files from `<src>` and add them to the
+container's filesystem at path `<dest>`.
+
+`<src>` must be the path to a file or directory relative to the source directory
+being built (also called the *context* of the build).
+
+`<dest>` is the absolute path to which the source will be copied inside the
+destination container.
+
+All new files and directories are created with a uid and gid of 0.
+
+> **Note**:
+> If you build using STDIN (`docker build - < somefile`), there is no
+> build context, so `COPY` can't be used.
+
+The copy obeys the following rules:
+
+- The `<src>` path must be inside the *context* of the build;
+  you cannot `COPY ../something /something`, because the first step of a
+  `docker build` is to send the context directory (and subdirectories) to the
+  docker daemon.
+
+- If `<src>` is a directory, the entire directory is copied, including
+  filesystem metadata.
+
+- If `<src>` is any other kind of file, it is copied individually along with
+  its metadata. In this case, if `<dest>` ends with a trailing slash `/`, it
+  will be considered a directory and the contents of `<src>` will be written
+  at `<dest>/base(<src>)`.
+
+- If `<dest>` does not end with a trailing slash, it will be considered a
+  regular file and the contents of `<src>` will be written at `<dest>`.
+
+- If `<dest>` doesn't exist, it is created along with all missing directories
+  in its path.
+
 ## ENTRYPOINT
 
 ENTRYPOINT has two forms:

integration-cli/build_tests/TestCopy/DirContentToExistDir/Dockerfile

@@ -0,0 +1,10 @@
+FROM busybox
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
+RUN echo 'dockerio:x:1001:' >> /etc/group
+RUN mkdir /exists
+RUN touch /exists/exists_file
+RUN chown -R dockerio.dockerio /exists
+COPY test_dir/ /exists/
+RUN [ $(ls -l / | grep exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
+RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
+RUN [ $(ls -l /exists/test_file | awk '{print $3":"$4}') = 'root:root' ]

integration-cli/build_tests/TestCopy/DirContentToRoot/Dockerfile

@@ -0,0 +1,8 @@
+FROM busybox
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
+RUN echo 'dockerio:x:1001:' >> /etc/group
+RUN touch /exists
+RUN chown dockerio.dockerio exists
+COPY test_dir /
+RUN [ $(ls -l /test_file | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l /exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]

integration-cli/build_tests/TestCopy/DisallowRemote/Dockerfile

@@ -0,0 +1,2 @@
+FROM busybox
+COPY https://index.docker.io/robots.txt /

integration-cli/build_tests/TestCopy/EtcToRoot/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY . /

integration-cli/build_tests/TestCopy/SingleFileToExistDir/Dockerfile

@@ -0,0 +1,10 @@
+FROM busybox
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
+RUN echo 'dockerio:x:1001:' >> /etc/group
+RUN mkdir /exists
+RUN touch /exists/exists_file
+RUN chown -R dockerio.dockerio /exists
+COPY test_file /exists/
+RUN [ $(ls -l / | grep exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
+RUN [ $(ls -l /exists/test_file | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' ]

integration-cli/build_tests/TestCopy/SingleFileToNonExistDir/Dockerfile

@@ -0,0 +1,9 @@
+FROM busybox
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
+RUN echo 'dockerio:x:1001:' >> /etc/group
+RUN touch /exists
+RUN chown dockerio.dockerio /exists
+COPY test_file /test_dir/
+RUN [ $(ls -l / | grep test_dir | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l /test_dir/test_file | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l /exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]

integration-cli/build_tests/TestCopy/SingleFileToRoot/Dockerfile

@@ -0,0 +1,9 @@
+FROM busybox
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
+RUN echo 'dockerio:x:1001:' >> /etc/group
+RUN touch /exists
+RUN chown dockerio.dockerio /exists
+COPY test_file /
+RUN [ $(ls -l /test_file | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l /test_file | awk '{print $1}') = '-rw-r--r--' ]
+RUN [ $(ls -l /exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]

integration-cli/build_tests/TestCopy/SingleFileToWorkdir/Dockerfile

@@ -0,0 +1,2 @@
+FROM busybox
+COPY test_file .

integration-cli/build_tests/TestCopy/WholeDirToRoot/Dockerfile

@@ -0,0 +1,11 @@
+FROM busybox
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
+RUN echo 'dockerio:x:1001:' >> /etc/group
+RUN touch /exists
+RUN chown dockerio.dockerio exists
+COPY test_dir /test_dir
+RUN [ $(ls -l / | grep test_dir | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l / | grep test_dir | awk '{print $1}') = 'drwxr-xr-x' ]
+RUN [ $(ls -l /test_dir/test_file | awk '{print $3":"$4}') = 'root:root' ]
+RUN [ $(ls -l /test_dir/test_file | awk '{print $1}') = '-rw-r--r--' ]
+RUN [ $(ls -l /exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]

integration-cli/docker_cli_build_test.go

@@ -237,6 +237,181 @@ func TestAddEtcToRoot(t *testing.T) {
 	logDone("build - add etc directory to root")
 }
 
+func TestCopySingleFileToRoot(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy", "SingleFileToRoot")
+	f, err := os.OpenFile(filepath.Join(buildDirectory, "test_file"), os.O_CREATE, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", ".")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - copy single file to root")
+}
+
+// Issue #3960: "ADD src ." hangs - adapted for COPY
+func TestCopySingleFileToWorkdir(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy", "SingleFileToWorkdir")
+	f, err := os.OpenFile(filepath.Join(buildDirectory, "test_file"), os.O_CREATE, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", ".")
+	buildCmd.Dir = buildDirectory
+	done := make(chan error)
+	go func() {
+		out, exitCode, err := runCommandWithOutput(buildCmd)
+		if err != nil || exitCode != 0 {
+			done <- fmt.Errorf("build failed to complete: %s %v", out, err)
+			return
+		}
+		done <- nil
+	}()
+	select {
+	case <-time.After(5 * time.Second):
+		if err := buildCmd.Process.Kill(); err != nil {
+			fmt.Printf("could not kill build (pid=%d): %v\n", buildCmd.Process.Pid, err)
+		}
+		t.Fatal("build timed out")
+	case err := <-done:
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - copy single file to workdir")
+}
+
+func TestCopySingleFileToExistDir(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", "SingleFileToExistDir")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - add single file to existing dir")
+}
+
+func TestCopySingleFileToNonExistDir(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", "SingleFileToNonExistDir")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - copy single file to non-existing dir")
+}
+
+func TestCopyDirContentToRoot(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", "DirContentToRoot")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - copy directory contents to root")
+}
+
+func TestCopyDirContentToExistDir(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", "DirContentToExistDir")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - copy directory contents to existing dir")
+}
+
+func TestCopyWholeDirToRoot(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy", "WholeDirToRoot")
+	test_dir := filepath.Join(buildDirectory, "test_dir")
+	if err := os.MkdirAll(test_dir, 0755); err != nil {
+		t.Fatal(err)
+	}
+	f, err := os.OpenFile(filepath.Join(test_dir, "test_file"), os.O_CREATE, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", ".")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+
+	logDone("build - copy whole directory to root")
+}
+
+func TestCopyEtcToRoot(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", "EtcToRoot")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	deleteImages("testcopyimg")
+	logDone("build - copy etc directory to root")
+}
+
+func TestCopyDisallowRemote(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "testcopyimg", "DisallowRemote")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+
+	if err == nil || exitCode == 0 {
+		t.Fatalf("building the image should've failed; output: %s", out)
+	}
+
+	deleteImages("testcopyimg")
+	logDone("build - copy - disallow copy from remote")
+}
+
 // Issue #5270 - ensure we throw a better error than "unexpected EOF"
 // when we can't access files in the context.
 func TestBuildWithInaccessibleFilesInContext(t *testing.T) {

server/buildfile.go

@@ -340,7 +340,7 @@ func (b *buildFile) CmdInsert(args string) error {
 }
 
 func (b *buildFile) CmdCopy(args string) error {
-	return fmt.Errorf("COPY has been deprecated. Please use ADD instead")
+	return b.runContextCommand(args, false, false, "COPY")
 }
 
 func (b *buildFile) CmdWorkdir(workdir string) error {
@@ -399,7 +399,7 @@ func (b *buildFile) checkPathForAddition(orig string) error {
 	return nil
 }
 
-func (b *buildFile) addContext(container *daemon.Container, orig, dest string, remote bool) error {
+func (b *buildFile) addContext(container *daemon.Container, orig, dest string, decompress bool) error {
 	var (
 		err        error
 		destExists = true
@@ -439,8 +439,8 @@ func (b *buildFile) addContext(container *daemon.Container, orig, dest string, r
 		return copyAsDirectory(origPath, destPath, destExists)
 	}
 
-	// If we are adding a remote file, do not try to untar it
-	if !remote {
+	// If we are adding a remote file (or we've been told not to decompress), do not try to untar it
+	if decompress {
 		// First try to unpack the source as an archive
 		// to support the untar feature we need to clean up the path a little bit
 		// because tar is very forgiving.  First we need to strip off the archive's
@@ -473,13 +473,13 @@ func (b *buildFile) addContext(container *daemon.Container, orig, dest string, r
 	return fixPermissions(resPath, 0, 0)
 }
 
-func (b *buildFile) CmdAdd(args string) error {
+func (b *buildFile) runContextCommand(args string, allowRemote bool, allowDecompression bool, cmdName string) error {
 	if b.context == nil {
-		return fmt.Errorf("No context given. Impossible to use ADD")
+		return fmt.Errorf("No context given. Impossible to use %s", cmdName)
 	}
 	tmp := strings.SplitN(args, " ", 2)
 	if len(tmp) != 2 {
-		return fmt.Errorf("Invalid ADD format")
+		return fmt.Errorf("Invalid %s format", cmdName)
 	}
 
 	orig, err := b.ReplaceEnvMatches(strings.Trim(tmp[0], " \t"))
@@ -493,7 +493,7 @@ func (b *buildFile) CmdAdd(args string) error {
 	}
 
 	cmd := b.config.Cmd
-	b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)}
+	b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, orig, dest)}
 	defer func(cmd []string) { b.config.Cmd = cmd }(cmd)
 	b.config.Image = b.image
 
@@ -502,11 +502,14 @@ func (b *buildFile) CmdAdd(args string) error {
 		destPath   = dest
 		remoteHash string
 		isRemote   bool
+		decompress = true
 	)
 
-	if utils.IsURL(orig) {
+	isRemote = utils.IsURL(orig)
+	if isRemote && !allowRemote {
+		return fmt.Errorf("Source can't be an URL for %s", cmdName)
+	} else if utils.IsURL(orig) {
 		// Initiate the download
-		isRemote = true
 		resp, err := utils.Download(orig)
 		if err != nil {
 			return err
@@ -608,7 +611,7 @@ func (b *buildFile) CmdAdd(args string) error {
 				hash = "file:" + h
 			}
 		}
-		b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", hash, dest)}
+		b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, hash, dest)}
 		hit, err := b.probeCache()
 		if err != nil {
 			return err
@@ -631,16 +634,23 @@ func (b *buildFile) CmdAdd(args string) error {
 	}
 	defer container.Unmount()
 
-	if err := b.addContext(container, origPath, destPath, isRemote); err != nil {
+	if !allowDecompression || isRemote {
+		decompress = false
+	}
+	if err := b.addContext(container, origPath, destPath, decompress); err != nil {
 		return err
 	}
 
-	if err := b.commit(container.ID, cmd, fmt.Sprintf("ADD %s in %s", orig, dest)); err != nil {
+	if err := b.commit(container.ID, cmd, fmt.Sprintf("%s %s in %s", cmdName, orig, dest)); err != nil {
 		return err
 	}
 	return nil
 }
 
+func (b *buildFile) CmdAdd(args string) error {
+	return b.runContextCommand(args, true, true, "ADD")
+}
+
 func (b *buildFile) create() (*daemon.Container, error) {
 	if b.image == "" {
 		return nil, fmt.Errorf("Please provide a source image with `from` prior to run")