From 49fd83a25e2e6604014de41d4f4099a7bc07a09b Mon Sep 17 00:00:00 2001 From: David Calavera Date: Fri, 24 Apr 2015 15:12:45 -0700 Subject: [PATCH] Use git url fragment to specify reference and dir context. Signed-off-by: David Calavera --- docs/sources/reference/commandline/cli.md | 25 ++++- integration-cli/docker_cli_build_test.go | 29 +++++ pkg/urlutil/git.go | 9 +- pkg/urlutil/git_test.go | 12 +++ utils/git.go | 60 +++++++++-- utils/git_test.go | 125 +++++++++++++++++++++- 6 files changed, 247 insertions(+), 13 deletions(-) diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 26659c8ffa..6829464323 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -637,13 +637,36 @@ an [*ADD*](/reference/builder/#add) instruction to reference a file in the context. The `URL` parameter can specify the location of a Git repository; -the repository acts as the build context. The system recursively clones the repository +the repository acts as the build context. The system recursively clones the repository and its submodules using a `git clone --depth 1 --recursive` command. This command runs in a temporary directory on your local host. After the command succeeds, the directory is sent to the Docker daemon as the context. Local clones give you the ability to access private repositories using local user credentials, VPN's, and so forth. +Git URLs accept context configuration in their fragment section, separated by a colon `:`. +The first part represents the reference that Git will check out, this can be either +a branch, a tag, or a commit SHA. The second part represents a subdirectory +inside the repository that will be used as a build context. + +For example, run this command to use a directory called `docker` in the branch `container`: + + $ docker build https://github.com/docker/rootfs.git#container:docker + +The following table represents all the valid suffixes with their build contexts: + +Build Syntax Suffix | Commit Used | Build Context Used +--------------------|-------------|------------------- +`myrepo.git` | `refs/heads/master` | `/` +`myrepo.git#mytag` | `refs/tags/mytag` | `/` +`myrepo.git#mybranch` | `refs/heads/mybranch` | `/` +`myrepo.git#abcdef` | `sha1 = abcdef` | `/` +`myrepo.git#:myfolder` | `refs/heads/master` | `/myfolder` +`myrepo.git#master:myfolder` | `refs/heads/master` | `/myfolder` +`myrepo.git#mytag:myfolder` | `refs/tags/mytag` | `/myfolder` +`myrepo.git#mybranch:myfolder` | `refs/heads/mybranch` | `/myfolder` +`myrepo.git#abcdef:myfolder` | `sha1 = abcdef` | `/myfolder` + Instead of specifying a context, you can pass a single Dockerfile in the `URL` or pipe the file in via `STDIN`. To pipe a Dockerfile from `STDIN`: diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 72d15c177b..b06e29ae29 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -4221,6 +4221,35 @@ func (s *DockerSuite) TestBuildFromGIT(c *check.C) { } } +func (s *DockerSuite) TestBuildFromGITWithContext(c *check.C) { + name := "testbuildfromgit" + defer deleteImages(name) + git, err := fakeGIT("repo", map[string]string{ + "docker/Dockerfile": `FROM busybox + ADD first /first + RUN [ -f /first ] + MAINTAINER docker`, + "docker/first": "test git data", + }, true) + if err != nil { + c.Fatal(err) + } + defer git.Close() + + u := fmt.Sprintf("%s#master:docker", git.RepoURL) + _, err = buildImageFromPath(name, u, true) + if err != nil { + c.Fatal(err) + } + res, err := inspectField(name, "Author") + if err != nil { + c.Fatal(err) + } + if res != "docker" { + c.Fatalf("Maintainer should be docker, got %s", res) + } +} + func (s *DockerSuite) TestBuildCleanupCmdOnEntrypoint(c *check.C) { name := "testbuildcmdcleanuponentrypoint" defer deleteImages(name) diff --git a/pkg/urlutil/git.go b/pkg/urlutil/git.go index ba88ddf6e6..dc4d6662e3 100644 --- a/pkg/urlutil/git.go +++ b/pkg/urlutil/git.go @@ -1,6 +1,9 @@ package urlutil -import "strings" +import ( + "regexp" + "strings" +) var ( validPrefixes = []string{ @@ -8,11 +11,13 @@ var ( "github.com/", "git@", } + + urlPathWithFragmentSuffix = regexp.MustCompile(".git(?:#.+)?$") ) // IsGitURL returns true if the provided str is a git repository URL. func IsGitURL(str string) bool { - if IsURL(str) && strings.HasSuffix(str, ".git") { + if IsURL(str) && urlPathWithFragmentSuffix.MatchString(str) { return true } for _, prefix := range validPrefixes { diff --git a/pkg/urlutil/git_test.go b/pkg/urlutil/git_test.go index 01dcea7da3..bb89d8b5fd 100644 --- a/pkg/urlutil/git_test.go +++ b/pkg/urlutil/git_test.go @@ -9,10 +9,15 @@ var ( "git@bitbucket.org:atlassianlabs/atlassian-docker.git", "https://github.com/docker/docker.git", "http://github.com/docker/docker.git", + "http://github.com/docker/docker.git#branch", + "http://github.com/docker/docker.git#:dir", } incompleteGitUrls = []string{ "github.com/docker/docker", } + invalidGitUrls = []string{ + "http://github.com/docker/docker.git:#branch", + } ) func TestValidGitTransport(t *testing.T) { @@ -35,9 +40,16 @@ func TestIsGIT(t *testing.T) { t.Fatalf("%q should be detected as valid Git url", url) } } + for _, url := range incompleteGitUrls { if IsGitURL(url) == false { t.Fatalf("%q should be detected as valid Git url", url) } } + + for _, url := range invalidGitUrls { + if IsGitURL(url) == true { + t.Fatalf("%q should not be detected as valid Git prefix", url) + } + } } diff --git a/utils/git.go b/utils/git.go index 18e002d184..ce8924d8af 100644 --- a/utils/git.go +++ b/utils/git.go @@ -4,7 +4,10 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" + "os" "os/exec" + "path/filepath" "strings" "github.com/docker/docker/pkg/urlutil" @@ -19,20 +22,26 @@ func GitClone(remoteURL string) (string, error) { return "", err } - clone := cloneArgs(remoteURL, root) + u, err := url.Parse(remoteURL) + if err != nil { + return "", err + } - if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { + fragment := u.Fragment + clone := cloneArgs(u, root) + + if output, err := git(clone...); err != nil { return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output) } - return root, nil + return checkoutGit(fragment, root) } -func cloneArgs(remoteURL, root string) []string { +func cloneArgs(remoteURL *url.URL, root string) []string { args := []string{"clone", "--recursive"} - shallow := true + shallow := len(remoteURL.Fragment) == 0 - if strings.HasPrefix(remoteURL, "http") { + if shallow && strings.HasPrefix(remoteURL.Scheme, "http") { res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { shallow = false @@ -43,5 +52,42 @@ func cloneArgs(remoteURL, root string) []string { args = append(args, "--depth", "1") } - return append(args, remoteURL, root) + if remoteURL.Fragment != "" { + remoteURL.Fragment = "" + } + + return append(args, remoteURL.String(), root) +} + +func checkoutGit(fragment, root string) (string, error) { + refAndDir := strings.SplitN(fragment, ":", 2) + + if len(refAndDir[0]) != 0 { + if output, err := gitWithinDir(root, "checkout", refAndDir[0]); err != nil { + return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output) + } + } + + if len(refAndDir) > 1 && len(refAndDir[1]) != 0 { + newCtx := filepath.Join(root, refAndDir[1]) + fi, err := os.Stat(newCtx) + if err != nil { + return "", err + } + if !fi.IsDir() { + return "", fmt.Errorf("Error setting git context, not a directory: %s", newCtx) + } + root = newCtx + } + + return root, nil +} + +func gitWithinDir(dir string, args ...string) ([]byte, error) { + a := []string{"--work-tree", dir, "--git-dir", filepath.Join(dir, ".git")} + return git(append(a, args...)...) +} + +func git(args ...string) ([]byte, error) { + return exec.Command("git", args...).CombinedOutput() } diff --git a/utils/git_test.go b/utils/git_test.go index a82841ae11..10b13e9627 100644 --- a/utils/git_test.go +++ b/utils/git_test.go @@ -2,9 +2,12 @@ package utils import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "reflect" "testing" ) @@ -22,7 +25,7 @@ func TestCloneArgsSmartHttp(t *testing.T) { w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q)) }) - args := cloneArgs(gitURL, "/tmp") + args := cloneArgs(serverURL, "/tmp") exp := []string{"clone", "--recursive", "--depth", "1", gitURL, "/tmp"} if !reflect.DeepEqual(args, exp) { t.Fatalf("Expected %v, got %v", exp, args) @@ -41,16 +44,132 @@ func TestCloneArgsDumbHttp(t *testing.T) { w.Header().Set("Content-Type", "text/plain") }) - args := cloneArgs(gitURL, "/tmp") + args := cloneArgs(serverURL, "/tmp") exp := []string{"clone", "--recursive", gitURL, "/tmp"} if !reflect.DeepEqual(args, exp) { t.Fatalf("Expected %v, got %v", exp, args) } } + func TestCloneArgsGit(t *testing.T) { - args := cloneArgs("git://github.com/docker/docker", "/tmp") + u, _ := url.Parse("git://github.com/docker/docker") + args := cloneArgs(u, "/tmp") exp := []string{"clone", "--recursive", "--depth", "1", "git://github.com/docker/docker", "/tmp"} if !reflect.DeepEqual(args, exp) { t.Fatalf("Expected %v, got %v", exp, args) } } + +func TestCloneArgsStripFragment(t *testing.T) { + u, _ := url.Parse("git://github.com/docker/docker#test") + args := cloneArgs(u, "/tmp") + exp := []string{"clone", "--recursive", "git://github.com/docker/docker", "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} + +func TestCheckoutGit(t *testing.T) { + root, err := ioutil.TempDir("", "docker-build-git-checkout") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + + gitDir := filepath.Join(root, "repo") + _, err = git("init", gitDir) + if err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "config", "user.email", "test@docker.com"); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "config", "user.name", "Docker test"); err != nil { + t.Fatal(err) + } + + if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0644); err != nil { + t.Fatal(err) + } + + subDir := filepath.Join(gitDir, "subdir") + if err = os.Mkdir(subDir, 0755); err != nil { + t.Fatal(err) + } + + if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0644); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "commit", "-am", "First commit"); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "checkout", "-b", "test"); err != nil { + t.Fatal(err) + } + + if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0644); err != nil { + t.Fatal(err) + } + + if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0644); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "commit", "-am", "Branch commit"); err != nil { + t.Fatal(err) + } + + if _, err = gitWithinDir(gitDir, "checkout", "master"); err != nil { + t.Fatal(err) + } + + cases := []struct { + frag string + exp string + fail bool + }{ + {"", "FROM scratch", false}, + {"master", "FROM scratch", false}, + {":subdir", "FROM scratch\nEXPOSE 5000", false}, + {":nosubdir", "", true}, // missing directory error + {":Dockerfile", "", true}, // not a directory error + {"master:nosubdir", "", true}, + {"master:subdir", "FROM scratch\nEXPOSE 5000", false}, + {"test", "FROM scratch\nEXPOSE 3000", false}, + {"test:", "FROM scratch\nEXPOSE 3000", false}, + {"test:subdir", "FROM busybox\nEXPOSE 5000", false}, + } + + for _, c := range cases { + r, err := checkoutGit(c.frag, gitDir) + + fail := err != nil + if fail != c.fail { + t.Fatalf("Expected %v failure, error was %v\n", c.fail, err) + } + if c.fail { + continue + } + + b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile")) + if err != nil { + t.Fatal(err) + } + + if string(b) != c.exp { + t.Fatalf("Expected %v, was %v\n", c.exp, string(b)) + } + } +}