diff --git a/Makefile b/Makefile index 6050000582..dd365dc30e 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ release: $(BINRELEASE) s3cmd -P put $(BINRELEASE) s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-$(RELEASE_VERSION).tgz s3cmd -P put docker-latest.tgz s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-latest.tgz s3cmd -P put $(SRCRELEASE)/bin/docker s3://get.docker.io/builds/`uname -s`/`uname -m`/docker + echo $(RELEASE_VERSION) > latest ; s3cmd -P put latest s3://get.docker.io/latest ; rm latest srcrelease: $(SRCRELEASE) deps: $(DOCKER_DIR) @@ -65,7 +66,6 @@ $(BINRELEASE): $(SRCRELEASE) rm -f $(BINRELEASE) cd $(SRCRELEASE); make; cp -R bin docker-$(RELEASE_VERSION); tar -f ../$(BINRELEASE) -zv -c docker-$(RELEASE_VERSION) cd $(SRCRELEASE); cp -R bin docker-latest; tar -f ../docker-latest.tgz -zv -c docker-latest - clean: @rm -rf $(dir $(DOCKER_BIN)) ifeq ($(GOPATH), $(BUILD_DIR)) diff --git a/Vagrantfile b/Vagrantfile index 7258af5bf7..aadabb8711 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -20,8 +20,6 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " - # Listen on all interfaces so that the daemon is accessible from the host - pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " diff --git a/api.go b/api.go index cc7482be71..f520d7a73b 100644 --- a/api.go +++ b/api.go @@ -87,7 +87,7 @@ func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reque if err != nil { return err } - status, err := auth.Login(authConfig) + status, err := auth.Login(authConfig, srv.HTTPRequestFactory()) if err != nil { return err } @@ -488,7 +488,12 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r return err } - if len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns() { + resolvConf, err := utils.GetResolvConf() + if err != nil { + return err + } + + if len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) { out.Warnings = append(out.Warnings, fmt.Sprintf("Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns)) config.Dns = defaultDns } @@ -793,12 +798,8 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") - tag := "" - if strings.Contains(repoName, ":") { - remoteParts := strings.Split(repoName, ":") - tag = remoteParts[1] - repoName = remoteParts[0] - } + rawNoCache := r.FormValue("nocache") + repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader @@ -844,8 +845,12 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ if err != nil { return err } + noCache, err := getBoolParam(rawNoCache) + if err != nil { + return err + } - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput) + b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache) id, err := b.Build(context) if err != nil { fmt.Fprintf(w, "Error build: %s\n", err) diff --git a/archive.go b/archive.go index 01af86006f..bb79cd34d4 100644 --- a/archive.go +++ b/archive.go @@ -173,7 +173,7 @@ func CopyWithTar(src, dst string) error { } // Create dst, copy src's content into it utils.Debugf("Creating dest directory: %s", dst) - if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) diff --git a/auth/auth.go b/auth/auth.go index 6dd6ceb620..b9e1ee153b 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker/utils" "io/ioutil" "net/http" "os" @@ -140,7 +141,7 @@ func SaveConfig(configFile *ConfigFile) error { } // try to register/login to the registry server -func Login(authConfig *AuthConfig) (string, error) { +func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { client := &http.Client{} reqStatusCode := 0 var status string @@ -171,7 +172,7 @@ func Login(authConfig *AuthConfig) (string, error) { "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil) + req, err := factory.NewRequest("GET", IndexServerAddress()+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/auth/auth_test.go b/auth/auth_test.go index d94d429da1..24a0666cf7 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -33,7 +33,7 @@ func TestLogin(t *testing.T) { os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") defer os.Setenv("DOCKER_INDEX_URL", "") authConfig := &AuthConfig{Username: "unittester", Password: "surlautrerivejetattendrai", Email: "noise+unittester@dotcloud.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func TestCreateAccount(t *testing.T) { token := hex.EncodeToString(tokenBuffer)[:12] username := "ut" + token authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestCreateAccount(t *testing.T) { t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status) } - status, err = Login(authConfig) + status, err = Login(authConfig, nil) if err == nil { t.Fatalf("Expected error but found nil instead") } diff --git a/builder.go b/builder.go index 420370b1e6..9124f76ac1 100644 --- a/builder.go +++ b/builder.go @@ -38,7 +38,9 @@ func (builder *Builder) Create(config *Config) (*Container, error) { MergeConfig(config, img.Config) } - if config.Cmd == nil || len(config.Cmd) == 0 { + if len(config.Entrypoint) != 0 && config.Cmd == nil { + config.Cmd = []string{} + } else if config.Cmd == nil || len(config.Cmd) == 0 { return nil, fmt.Errorf("No command specified") } @@ -80,7 +82,12 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return nil, err } - if len(config.Dns) == 0 && len(builder.runtime.Dns) == 0 && utils.CheckLocalDns() { + resolvConf, err := utils.GetResolvConf() + if err != nil { + return nil, err + } + + if len(config.Dns) == 0 && len(builder.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) { //"WARNING: Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns builder.runtime.Dns = defaultDns } diff --git a/buildfile.go b/buildfile.go index fda6e5bf41..00c747d6d0 100644 --- a/buildfile.go +++ b/buildfile.go @@ -26,11 +26,12 @@ type buildFile struct { builder *Builder srv *Server - image string - maintainer string - config *Config - context string - verbose bool + image string + maintainer string + config *Config + context string + verbose bool + utilizeCache bool tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -92,17 +93,21 @@ func (b *buildFile) CmdRun(args string) error { b.config.Cmd = nil MergeConfig(b.config, config) + defer func(cmd []string) { b.config.Cmd = cmd }(cmd) + utils.Debugf("Command to be executed: %v", b.config.Cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } cid, err := b.run() @@ -112,7 +117,7 @@ func (b *buildFile) CmdRun(args string) error { if err := b.commit(cid, cmd, "run"); err != nil { return err } - b.config.Cmd = cmd + return nil } @@ -397,16 +402,19 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } + container, err := b.builder.Create(b.config) if err != nil { return err @@ -500,7 +508,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { +func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache bool) BuildFile { return &buildFile{ builder: NewBuilder(srv.runtime), runtime: srv.runtime, @@ -510,5 +518,6 @@ func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, + utilizeCache: utilizeCache, } } diff --git a/buildfile_test.go b/buildfile_test.go index 78e53b8419..9d002f0665 100644 --- a/buildfile_test.go +++ b/buildfile_test.go @@ -195,21 +195,23 @@ func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { func TestBuild(t *testing.T) { for _, ctx := range testContexts { - buildImage(ctx, t) + buildImage(ctx, t, nil, true) } } -func buildImage(context testContextTemplate, t *testing.T) *Image { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image { + if srv == nil { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) - srv := &Server{ - runtime: runtime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), + srv = &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } } httpServer, err := mkTestingFileServer(context.remoteFiles) @@ -224,10 +226,10 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { } port := httpServer.URL[idx+1:] - ip := runtime.networkManager.bridgeNetwork.IP + ip := srv.runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false) + buildfile := NewBuildFile(srv, ioutil.Discard, false, useCache) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) @@ -245,7 +247,7 @@ func TestVolume(t *testing.T) { from {IMAGE} volume /test cmd Hello world - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if len(img.Config.Volumes) == 0 { t.Fail() @@ -261,7 +263,7 @@ func TestBuildMaintainer(t *testing.T) { img := buildImage(testContextTemplate{` from {IMAGE} maintainer dockerio - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if img.Author != "dockerio" { t.Fail() @@ -273,7 +275,7 @@ func TestBuildEnv(t *testing.T) { from {IMAGE} env port 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) hasEnv := false for _, envVar := range img.Config.Env { if envVar == "port=4243" { @@ -291,7 +293,7 @@ func TestBuildCmd(t *testing.T) { from {IMAGE} cmd ["/bin/echo", "Hello World"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -308,7 +310,7 @@ func TestBuildExpose(t *testing.T) { from {IMAGE} expose 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -320,8 +322,104 @@ func TestBuildEntrypoint(t *testing.T) { from {IMAGE} entrypoint ["/bin/echo"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Entrypoint[0] != "/bin/echo" { } } + +// testing #1405 - config.Cmd does not get cleaned up if +// utilizing cache +func TestBuildEntrypointRunCleanup(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + img := buildImage(testContextTemplate{` + from {IMAGE} + run echo "hello" + `, + nil, nil}, t, srv, true) + + img = buildImage(testContextTemplate{` + from {IMAGE} + run echo "hello" + add foo /foo + entrypoint ["/bin/echo"] + `, + [][2]string{{"foo", "HEYO"}}, nil}, t, srv, true) + + if len(img.Config.Cmd) != 0 { + t.Fail() + } +} + +func TestBuildImageWithCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, true) + + if imageId != img.ID { + t.Logf("Image ids should match: %s != %s", imageId, img.ID) + t.Fail() + } +} + +func TestBuildImageWithoutCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, false) + + if imageId == img.ID { + t.Logf("Image ids should not match: %s == %s", imageId, img.ID) + t.Fail() + } +} diff --git a/commands.go b/commands.go index cf4941dbe1..aa991811ca 100644 --- a/commands.go +++ b/commands.go @@ -158,9 +158,9 @@ func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) { func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") - tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") + tag := cmd.String("t", "", "Repository name (and optionally a tag) to be applied to the resulting image in case of success") suppressOutput := cmd.Bool("q", false, "Suppress verbose build output") - + noCache := cmd.Bool("no-cache", false, "Do not use cache when building the image") if err := cmd.Parse(args); err != nil { return nil } @@ -208,6 +208,9 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if isRemote { v.Set("remote", cmd.Arg(0)) } + if *noCache { + v.Set("nocache", "1") + } req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) if err != nil { return err @@ -448,6 +451,15 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if out.GoVersion != "" { fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } + + release := utils.GetReleaseVersion() + if release != "" { + fmt.Fprintf(cli.out, "Last stable version: %s", release) + if strings.Trim(VERSION, "-dev") != release || strings.Trim(out.Version, "-dev") != release { + fmt.Fprintf(cli.out, ", please update docker") + } + fmt.Fprintf(cli.out, "\n") + } return nil } @@ -814,10 +826,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } - if err := cli.checkIfLogged("push"); err != nil { - return err - } - // If we're not using a custom registry, we know the restrictions // applied to repository names and can warn the user in advance. // Custom repositories can have different rules, and we must also @@ -826,13 +834,22 @@ func (cli *DockerCli) CmdPush(args ...string) error { return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.configFile.Configs[auth.IndexServerAddress()].Username, name) } - buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) - if err != nil { - return err + v := url.Values{} + push := func() error { + buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) + if err != nil { + return err + } + + return cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out) } - v := url.Values{} - if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { + if err := push(); err != nil { + if err == fmt.Errorf("Authentication is required.") { + if err = cli.checkIfLogged("push"); err == nil { + return push() + } + } return err } return nil diff --git a/container.go b/container.go index d610c3c7d4..56a4e191a1 100644 --- a/container.go +++ b/container.go @@ -266,7 +266,8 @@ func (container *Container) FromDisk() error { return err } // Load container settings - if err := json.Unmarshal(data, container); err != nil { + // udp broke compat of docker.PortMapping, but it's not used when loading a container, we can skip it + if err := json.Unmarshal(data, container); err != nil && !strings.Contains(err.Error(), "docker.PortMapping") { return err } return nil @@ -652,6 +653,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-e", "HOME=/", "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "-e", "container=lxc", + "-e", "HOSTNAME="+container.Config.Hostname, ) for _, elem := range container.Config.Env { diff --git a/container_test.go b/container_test.go index a1ac0bd33a..2f9da15f6e 100644 --- a/container_test.go +++ b/container_test.go @@ -960,6 +960,7 @@ func TestEnv(t *testing.T) { "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME=/", "container=lxc", + "HOSTNAME=" + container.ShortID(), } sort.Strings(goodEnv) if len(goodEnv) != len(actualEnv) { @@ -995,6 +996,28 @@ func TestEntrypoint(t *testing.T) { } } +func TestEntrypointNoCmd(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + container, err := NewBuilder(runtime).Create( + &Config{ + Image: GetTestImage(runtime).ID, + Entrypoint: []string{"/bin/echo", "foobar"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + output, err := container.Output() + if err != nil { + t.Fatal(err) + } + if strings.Trim(string(output), "\r\n") != "foobar" { + t.Error(string(output)) + } +} + func grepFile(t *testing.T, path string, pattern string) { f, err := os.Open(path) if err != nil { diff --git a/docs/sources/api/docker_remote_api_v1.0.rst b/docs/sources/api/docker_remote_api_v1.0.rst index 5aa98cbe59..f499a5f53a 100644 --- a/docs/sources/api/docker_remote_api_v1.0.rst +++ b/docs/sources/api/docker_remote_api_v1.0.rst @@ -832,7 +832,7 @@ Build an image from Dockerfile via stdin {{ STREAM }} - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/api/docker_remote_api_v1.2.rst b/docs/sources/api/docker_remote_api_v1.2.rst index 96ee6bb9bb..bd869ca824 100644 --- a/docs/sources/api/docker_remote_api_v1.2.rst +++ b/docs/sources/api/docker_remote_api_v1.2.rst @@ -870,7 +870,7 @@ Build an image from Dockerfile via stdin {{ STREAM }} - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :query remote: resource to fetch, as URI :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/api/docker_remote_api_v1.3.rst b/docs/sources/api/docker_remote_api_v1.3.rst index 401855365e..f2b4062fef 100644 --- a/docs/sources/api/docker_remote_api_v1.3.rst +++ b/docs/sources/api/docker_remote_api_v1.3.rst @@ -925,7 +925,7 @@ Build an image from Dockerfile via stdin The Content-type header should be set to "application/tar". - :query t: tag to be applied to the resulting image in case of success + :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success :query q: suppress verbose build output :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/api/docker_remote_api_v1.4.rst b/docs/sources/api/docker_remote_api_v1.4.rst index 6ee0b35fa2..7406a42b9d 100644 --- a/docs/sources/api/docker_remote_api_v1.4.rst +++ b/docs/sources/api/docker_remote_api_v1.4.rst @@ -926,8 +926,9 @@ Build an image from Dockerfile via stdin The Content-type header should be set to "application/tar". - :query t: tag to be applied to the resulting image in case of success + :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success :query q: suppress verbose build output + :query nocache: do not use the cache when building the image :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/commandline/command/build.rst b/docs/sources/commandline/command/build.rst index 45b6d2ec8e..80e5d2c647 100644 --- a/docs/sources/commandline/command/build.rst +++ b/docs/sources/commandline/command/build.rst @@ -10,8 +10,9 @@ Usage: docker build [OPTIONS] PATH | URL | - Build a new container image from the source code at PATH - -t="": Tag to be applied to the resulting image in case of success. + -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success. -q=false: Suppress verbose build output. + -no-cache: Do not use the cache when building the image. When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context @@ -28,6 +29,13 @@ Examples | If the absolute path is provided instead of '.', only the files and directories required by the ADD commands from the Dockerfile will be added to the context and transferred to the docker daemon. | +.. code-block:: bash + + docker build -t vieux/apache:2.0 . + +| This will build like the preview example, but it will then tag the resulting image, the repository name will be 'vieux/apache' and the tag will be '2.0' + + .. code-block:: bash docker build - < Dockerfile diff --git a/docs/sources/installation/kernel.rst b/docs/sources/installation/kernel.rst index 58730f8191..7c5715a62d 100644 --- a/docs/sources/installation/kernel.rst +++ b/docs/sources/installation/kernel.rst @@ -15,12 +15,11 @@ In short, Docker has the following kernel requirements: - Cgroups and namespaces must be enabled. - - The officially supported kernel is the one recommended by the - :ref:`ubuntu_linux` installation path. It is the one that most developers - will use, and the one that receives the most attention from the core - contributors. If you decide to go with a different kernel and hit a bug, - please try to reproduce it with the official kernels first. +The officially supported kernel is the one recommended by the +:ref:`ubuntu_linux` installation path. It is the one that most developers +will use, and the one that receives the most attention from the core +contributors. If you decide to go with a different kernel and hit a bug, +please try to reproduce it with the official kernels first. If you cannot or do not want to use the "official" kernels, here is some technical background about the features (both optional and diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index ed592d3a9d..cf41b79266 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -19,6 +19,8 @@ Docker has the following dependencies * Linux kernel 3.8 (read more about :ref:`kernel`) * AUFS file system support (we are working on BTRFS support as an alternative) +Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ + .. _ubuntu_precise: Ubuntu Precise 12.04 (LTS) (64-bit) @@ -135,3 +137,35 @@ Verify it worked **Done!**, now continue with the :ref:`hello_world` example. + + +.. _ufw: + +Docker and UFW +^^^^^^^^^^^^^^ + +Docker uses a bridge to manage containers networking, by default UFW drop all `forwarding`, a first step is to enable forwarding: + +.. code-block:: bash + + sudo nano /etc/default/ufw + ---- + # Change: + # DEFAULT_FORWARD_POLICY="DROP" + # to + DEFAULT_FORWARD_POLICY="ACCEPT" + +Then reload UFW: + +.. code-block:: bash + + sudo ufw reload + + +UFW's default set of rules denied all `incoming`, so if you want to be able to reach your containers from another host, +you should allow incoming connections on the docker port (default 4243): + +.. code-block:: bash + + sudo ufw allow 4243/tcp + diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index aa2fd6b92a..9f5ee8b795 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -182,7 +182,7 @@ The copy obeys the following rules: written at ````. * If ```` doesn't exist, it is created along with all missing directories in its path. All new files and directories are created - with mode 0700, uid and gid 0. + with mode 0755, uid and gid 0. 3.8 ENTRYPOINT -------------- diff --git a/docs/theme/docker/layout.html b/docs/theme/docker/layout.html index 0b22f22fab..ca26f44dc0 100755 --- a/docs/theme/docker/layout.html +++ b/docs/theme/docker/layout.html @@ -79,7 +79,7 @@
- +
diff --git a/registry/registry.go b/registry/registry.go index 4e9dd8895f..0d2a64e053 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -69,7 +69,8 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && + nameParts[0] != "localhost" { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return auth.IndexServerAddress(), reposName, err @@ -100,13 +101,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } -// VersionInfo is used to model entities which has a version. -// It is basically a tupple with name and version. -type VersionInfo interface { - Name() string - Version() string -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -121,33 +115,18 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -// Set the user agent field in the header based on the versions provided -// in NewRegistry() and extra. -func (r *Registry) setUserAgent(req *http.Request, extra ...VersionInfo) { - if len(r.baseVersions)+len(extra) == 0 { - return - } - if len(extra) == 0 { - req.Header.Set("User-Agent", r.baseVersionsStr) - } else { - req.Header.Set("User-Agent", appendVersions(r.baseVersionsStr, extra...)) - } - return -} - // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err } @@ -170,7 +149,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -185,19 +164,18 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) @@ -213,12 +191,11 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -239,7 +216,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -281,7 +257,6 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { @@ -289,12 +264,12 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } var tokens []string @@ -339,7 +314,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } @@ -375,13 +350,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { @@ -391,7 +365,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -399,7 +373,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil } @@ -410,14 +384,13 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr tarsumLayer := &utils.TarSum{Reader: layer} - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -427,15 +400,15 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { - req, err := http.NewRequest(method, urlStr, body) + req, err := r.reqFactory.NewRequest(method, urlStr, body) if err != nil { return nil, err } @@ -455,7 +428,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -463,7 +435,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -500,7 +472,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -521,7 +492,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -540,7 +510,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -564,7 +534,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) } } @@ -576,7 +546,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) - req, err := http.NewRequest("GET", u, nil) + req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } @@ -586,7 +556,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { @@ -628,52 +598,12 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - baseVersions []VersionInfo - baseVersionsStr string + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory } -func validVersion(version VersionInfo) bool { - stopChars := " \t\r\n/" - if strings.ContainsAny(version.Name(), stopChars) { - return false - } - if strings.ContainsAny(version.Version(), stopChars) { - return false - } - return true -} - -// Convert versions to a string and append the string to the string base. -// -// Each VersionInfo will be converted to a string in the format of -// "product/version", where the "product" is get from the Name() method, while -// version is get from the Version() method. Several pieces of verson information -// will be concatinated and separated by space. -func appendVersions(base string, versions ...VersionInfo) string { - if len(versions) == 0 { - return base - } - - var buf bytes.Buffer - if len(base) > 0 { - buf.Write([]byte(base)) - } - - for _, v := range versions { - if !validVersion(v) { - continue - } - buf.Write([]byte(v.Name())) - buf.Write([]byte("/")) - buf.Write([]byte(v.Version())) - buf.Write([]byte(" ")) - } - return buf.String() -} - -func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionInfo) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -689,7 +619,7 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...Versi if err != nil { return nil, err } - r.baseVersions = baseVersions - r.baseVersionsStr = appendVersions("", baseVersions...) + + r.reqFactory = factory return r, nil } diff --git a/server.go b/server.go index 7a7912d0a9..7607851f49 100644 --- a/server.go +++ b/server.go @@ -52,9 +52,9 @@ func (v *simpleVersionInfo) Version() string { // docker, go, git-commit (of the docker) and the host's kernel. // // Such information will be used on call to NewRegistry(). -func (srv *Server) versionInfos() []registry.VersionInfo { +func (srv *Server) versionInfos() []utils.VersionInfo { v := srv.DockerVersion() - ret := make([]registry.VersionInfo, 0, 4) + ret := make([]utils.VersionInfo, 0, 4) ret = append(ret, &simpleVersionInfo{"docker", v.Version}) if len(v.GoVersion) > 0 { @@ -102,7 +102,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - r, err := registry.NewRegistry(srv.runtime.root, nil, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, nil, srv.HTTPRequestFactory()) if err != nil { return nil, err } @@ -580,7 +580,7 @@ func (srv *Server) poolRemove(kind, key string) error { } func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, parallel bool) error { - r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err != nil { return err } @@ -740,7 +740,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err2 != nil { return err2 } @@ -997,13 +997,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - var tag string - if strings.Contains(name, ":") { - nameParts := strings.Split(name, ":") - name = nameParts[0] - tag = nameParts[1] - } - + name, tag := utils.ParseRepositoryTag(name) return srv.deleteImage(img, name, tag) } @@ -1190,11 +1184,21 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( pushingPool: make(map[string]struct{}), events: make([]utils.JSONMessage, 0, 64), //only keeps the 64 last events listeners: make(map[string]chan utils.JSONMessage), + reqFactory: nil, } runtime.srv = srv return srv, nil } +func (srv *Server) HTTPRequestFactory() *utils.HTTPRequestFactory { + if srv.reqFactory == nil { + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + factory := utils.NewHTTPRequestFactory(ud) + srv.reqFactory = factory + } + return srv.reqFactory +} + func (srv *Server) LogEvent(action, id string) { now := time.Now().Unix() jm := utils.JSONMessage{Status: action, ID: id, Time: now} @@ -1215,4 +1219,5 @@ type Server struct { pushingPool map[string]struct{} events []utils.JSONMessage listeners map[string]chan utils.JSONMessage + reqFactory *utils.HTTPRequestFactory } diff --git a/server_test.go b/server_test.go index 57e2376565..104e94bb79 100644 --- a/server_test.go +++ b/server_test.go @@ -21,16 +21,20 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } + if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { t.Fatal(err) } + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil { + t.Fatal(err) + } images, err := srv.Images(false, "") if err != nil { t.Fatal(err) } - if len(images) != len(initialImages)+2 { + if len(images) != len(initialImages)+3 { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } @@ -43,6 +47,19 @@ func TestContainerTagImageDelete(t *testing.T) { t.Fatal(err) } + if len(images) != len(initialImages)+2 { + t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) + } + + if _, err := srv.ImageDelete("utest:5000/docker:tag3", true); err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + if len(images) != len(initialImages)+1 { t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images)) } diff --git a/testing/README.rst b/testing/README.rst index 3b11092f9f..ce5aa837a4 100644 --- a/testing/README.rst +++ b/testing/README.rst @@ -40,6 +40,10 @@ Deployment export SMTP_USER=xxxxxxxxxxxx export SMTP_PWD=xxxxxxxxxxxx + # Define docker registry functional test credentials + export REGISTRY_USER=xxxxxxxxxxxx + export REGISTRY_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/testing/Vagrantfile b/testing/Vagrantfile index 47257201dc..e76a951508 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -29,7 +29,9 @@ Vagrant::Config.run do |config| "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ + "#{CFG_PATH}/setup_credentials.sh #{USER} " \ + "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/testing/buildbot/credentials.cfg b/testing/buildbot/credentials.cfg new file mode 100644 index 0000000000..fbdd35d578 --- /dev/null +++ b/testing/buildbot/credentials.cfg @@ -0,0 +1,5 @@ +# Credentials for tests. Buildbot source this file on tests +# when needed. + +# Docker registry credentials. Format: 'username:password' +export DOCKER_CREDS='' diff --git a/testing/buildbot/master.cfg b/testing/buildbot/master.cfg index 61912808ec..29926dbe5f 100644 --- a/testing/buildbot/master.cfg +++ b/testing/buildbot/master.cfg @@ -19,6 +19,7 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' +BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' @@ -41,16 +42,19 @@ c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER + # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'coverage'])] + 'registry','coverage'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] + # Builders +# Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " @@ -58,6 +62,7 @@ factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' @@ -69,6 +74,17 @@ factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] +# Registry Functionaltest builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry', logEnviron=False, + command='. {0}/master/credentials.cfg; ' + '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, + DOCKER_PATH), usePTY=True)) + +c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], + factory=factory)] + + # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') diff --git a/testing/buildbot/requirements.txt b/testing/buildbot/requirements.txt index 0e451b017d..4e183ba062 100644 --- a/testing/buildbot/requirements.txt +++ b/testing/buildbot/requirements.txt @@ -4,3 +4,4 @@ buildbot==0.8.7p1 buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 +flask==0.10.1 diff --git a/testing/buildbot/setup_credentials.sh b/testing/buildbot/setup_credentials.sh new file mode 100755 index 0000000000..f093815d60 --- /dev/null +++ b/testing/buildbot/setup_credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Setup of test credentials. Called by Vagrantfile +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +USER=$1 +REGISTRY_USER=$2 +REGISTRY_PWD=$3 + +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" + +function run { su $USER -c "$1"; } + +run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" +cd $BUILDBOT_PATH/master +run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/testing/functionaltests/test_registry.sh b/testing/functionaltests/test_registry.sh new file mode 100755 index 0000000000..095a731631 --- /dev/null +++ b/testing/functionaltests/test_registry.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Cleanup +rm -rf docker-registry + +# Get latest docker registry +git clone https://github.com/dotcloud/docker-registry.git + +# Configure and run registry tests +cd docker-registry; cp config_sample.yml config.yml +cd test; python -m unittest workflow diff --git a/utils/http.go b/utils/http.go new file mode 100644 index 0000000000..8c1e4b7a79 --- /dev/null +++ b/utils/http.go @@ -0,0 +1,134 @@ +package utils + +import ( + "bytes" + "io" + "net/http" + "strings" +) + +// VersionInfo is used to model entities which has a version. +// It is basically a tupple with name and version. +type VersionInfo interface { + Name() string + Version() string +} + +func validVersion(version VersionInfo) bool { + stopChars := " \t\r\n/" + if strings.ContainsAny(version.Name(), stopChars) { + return false + } + if strings.ContainsAny(version.Version(), stopChars) { + return false + } + return true +} + +// Convert versions to a string and append the string to the string base. +// +// Each VersionInfo will be converted to a string in the format of +// "product/version", where the "product" is get from the Name() method, while +// version is get from the Version() method. Several pieces of verson information +// will be concatinated and separated by space. +func appendVersions(base string, versions ...VersionInfo) string { + if len(versions) == 0 { + return base + } + + var buf bytes.Buffer + if len(base) > 0 { + buf.Write([]byte(base)) + } + + for _, v := range versions { + name := []byte(v.Name()) + version := []byte(v.Version()) + + if len(name) == 0 || len(version) == 0 { + continue + } + if !validVersion(v) { + continue + } + buf.Write([]byte(v.Name())) + buf.Write([]byte("/")) + buf.Write([]byte(v.Version())) + buf.Write([]byte(" ")) + } + return buf.String() +} + +// HTTPRequestDecorator is used to change an instance of +// http.Request. It could be used to add more header fields, +// change body, etc. +type HTTPRequestDecorator interface { + // ChangeRequest() changes the request accordingly. + // The changed request will be returned or err will be non-nil + // if an error occur. + ChangeRequest(req *http.Request) (newReq *http.Request, err error) +} + +// HTTPUserAgentDecorator appends the product/version to the user agent field +// of a request. +type HTTPUserAgentDecorator struct { + versions []VersionInfo +} + +func NewHTTPUserAgentDecorator(versions ...VersionInfo) HTTPRequestDecorator { + ret := new(HTTPUserAgentDecorator) + ret.versions = versions + return ret +} + +func (self *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) { + if req == nil { + return req, nil + } + + userAgent := appendVersions(req.UserAgent(), self.versions...) + if len(userAgent) > 0 { + req.Header.Set("User-Agent", userAgent) + } + return req, nil +} + +// HTTPRequestFactory creates an HTTP request +// and applies a list of decorators on the request. +type HTTPRequestFactory struct { + decorators []HTTPRequestDecorator +} + +func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory { + ret := new(HTTPRequestFactory) + ret.decorators = d + return ret +} + +// NewRequest() creates a new *http.Request, +// applies all decorators in the HTTPRequestFactory on the request, +// then applies decorators provided by d on the request. +func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) { + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + return nil, err + } + + // By default, a nil factory should work. + if self == nil { + return req, nil + } + for _, dec := range self.decorators { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + for _, dec := range d { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + return req, err +} diff --git a/utils/utils.go b/utils/utils.go index f91fb0663c..e0083d02d8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -605,17 +605,37 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + type JSONMessage struct { - Status string `json:"status,omitempty"` - Progress string `json:"progress,omitempty"` - Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + Status string `json:"status,omitempty"` + Progress string `json:"progress,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message +} + +func NewHTTPRequestError(msg string, res *http.Response) error { + return &JSONError{ + Message: msg, + Code: res.StatusCode, + } } func (jm *JSONMessage) Display(out io.Writer) error { - if jm.Error != "" { - return fmt.Errorf(jm.Error) + if jm.Error != nil { + if jm.Error.Code == 401 { + return fmt.Errorf("Authentication is required.") + } + return jm.Error } fmt.Fprintf(out, "%c[2K", 27) if jm.Time != 0 { @@ -694,7 +714,11 @@ func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []b func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil { + jsonError, ok := err.(*JSONError) + if !ok { + jsonError = &JSONError{Message: err.Error()} + } + if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") @@ -726,17 +750,29 @@ func IsGIT(str string) bool { return strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "github.com/") } -func CheckLocalDns() bool { +// GetResolvConf opens and read the content of /etc/resolv.conf. +// It returns it as byte slice. +func GetResolvConf() ([]byte, error) { resolv, err := ioutil.ReadFile("/etc/resolv.conf") if err != nil { Debugf("Error openning resolv.conf: %s", err) - return false + return nil, err } - for _, ip := range []string{ - "127.0.0.1", - "127.0.1.1", + return resolv, nil +} + +// CheckLocalDns looks into the /etc/resolv.conf, +// it returns true if there is a local nameserver or if there is no nameserver. +func CheckLocalDns(resolvConf []byte) bool { + if !bytes.Contains(resolvConf, []byte("nameserver")) { + return true + } + + for _, ip := range [][]byte{ + []byte("127.0.0.1"), + []byte("127.0.1.1"), } { - if strings.Contains(string(resolv), ip) { + if bytes.Contains(resolvConf, ip) { return true } } @@ -768,6 +804,22 @@ func ParseHost(host string, port int, addr string) string { return fmt.Sprintf("tcp://%s:%d", host, port) } +func GetReleaseVersion() string { + resp, err := http.Get("http://get.docker.io/latest") + if err != nil { + return "" + } + defer resp.Body.Close() + if resp.ContentLength > 24 || resp.StatusCode != 200 { + return "" + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + return strings.TrimSpace(string(body)) +} + // Get a repos name and returns the right reposName + tag // The tag can be confusing because of a port in a repository name. // Ex: localhost.localdomain:5000/samalba/hipache:latest diff --git a/utils/utils_test.go b/utils/utils_test.go index 5caa809f67..882165ac54 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -282,3 +282,58 @@ func TestParseHost(t *testing.T) { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } } + +func TestParseRepositoryTag(t *testing.T) { + if repo, tag := ParseRepositoryTag("root"); repo != "root" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) + } +} + +func TestGetResolvConf(t *testing.T) { + resolvConfUtils, err := GetResolvConf() + if err != nil { + t.Fatal(err) + } + resolvConfSystem, err := ioutil.ReadFile("/etc/resolv.conf") + if err != nil { + t.Fatal(err) + } + if string(resolvConfUtils) != string(resolvConfSystem) { + t.Fatalf("/etc/resolv.conf and GetResolvConf have different content.") + } +} + +func TestCheclLocalDns(t *testing.T) { + for resolv, result := range map[string]bool{`# Dynamic +nameserver 10.0.2.3 +search dotcloud.net`: false, + `# Dynamic +nameserver 127.0.0.1 +search dotcloud.net`: true, + `# Dynamic +nameserver 127.0.1.1 +search dotcloud.net`: true, + `# Dynamic +`: true, + ``: true, + } { + if CheckLocalDns([]byte(resolv)) != result { + t.Fatalf("Wrong local dns detection: {%s} should be %v", resolv, result) + } + } +} diff --git a/utils_test.go b/utils_test.go index c4adeb4a74..91df6183fc 100644 --- a/utils_test.go +++ b/utils_test.go @@ -191,7 +191,7 @@ func TestMergeConfig(t *testing.T) { if len(configUser.Volumes) != 3 { t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes)) } - for v, _ := range configUser.Volumes { + for v := range configUser.Volumes { if v != "/test1" && v != "/test2" && v != "/test3" { t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v) }