diff --git a/api_test.go b/api_test.go index c2e7db4bd0..4306f74100 100644 --- a/api_test.go +++ b/api_test.go @@ -19,25 +19,25 @@ import ( func TestGetBoolParam(t *testing.T) { if ret, err := getBoolParam("true"); err != nil || !ret { - t.Fatalf("true -> true, nil | got %b %s", ret, err) + t.Fatalf("true -> true, nil | got %t %s", ret, err) } if ret, err := getBoolParam("True"); err != nil || !ret { - t.Fatalf("True -> true, nil | got %b %s", ret, err) + t.Fatalf("True -> true, nil | got %t %s", ret, err) } if ret, err := getBoolParam("1"); err != nil || !ret { - t.Fatalf("1 -> true, nil | got %b %s", ret, err) + t.Fatalf("1 -> true, nil | got %t %s", ret, err) } if ret, err := getBoolParam(""); err != nil || ret { - t.Fatalf("\"\" -> false, nil | got %b %s", ret, err) + t.Fatalf("\"\" -> false, nil | got %t %s", ret, err) } if ret, err := getBoolParam("false"); err != nil || ret { - t.Fatalf("false -> false, nil | got %b %s", ret, err) + t.Fatalf("false -> false, nil | got %t %s", ret, err) } if ret, err := getBoolParam("0"); err != nil || ret { - t.Fatalf("0 -> false, nil | got %b %s", ret, err) + t.Fatalf("0 -> false, nil | got %t %s", ret, err) } if ret, err := getBoolParam("faux"); err == nil || ret { - t.Fatalf("faux -> false, err | got %b %s", ret, err) + t.Fatalf("faux -> false, err | got %t %s", ret, err) } } diff --git a/commands.go b/commands.go index 3ce8c8eb98..4ef63f4a22 100644 --- a/commands.go +++ b/commands.go @@ -345,7 +345,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } auth.SaveConfig(cli.authConfig) if out2.Status != "" { - fmt.Fprintln(cli.out, "%s\n", out2.Status) + fmt.Fprintf(cli.out, "%s\n", out2.Status) } return nil } @@ -370,7 +370,7 @@ func (cli *DockerCli) CmdWait(args ...string) error { if err != nil { return err } - fmt.Fprintf(cli.out, "%s\n", out.StatusCode) + fmt.Fprintf(cli.out, "%d\n", out.StatusCode) } } return nil @@ -405,7 +405,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { fmt.Fprintf(cli.out, "Git commit: %s\n", out.GitCommit) } if out.GoVersion != "" { - fmt.Fprintln(cli.out, "Go version: %s\n", out.GoVersion) + fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } return nil } @@ -513,7 +513,7 @@ func (cli *DockerCli) CmdStart(args ...string) error { if err != nil { fmt.Fprintf(cli.err, "%s\n", err) } else { - fmt.Fprintln(cli.out, "%s\n", name) + fmt.Fprintf(cli.out, "%s\n", name) } } return nil @@ -736,23 +736,30 @@ func (cli *DockerCli) CmdPush(args ...string) error { return err } - if len(strings.SplitN(name, "/", 2)) == 1 { - return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) + if *registry == "" { + // If we're not using a custom registry, we know the restrictions + // applied to repository names and can warn the user in advance. + // Custom repositories can have different rules, and we must also + // allow pushing by image ID. + if len(strings.SplitN(name, "/", 2)) == 1 { + return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.authConfig.Username, name) + } + + nameParts := strings.SplitN(name, "/", 2) + validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + if !validNamespace.MatchString(nameParts[0]) { + return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) + } + validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + if !validRepo.MatchString(nameParts[1]) { + return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) + } } buf, err := json.Marshal(cli.authConfig) if err != nil { return err } - nameParts := strings.SplitN(name, "/", 2) - validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) - if !validNamespace.MatchString(nameParts[0]) { - return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", nameParts[0]) - } - validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) - if !validRepo.MatchString(nameParts[1]) { - return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", nameParts[1]) - } v := url.Values{} v.Set("registry", *registry) diff --git a/docs/sources/conf.py b/docs/sources/conf.py index 41dba70201..ea8e1f43f0 100644 --- a/docs/sources/conf.py +++ b/docs/sources/conf.py @@ -30,6 +30,7 @@ import sys, os html_additional_pages = { 'concepts/containers': 'redirect_home.html', 'concepts/introduction': 'redirect_home.html', + 'builder/basics': 'redirect_build.html', } diff --git a/docs/theme/docker/layout.html b/docs/theme/docker/layout.html index 707888a927..f4f17f2dba 100755 --- a/docs/theme/docker/layout.html +++ b/docs/theme/docker/layout.html @@ -40,8 +40,11 @@ {%- set script_files = script_files + ['_static/js/docs.js'] %} + {%- if pagename == 'index' %} + + {% else %} - + {% endif %} {%- for cssfile in css_files %} {%- endfor %} diff --git a/docs/theme/docker/redirect_build.html b/docs/theme/docker/redirect_build.html new file mode 100644 index 0000000000..1f26fc3aaa --- /dev/null +++ b/docs/theme/docker/redirect_build.html @@ -0,0 +1,12 @@ + + + + Page Moved + + + + +This page has moved. Perhaps you should visit the Builder page + + + diff --git a/docs/theme/docker/redirect_home.html b/docs/theme/docker/redirect_home.html index 41f8da244d..109239f819 100644 --- a/docs/theme/docker/redirect_home.html +++ b/docs/theme/docker/redirect_home.html @@ -2,7 +2,7 @@ Page Moved - + diff --git a/graph.go b/graph.go index f2b2ccec8e..0bf7eccdbe 100644 --- a/graph.go +++ b/graph.go @@ -189,7 +189,7 @@ func (graph *Graph) Mktemp(id string) (string, error) { return "", fmt.Errorf("Couldn't create temp: %s", err) } if tmp.Exists(id) { - return "", fmt.Errorf("Image %d already exists", id) + return "", fmt.Errorf("Image %s already exists", id) } return tmp.imageRoot(id), nil } diff --git a/registry/registry.go b/registry/registry.go index ed1cd4056f..622c09b3f3 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -64,20 +64,19 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s } // Check if an image exists in the Registry -func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { +func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } - req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) if err != nil { return false } res.Body.Close() - return res.StatusCode == 307 + return res.StatusCode == 200 } func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { @@ -163,7 +162,10 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("%s://%s/v1/repositories/%s/tags", UrlScheme(), host, repository) + endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) + if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { + endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint) + } req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -173,6 +175,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ if err != nil { return nil, err } + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() @@ -257,7 +260,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = fmt.Sprintf("%s://%s/v1", UrlScheme(), registry) + registry = registry + "/v1" // FIXME: try json with UTF8 req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { @@ -293,7 +296,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = fmt.Sprintf("%s://%s/v1", UrlScheme(), registry) + registry = registry + "/v1" req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) if err != nil { return err @@ -331,7 +334,7 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = fmt.Sprintf("%s://%s/v1", UrlScheme(), registry) + registry = registry + "/v1" req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { diff --git a/server.go b/server.go index 9a8eccc710..0a88b309ec 100644 --- a/server.go +++ b/server.go @@ -351,26 +351,49 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag string, sf *utils.StreamFormatter) error { +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, remote, askedTag, registryEp string, sf *utils.StreamFormatter) error { out.Write(sf.FormatStatus("Pulling repository %s from %s", local, auth.IndexServerAddress())) - repoData, err := r.GetRepositoryData(remote) - if err != nil { - return err - } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err + var repoData *registry.RepositoryData + var err error + if registryEp == "" { + repoData, err = r.GetRepositoryData(remote) + if err != nil { + return err + } + + utils.Debugf("Updating checksums") + // Reload the json file to make sure not to overwrite faster sums + if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { + return err + } + } else { + repoData = ®istry.RepositoryData{ + Tokens: []string{}, + ImgList: make(map[string]*registry.ImgData), + Endpoints: []string{registryEp}, + } } utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) if err != nil { + utils.Debugf("%v", err) return err } + + if registryEp != "" { + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + ID: id, + Tag: tag, + Checksum: "", + } + } + } + utils.Debugf("Registering tags") - // If not specific tag have been asked, take all + // If no tag has been specified, pull them all if askedTag == "" { for tag, id := range tagsList { repoData.ImgList[id].Tag = tag @@ -392,8 +415,10 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, local, re out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, remote)) success := false for _, ep := range repoData.Endpoints { - ep = fmt.Sprintf("%s://%s/v1", registry.UrlScheme(), ep) - if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { + ep = fmt.Sprintf("%s://%s", registry.UrlScheme(), ep) + } + if err := srv.pullImage(r, out, img.ID, ep+"/v1", repoData.Tokens, sf); err != nil { out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) continue } @@ -453,7 +478,6 @@ func (srv *Server) poolRemove(kind, key string) error { } return nil } - func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { r, err := registry.NewRegistry(srv.runtime.root, authConfig) if err != nil { @@ -464,21 +488,20 @@ func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, sf *util } defer srv.poolRemove("pull", name+":"+tag) - out = utils.NewWriteFlusher(out) - if endpoint != "" { - if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { - return err - } - return nil - } remote := name parts := strings.Split(name, "/") if len(parts) > 2 { remote = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } - if err := srv.pullRepository(r, out, name, remote, tag, sf); err != nil { - return err + out = utils.NewWriteFlusher(out) + err = srv.pullRepository(r, out, name, remote, tag, endpoint, sf) + if err != nil && endpoint != "" { + if err := srv.pullImage(r, out, name, endpoint, nil, sf); err != nil { + return err + } + return nil } + return nil } @@ -548,7 +571,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return imgList, nil } -func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string, sf *utils.StreamFormatter) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name, registryEp string, localRepo map[string]string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) out.Write(sf.FormatStatus("Processing checksums")) imgList, err := srv.getImageList(localRepo) @@ -556,25 +579,51 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri return err } out.Write(sf.FormatStatus("Sending image list")) - srvName := name parts := strings.Split(name, "/") if len(parts) > 2 { srvName = fmt.Sprintf("src/%s", url.QueryEscape(strings.Join(parts, "/"))) } - repoData, err := r.PushImageJSONIndex(srvName, imgList, false, nil) - if err != nil { - return err + var repoData *registry.RepositoryData + if registryEp == "" { + repoData, err = r.PushImageJSONIndex(name, imgList, false, nil) + if err != nil { + return err + } + } else { + repoData = ®istry.RepositoryData{ + ImgList: make(map[string]*registry.ImgData), + Tokens: []string{}, + Endpoints: []string{registryEp}, + } + tagsList, err := r.GetRemoteTags(repoData.Endpoints, name, repoData.Tokens) + if err != nil && err.Error() != "Repository not found" { + return err + } else if err == nil { + for tag, id := range tagsList { + repoData.ImgList[id] = ®istry.ImgData{ + ID: id, + Tag: tag, + Checksum: "", + } + } + } } for _, ep := range repoData.Endpoints { + if !(strings.HasPrefix(ep, "http://") || strings.HasPrefix(ep, "https://")) { + ep = "https://" + ep + } out.Write(sf.FormatStatus("Pushing repository %s to %s (%d tags)", name, ep, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { out.Write(sf.FormatStatus("Image %s already on registry, skipping", name)) continue + } else if registryEp != "" && r.LookupRemoteImage(elem.ID, registryEp, repoData.Tokens) { + fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) + continue } if err := srv.pushImage(r, out, name, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? @@ -587,9 +636,12 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name stri } } - if _, err := r.PushImageJSONIndex(srvName, imgList, true, repoData.Endpoints); err != nil { - return err + if registryEp == "" { + if _, err := r.PushImageJSONIndex(name, imgList, true, repoData.Endpoints); err != nil { + return err + } } + return nil } @@ -666,11 +718,12 @@ func (srv *Server) ImagePush(name, endpoint string, out io.Writer, sf *utils.Str if err2 != nil { return err2 } + if err != nil { out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", name, len(srv.runtime.repositories.Repositories[name]))) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(r, out, name, localRepo, sf); err != nil { + if err := srv.pushRepository(r, out, name, endpoint, localRepo, sf); err != nil { return err } return nil diff --git a/testing/Vagrantfile b/testing/Vagrantfile index f2f6ca8248..e3b25a6f9d 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -19,9 +19,10 @@ Vagrant::Config.run do |config| config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." config.vm.network :hostonly, BUILDBOT_IP + # Deploy buildbot and its dependencies if it was not done if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? - pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " + pkg_cmd = "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; " # Deploy buildbot CI pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ "pip install -r #{CFG_PATH}/requirements.txt; " \ @@ -29,7 +30,7 @@ Vagrant::Config.run do |config| "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ - "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ + "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; " # Activate new kernel pkg_cmd << "shutdown -r +1; " @@ -40,6 +41,7 @@ end # Providers were added on Vagrant >= 1.1.0 Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| config.vm.provider :aws do |aws, override| + aws.tags = { 'Name' => 'docker-ci' } aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"] aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"] aws.keypair_name = ENV["AWS_KEYPAIR_NAME"] diff --git a/utils/utils.go b/utils/utils.go index 6e12f1f1bb..2f2a52867e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -78,7 +78,7 @@ func (r *progressReader) Read(p []byte) (n int, err error) { read, err := io.ReadCloser(r.reader).Read(p) r.readProgress += read - updateEvery := 1024*512 //512kB + updateEvery := 1024 * 512 //512kB if r.readTotal > 0 { // Update progress for every 1% read if 1% < 512kB if increment := int(0.01 * float64(r.readTotal)); increment < updateEvery { @@ -87,7 +87,7 @@ func (r *progressReader) Read(p []byte) (n int, err error) { } if r.readProgress-r.lastUpdate > updateEvery || err != nil { if r.readTotal > 0 { - fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%2.0f%%",float64(r.readProgress)/float64(r.readTotal)*100)) + fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%2.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) } else { fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a") } @@ -133,7 +133,7 @@ func HumanDuration(d time.Duration) string { } else if hours < 24*365*2 { return fmt.Sprintf("%d months", hours/24/30) } - return fmt.Sprintf("%d years", d.Hours()/24/365) + return fmt.Sprintf("%f years", d.Hours()/24/365) } // HumanSize returns a human-readable approximation of a size