From be7560890644759c0ddd229afed208c537f075d7 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 26 Apr 2013 14:00:09 -0700 Subject: [PATCH] Fixed checksum computing. Ensure checksum is computed when image metadata is loaded from disk. Fixed docker push workflow. Moved hash computing to utils --- container.go | 10 +-- graph.go | 9 ++- image.go | 29 ++++++++ registry.go | 190 ++++++++++++++++++++++++++++----------------------- utils.go | 10 +++ 5 files changed, 155 insertions(+), 93 deletions(-) diff --git a/container.go b/container.go index 519137ff85..5040203227 100644 --- a/container.go +++ b/container.go @@ -1,8 +1,6 @@ package docker import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -698,15 +696,11 @@ func (container *Container) ExportRw() (Archive, error) { } func (container *Container) RwChecksum() (string, error) { - h := sha256.New() - rwData, err := container.ExportRw() + rwData, err := Tar(container.rwPath(), Xz) if err != nil { return "", err } - if _, err := io.Copy(h, rwData); err != nil { - return "", err - } - return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil + return HashData(rwData) } func (container *Container) Export() (Archive, error) { diff --git a/graph.go b/graph.go index a2a6f47af7..d7a53be7c3 100644 --- a/graph.go +++ b/graph.go @@ -81,6 +81,13 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph + + if img.Checksum == "" { + err := img.FixChecksum() + if err != nil { + return nil, err + } + } return img, nil } @@ -98,7 +105,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - // FIXME: If an image is pulled from a raw URL (not created from a container), + // FIXME: If an image is imported from a raw URL (not created from a container), // its checksum will not be computed, which will cause a push to fail checksum, err := container.RwChecksum() if err != nil { diff --git a/image.go b/image.go index 8090e583e6..4d625d6c0f 100644 --- a/image.go +++ b/image.go @@ -52,6 +52,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } + return &img, nil } @@ -258,3 +259,31 @@ func (img *Image) layer() (string, error) { } return layerPath(root), nil } + +func (img *Image) FixChecksum() error { + layer, err := img.layer() + if err != nil { + return err + } + layerData, err := Tar(layer, Xz) + if err != nil { + return err + } + sum, err := HashData(layerData) + if err != nil { + return err + } + img.Checksum = sum + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + root, err := img.root() + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + return nil +} diff --git a/registry.go b/registry.go index af69c58adb..d9850843d1 100644 --- a/registry.go +++ b/registry.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/dotcloud/docker/auth" + "github.com/shin-/cookiejar" "io" "io/ioutil" "net/http" @@ -14,7 +15,6 @@ import ( ) //FIXME: Set the endpoint in a conf file or via commandline -//const INDEX_ENDPOINT = "http://registry-creack.dotcloud.com/v1" const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" // Build an Image object from raw json data @@ -47,6 +47,13 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { return ret, nil } +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + return c.Do(req) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { @@ -57,7 +64,6 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return nil, err } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -82,7 +88,9 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] func (graph *Graph) getHttpClient() *http.Client { if graph.httpClient == nil { graph.httpClient = new(http.Client) + graph.httpClient.Jar = cookiejar.NewCookieJar() } + Debugf("cookies: %v",graph.httpClient.Jar) return graph.httpClient } @@ -111,7 +119,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return nil, nil, fmt.Errorf("Failed to download json: %s", err) } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -139,7 +146,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -271,7 +277,6 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return fmt.Errorf("Index response didn't contain any endpoints") } - // FIXME: If askedTag is empty, fetch all tags. var tagsList map[string]string if askedTag == "" { tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) @@ -316,86 +321,86 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return nil } -// Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { +func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, token []string) error { + if parent, err := img.GetParent(); err != nil { + return err + } else if parent != nil { + if err := pushImageRec(graph, stdout, parent, registry, token); err != nil { + return err + } + } client := graph.getHttpClient() - registry = "https://" + registry + "/v1" + jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) + } - // FIXME: Factorize the code - // FIXME: Do the puts in goroutines - if err := imgOrig.WalkHistory(func(img *Image) error { + fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) - if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) - } - - fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - - // FIXME: try json with UTF8 - jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - switch res.StatusCode { - case 204: - // Case where the image is already on the Registry - fmt.Fprintf(stdout, "Image %s already uploaded ; skipping.", img.Id) - return nil - default: - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - } - - fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - tmp, err := ioutil.ReadAll(layerData2) - if err != nil { - return err - } - layerLength := len(tmp) - - layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", - ProgressReader(layerData.(io.ReadCloser), layerLength, stdout)) - if err != nil { - return err - } - req3.ContentLength = int64(tmpLayer.Size) - - req3.TransferEncoding = []string{"none"} - req3.Header["X-Docker-Token"] = token - res3, err := client.Do(req3) - if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) - } - res3.Body.Close() - if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) - } - return nil - }); err != nil { + // FIXME: try json with UTF8 + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) + if err != nil { return err } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + res, err := doWithCookies(client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + errBody = []byte(err.Error()) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) + return nil + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + + fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) + + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", + layerData) + if err != nil { + return err + } + + req3.ContentLength = -1 + req3.TransferEncoding = []string{"chunked"} + req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + fmt.Printf("%v", req3.Header) + res3, err := doWithCookies(client, req3) + if err != nil { + return fmt.Errorf("Failed to upload layer: %s", err) + } + res3.Body.Close() + if res3.StatusCode != 200 { + return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) + } return nil } +// Push a local image to the registry with its history if needed +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { + registry = "https://" + registry + "/v1" + return pushImageRec(graph, stdout, imgOrig, registry, token) +} + // push a tag on the registry. // Remote has the format '/ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { @@ -413,9 +418,13 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []stri client := graph.getHttpClient() req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + if err != nil { + return err + } req.Header.Add("Content-type", "application/json") - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) + req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.ContentLength = int64(len(revision)) + res, err := doWithCookies(client, req) if err != nil { return err } @@ -441,7 +450,7 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, registry, tag, token); err != nil { + if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { return err } return nil @@ -453,15 +462,25 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re client := graph.getHttpClient() checksums, err := graph.Checksums(localRepo) + imgList := make([]map[string]string, len(checksums)) if err != nil { return err } - req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, nil) + for i, obj := range checksums { + imgList[i] = map[string]string{"id": obj["id"]} + } + imgListJson, err := json.Marshal(imgList) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, bytes.NewReader(imgListJson)) if err != nil { return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { @@ -470,11 +489,12 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re res.Body.Close() for res.StatusCode >= 300 && res.StatusCode < 400 { Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = http.NewRequest("PUT", res.Header.Get("Location"), nil) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) if err != nil { return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") res, err = client.Do(req) if err != nil { @@ -490,6 +510,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re var token, endpoints []string if res.Header.Get("X-Docker-Token") != "" { token = res.Header["X-Docker-Token"] + Debugf("Auth token: %v", token) } else { Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") @@ -515,13 +536,14 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewBuffer(checksumsJson)) + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson)) if err != nil { return err } req2.SetBasicAuth(authConfig.Username, authConfig.Password) req2.Header["X-Docker-Endpoints"] = endpoints - res2, err := client.Do(req) + req2.ContentLength = int64(len(checksumsJson)) + res2, err := client.Do(req2) if err != nil { return err } diff --git a/utils.go b/utils.go index 297b798af8..bcd3e0d1b3 100644 --- a/utils.go +++ b/utils.go @@ -2,6 +2,8 @@ package docker import ( "bytes" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "github.com/dotcloud/docker/rcli" @@ -456,3 +458,11 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil +} \ No newline at end of file