mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
Merge pull request #568 from dotcloud/improve_checksum-2
* Runtime: Store the actual archive on commit * Registry: Improve the checksum process - Registry: Fix error 400 on push
This commit is contained in:
commit
8fb8a08ff2
3 changed files with 206 additions and 125 deletions
30
graph.go
30
graph.go
|
@ -1,6 +1,7 @@
|
|||
package docker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
@ -112,7 +113,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
|
|||
img.Container = container.Id
|
||||
img.ContainerConfig = *container.Config
|
||||
}
|
||||
if err := graph.Register(layerData, img); err != nil {
|
||||
if err := graph.Register(layerData, true, img); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
go img.Checksum()
|
||||
|
@ -121,7 +122,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
|
|||
|
||||
// Register imports a pre-existing image into the graph.
|
||||
// FIXME: pass img as first argument
|
||||
func (graph *Graph) Register(layerData Archive, img *Image) error {
|
||||
func (graph *Graph) Register(layerData Archive, store bool, img *Image) error {
|
||||
if err := ValidateId(img.Id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -134,7 +135,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("Mktemp failed: %s", err)
|
||||
}
|
||||
if err := StoreImage(img, layerData, tmp); err != nil {
|
||||
if err := StoreImage(img, layerData, tmp, store); err != nil {
|
||||
return err
|
||||
}
|
||||
// Commit
|
||||
|
@ -300,3 +301,26 @@ func (graph *Graph) Heads() (map[string]*Image, error) {
|
|||
func (graph *Graph) imageRoot(id string) string {
|
||||
return path.Join(graph.Root, id)
|
||||
}
|
||||
|
||||
func (graph *Graph) getStoredChecksums() (map[string]string, error) {
|
||||
checksums := make(map[string]string)
|
||||
// FIXME: Store the checksum in memory
|
||||
|
||||
if checksumDict, err := ioutil.ReadFile(path.Join(graph.Root, "checksums")); err == nil {
|
||||
if err := json.Unmarshal(checksumDict, &checksums); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return checksums, nil
|
||||
}
|
||||
|
||||
func (graph *Graph) storeChecksums(checksums map[string]string) error {
|
||||
checksumJson, err := json.Marshal(checksums)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ioutil.WriteFile(path.Join(graph.Root, "checksums"), checksumJson, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
82
image.go
82
image.go
|
@ -56,7 +56,7 @@ func LoadImage(root string) (*Image, error) {
|
|||
return img, nil
|
||||
}
|
||||
|
||||
func StoreImage(img *Image, layerData Archive, root string) error {
|
||||
func StoreImage(img *Image, layerData Archive, root string, store bool) error {
|
||||
// Check that root doesn't already exist
|
||||
if _, err := os.Stat(root); err == nil {
|
||||
return fmt.Errorf("Image %s already exists", img.Id)
|
||||
|
@ -68,6 +68,28 @@ func StoreImage(img *Image, layerData Archive, root string) error {
|
|||
if err := os.MkdirAll(layer, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if store {
|
||||
layerArchive := layerArchivePath(root)
|
||||
file, err := os.OpenFile(layerArchive, os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// FIXME: Retrieve the image layer size from here?
|
||||
if _, err := io.Copy(file, layerData); err != nil {
|
||||
return err
|
||||
}
|
||||
// FIXME: Don't close/open, read/write instead of Copy
|
||||
file.Close()
|
||||
|
||||
file, err = os.Open(layerArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
layerData = file
|
||||
}
|
||||
|
||||
if err := Untar(layerData, layer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -86,6 +108,10 @@ func layerPath(root string) string {
|
|||
return path.Join(root, "layer")
|
||||
}
|
||||
|
||||
func layerArchivePath(root string) string {
|
||||
return path.Join(root, "layer.tar.xz")
|
||||
}
|
||||
|
||||
func jsonPath(root string) string {
|
||||
return path.Join(root, "json")
|
||||
}
|
||||
|
@ -269,16 +295,12 @@ func (img *Image) Checksum() (string, error) {
|
|||
return "", err
|
||||
}
|
||||
|
||||
checksumDictPth := path.Join(root, "..", "..", "checksums")
|
||||
checksums := make(map[string]string)
|
||||
|
||||
if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
|
||||
if err := json.Unmarshal(checksumDict, &checksums); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if checksum, ok := checksums[img.Id]; ok {
|
||||
return checksum, nil
|
||||
}
|
||||
checksums, err := img.graph.getStoredChecksums()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if checksum, ok := checksums[img.Id]; ok {
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
layer, err := img.layer()
|
||||
|
@ -290,9 +312,20 @@ func (img *Image) Checksum() (string, error) {
|
|||
return "", err
|
||||
}
|
||||
|
||||
layerData, err := Tar(layer, Xz)
|
||||
if err != nil {
|
||||
return "", err
|
||||
var layerData io.Reader
|
||||
|
||||
if file, err := os.Open(layerArchivePath(root)); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
layerData, err = Tar(layer, Xz)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
defer file.Close()
|
||||
layerData = file
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
|
@ -306,24 +339,23 @@ func (img *Image) Checksum() (string, error) {
|
|||
if _, err := io.Copy(h, layerData); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
hash := "sha256:" + hex.EncodeToString(h.Sum(nil))
|
||||
checksums[img.Id] = hash
|
||||
|
||||
// Reload the json file to make sure not to overwrite faster sums
|
||||
img.graph.lockSumFile.Lock()
|
||||
defer img.graph.lockSumFile.Unlock()
|
||||
if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
|
||||
if err := json.Unmarshal(checksumDict, &checksums); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
checksumJson, err := json.Marshal(checksums)
|
||||
|
||||
checksums, err = img.graph.getStoredChecksums()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
checksums[img.Id] = hash
|
||||
|
||||
// Dump the checksums to disc
|
||||
if err := img.graph.storeChecksums(checksums); err != nil {
|
||||
return hash, err
|
||||
}
|
||||
if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil {
|
||||
return hash, err
|
||||
}
|
||||
|
||||
return hash, nil
|
||||
}
|
||||
|
|
219
registry.go
219
registry.go
|
@ -10,6 +10,7 @@ import (
|
|||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
@ -259,7 +260,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []
|
|||
// FIXME: Keep goging in case of error?
|
||||
return err
|
||||
}
|
||||
if err = graph.Register(layer, img); err != nil {
|
||||
if err = graph.Register(layer, false, img); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -314,10 +315,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re
|
|||
// Reload the json file to make sure not to overwrite faster sums
|
||||
err = func() error {
|
||||
localChecksums := make(map[string]string)
|
||||
remoteChecksums := []struct {
|
||||
Id string `json: "id"`
|
||||
Checksum string `json: "checksum"`
|
||||
}{}
|
||||
remoteChecksums := []ImgListJson{}
|
||||
checksumDictPth := path.Join(graph.Root, "..", "checksums")
|
||||
|
||||
if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil {
|
||||
|
@ -395,14 +393,10 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re
|
|||
return nil
|
||||
}
|
||||
|
||||
func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, token []string) error {
|
||||
if parent, err := img.GetParent(); err != nil {
|
||||
return err
|
||||
} else if parent != nil {
|
||||
if err := pushImageRec(graph, stdout, parent, registry, token); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Push a local image to the registry
|
||||
func (graph *Graph) PushImage(stdout io.Writer, img *Image, registry string, token []string) error {
|
||||
registry = "https://" + registry + "/v1"
|
||||
|
||||
client := graph.getHttpClient()
|
||||
jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json"))
|
||||
if err != nil {
|
||||
|
@ -425,6 +419,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t
|
|||
return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err)
|
||||
}
|
||||
req.Header.Set("X-Docker-Checksum", checksum)
|
||||
Debugf("Setting checksum for %s: %s", img.ShortId(), checksum)
|
||||
res, err := doWithCookies(client, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to upload metadata: %s", err)
|
||||
|
@ -450,14 +445,35 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t
|
|||
}
|
||||
|
||||
fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id)
|
||||
|
||||
layerData, err := graph.TempLayerArchive(img.Id, Xz, stdout)
|
||||
root, err := img.root()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to generate layer archive: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var layerData *TempArchive
|
||||
// If the archive exists, use it
|
||||
file, err := os.Open(layerArchivePath(root))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// If the archive does not exist, create one from the layer
|
||||
layerData, err = graph.TempLayerArchive(img.Id, Xz, stdout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to generate layer archive: %s", err)
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
defer file.Close()
|
||||
st, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
layerData = &TempArchive{file, st.Size()}
|
||||
}
|
||||
|
||||
req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer",
|
||||
ProgressReader(layerData, -1, stdout, ""))
|
||||
ProgressReader(layerData, int(layerData.Size), stdout, ""))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -482,12 +498,6 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t
|
|||
return nil
|
||||
}
|
||||
|
||||
// Push a local image to the registry with its history if needed
|
||||
func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error {
|
||||
registry = "https://" + registry + "/v1"
|
||||
return pushImageRec(graph, stdout, imgOrig, registry, token)
|
||||
}
|
||||
|
||||
// push a tag on the registry.
|
||||
// Remote has the format '<user>/<repo>
|
||||
func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error {
|
||||
|
@ -537,48 +547,89 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry
|
|||
return nil
|
||||
}
|
||||
|
||||
// Retrieve the checksum of an image
|
||||
// Priority:
|
||||
// - Check on the stored checksums
|
||||
// - Check if the archive exists, if it does not, ask the registry
|
||||
// - If the archive does exists, process the checksum from it
|
||||
// - If the archive does not exists and not found on registry, process checksum from layer
|
||||
func (graph *Graph) getChecksum(imageId string) (string, error) {
|
||||
// FIXME: Use in-memory map instead of reading the file each time
|
||||
if sums, err := graph.getStoredChecksums(); err != nil {
|
||||
return "", err
|
||||
} else if checksum, exists := sums[imageId]; exists {
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
img, err := graph.Get(imageId)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if _, err := os.Stat(layerArchivePath(graph.imageRoot(imageId))); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// TODO: Ask the registry for the checksum
|
||||
// As the archive is not there, it is supposed to come from a pull.
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
checksum, err := img.Checksum()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return checksum, nil
|
||||
}
|
||||
|
||||
type ImgListJson struct {
|
||||
Id string `json:"id"`
|
||||
Checksum string `json:"checksum,omitempty"`
|
||||
tag string
|
||||
}
|
||||
|
||||
// Push a repository to the registry.
|
||||
// Remote has the format '<user>/<repo>
|
||||
func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error {
|
||||
client := graph.getHttpClient()
|
||||
// FIXME: Do not reset the cookie each time? (need to reset it in case updating latest of a repo and repushing)
|
||||
client.Jar = cookiejar.NewCookieJar()
|
||||
var imgList []*ImgListJson
|
||||
|
||||
checksums, err := graph.Checksums(stdout, localRepo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(stdout, "Processing checksums\n")
|
||||
imageSet := make(map[string]struct{})
|
||||
|
||||
imgList := make([]map[string]string, len(checksums))
|
||||
checksums2 := make([]map[string]string, len(checksums))
|
||||
|
||||
uploadedImages, err := graph.getImagesInRepository(remote, authConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error occured while fetching the list: %s", err)
|
||||
}
|
||||
|
||||
// Filter list to only send images/checksums not already uploaded
|
||||
i := 0
|
||||
for _, obj := range checksums {
|
||||
found := false
|
||||
for _, uploadedImg := range uploadedImages {
|
||||
if obj["id"] == uploadedImg["id"] && uploadedImg["checksum"] != "" {
|
||||
found = true
|
||||
break
|
||||
for tag, id := range localRepo {
|
||||
img, err := graph.Get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
img.WalkHistory(func(img *Image) error {
|
||||
if _, exists := imageSet[img.Id]; exists {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
imgList[i] = map[string]string{"id": obj["id"]}
|
||||
checksums2[i] = obj
|
||||
i += 1
|
||||
}
|
||||
imageSet[img.Id] = struct{}{}
|
||||
checksum, err := graph.getChecksum(img.Id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imgList = append([]*ImgListJson{{
|
||||
Id: img.Id,
|
||||
Checksum: checksum,
|
||||
tag: tag,
|
||||
}}, imgList...)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
checksums = checksums2[:i]
|
||||
imgList = imgList[:i]
|
||||
|
||||
imgListJson, err := json.Marshal(imgList)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Debugf("json sent: %s\n", imgListJson)
|
||||
|
||||
fmt.Fprintf(stdout, "Sending image list\n")
|
||||
req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/", bytes.NewReader(imgListJson))
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -586,11 +637,13 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
|
|||
req.SetBasicAuth(authConfig.Username, authConfig.Password)
|
||||
req.ContentLength = int64(len(imgListJson))
|
||||
req.Header.Set("X-Docker-Token", "true")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
for res.StatusCode >= 300 && res.StatusCode < 400 {
|
||||
Debugf("Redirected to %s\n", res.Header.Get("Location"))
|
||||
req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson))
|
||||
|
@ -600,6 +653,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
|
|||
req.SetBasicAuth(authConfig.Username, authConfig.Password)
|
||||
req.ContentLength = int64(len(imgListJson))
|
||||
req.Header.Set("X-Docker-Token", "true")
|
||||
|
||||
res, err = client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -608,7 +662,11 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
|
|||
}
|
||||
|
||||
if res.StatusCode != 200 && res.StatusCode != 201 {
|
||||
return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote)
|
||||
errBody, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody)
|
||||
}
|
||||
|
||||
var token, endpoints []string
|
||||
|
@ -624,74 +682,41 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
|
|||
return fmt.Errorf("Index response didn't contain any endpoints")
|
||||
}
|
||||
|
||||
// FIXME: Send only needed images
|
||||
for _, registry := range endpoints {
|
||||
fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry,
|
||||
len(localRepo))
|
||||
fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, len(localRepo))
|
||||
// For each image within the repo, push them
|
||||
for tag, imgId := range localRepo {
|
||||
if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, token); err != nil {
|
||||
for _, elem := range imgList {
|
||||
if err := graph.pushPrimitive(stdout, remote, elem.tag, elem.Id, registry, token); err != nil {
|
||||
// FIXME: Continue on error?
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
checksumsJson, err := json.Marshal(checksums)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson))
|
||||
req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(imgListJson))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req2.SetBasicAuth(authConfig.Username, authConfig.Password)
|
||||
req2.Header["X-Docker-Endpoints"] = endpoints
|
||||
req2.ContentLength = int64(len(checksumsJson))
|
||||
req2.ContentLength = int64(len(imgListJson))
|
||||
res2, err := client.Do(req2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res2.Body.Close()
|
||||
defer res2.Body.Close()
|
||||
if res2.StatusCode != 204 {
|
||||
return fmt.Errorf("Error: Status %d trying to push checksums %s", res.StatusCode, remote)
|
||||
if errBody, err := ioutil.ReadAll(res2.Body); err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res2.StatusCode, remote, errBody)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) {
|
||||
checksums := make(map[string]string)
|
||||
for _, id := range repo {
|
||||
img, err := graph.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = img.WalkHistory(func(image *Image) error {
|
||||
fmt.Fprintf(output, "Computing checksum for image %s\n", image.Id)
|
||||
if _, exists := checksums[image.Id]; !exists {
|
||||
checksums[image.Id], err = image.Checksum()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
i := 0
|
||||
result := make([]map[string]string, len(checksums))
|
||||
for id, sum := range checksums {
|
||||
result[i] = map[string]string{
|
||||
"id": id,
|
||||
"checksum": sum,
|
||||
}
|
||||
i++
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type SearchResults struct {
|
||||
Query string `json:"query"`
|
||||
NumResults int `json:"num_results"`
|
||||
|
|
Loading…
Reference in a new issue