From cd0de83917e775fcdcd3ce4e592b627b7eeeac64 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 28 May 2013 17:12:24 -0700 Subject: [PATCH 1/2] Cereate a new registry object for each request (~session) --- api.go | 15 ++++++++--- registry/registry.go | 3 +++ server.go | 61 ++++++++++++++++++++++---------------------- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/api.go b/api.go index 941e87e6f0..82fb4b4ad3 100644 --- a/api.go +++ b/api.go @@ -60,7 +60,12 @@ func getBoolParam(value string) (bool, error) { } func getAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - b, err := json.Marshal(srv.registry.GetAuthConfig(false)) + // FIXME: Handle multiple login at once + authConfig, err := auth.LoadConfig(srv.runtime.root) + if err != nil { + return err + } + b, err := json.Marshal(&auth.AuthConfig{Username: authConfig.Username, Email: authConfig.Email}) if err != nil { return err } @@ -69,11 +74,16 @@ func getAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reques } func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + // FIXME: Handle multiple login at once config := &auth.AuthConfig{} if err := json.NewDecoder(r.Body).Decode(config); err != nil { return err } - authConfig := srv.registry.GetAuthConfig(true) + + authConfig, err := auth.LoadConfig(srv.runtime.root) + if err != nil { + return err + } if config.Username == authConfig.Username { config.Password = authConfig.Password } @@ -83,7 +93,6 @@ func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reque if err != nil { return err } - srv.registry.ResetClient(newAuthConfig) if status != "" { b, err := json.Marshal(&ApiAuth{Status: status}) diff --git a/registry/registry.go b/registry/registry.go index bd361b5e74..36b01d643a 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -330,6 +330,9 @@ func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validat if validate { suffix = "images" } + + utils.Debugf("Image list pushed to index:\n%s\n", imgListJson) + req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJson)) if err != nil { return nil, err diff --git a/server.go b/server.go index 3303c7c5a1..7c78db2c90 100644 --- a/server.go +++ b/server.go @@ -49,7 +49,8 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]ApiSearch, error) { - results, err := srv.registry.SearchRepositories(term) + + results, err := registry.NewRegistry(srv.runtime.root).SearchRepositories(term) if err != nil { return nil, err } @@ -291,8 +292,8 @@ func (srv *Server) ContainerTag(name, repo, tag string, force bool) error { return nil } -func (srv *Server) pullImage(out io.Writer, imgId, registry string, token []string, json bool) error { - history, err := srv.registry.GetRemoteHistory(imgId, registry, token) +func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgId, endpoint string, token []string, json bool) error { + history, err := r.GetRemoteHistory(imgId, endpoint, token) if err != nil { return err } @@ -302,7 +303,7 @@ func (srv *Server) pullImage(out io.Writer, imgId, registry string, token []stri for _, id := range history { if !srv.runtime.graph.Exists(id) { fmt.Fprintf(out, utils.FormatStatus("Pulling %s metadata", json), id) - imgJson, err := srv.registry.GetRemoteImageJson(id, registry, token) + imgJson, err := r.GetRemoteImageJson(id, endpoint, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -314,7 +315,7 @@ func (srv *Server) pullImage(out io.Writer, imgId, registry string, token []stri // Get the layer fmt.Fprintf(out, utils.FormatStatus("Pulling %s fs layer", json), id) - layer, contentLength, err := srv.registry.GetRemoteImageLayer(img.Id, registry, token) + layer, contentLength, err := r.GetRemoteImageLayer(img.Id, endpoint, token) if err != nil { return err } @@ -326,9 +327,9 @@ func (srv *Server) pullImage(out io.Writer, imgId, registry string, token []stri return nil } -func (srv *Server) pullRepository(out io.Writer, remote, askedTag string, json bool) error { +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, remote, askedTag string, json bool) error { fmt.Fprintf(out, utils.FormatStatus("Pulling repository %s from %s", json), remote, auth.IndexServerAddress()) - repoData, err := srv.registry.GetRepositoryData(remote) + repoData, err := r.GetRepositoryData(remote) if err != nil { return err } @@ -340,7 +341,7 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string, json b } utils.Debugf("Retrieving the tag list") - tagsList, err := srv.registry.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) + tagsList, err := r.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) if err != nil { return err } @@ -367,7 +368,7 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string, json b fmt.Fprintf(out, utils.FormatStatus("Pulling image %s (%s) from %s", json), img.Id, img.Tag, remote) success := false for _, ep := range repoData.Endpoints { - if err := srv.pullImage(out, img.Id, "https://"+ep+"/v1", repoData.Tokens, json); err != nil { + if err := srv.pullImage(r, out, img.Id, "https://"+ep+"/v1", repoData.Tokens, json); err != nil { fmt.Fprintf(out, utils.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint\n", json), askedTag, err) continue } @@ -393,16 +394,17 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string, json b return nil } -func (srv *Server) ImagePull(name, tag, registry string, out io.Writer, json bool) error { +func (srv *Server) ImagePull(name, tag, endpoint string, out io.Writer, json bool) error { + r := registry.NewRegistry(srv.runtime.root) out = utils.NewWriteFlusher(out) - if registry != "" { - if err := srv.pullImage(out, name, registry, nil, json); err != nil { + if endpoint != "" { + if err := srv.pullImage(r, out, name, endpoint, nil, json); err != nil { return err } return nil } - if err := srv.pullRepository(out, name, tag, json); err != nil { + if err := srv.pullRepository(r, out, name, tag, json); err != nil { return err } @@ -475,21 +477,20 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return imgList, nil } -func (srv *Server) pushRepository(out io.Writer, name string, localRepo map[string]string) error { +func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, name string, localRepo map[string]string) error { out = utils.NewWriteFlusher(out) fmt.Fprintf(out, "Processing checksums\n") imgList, err := srv.getImageList(localRepo) if err != nil { return err } - fmt.Fprintf(out, "Sending image list\n") + fmt.Fprintf(out, "Sending images list\n") - repoData, err := srv.registry.PushImageJsonIndex(name, imgList, false) + repoData, err := r.PushImageJsonIndex(name, imgList, false) if err != nil { return err } - // FIXME: Send only needed images for _, ep := range repoData.Endpoints { fmt.Fprintf(out, "Pushing repository %s to %s (%d tags)\r\n", name, ep, len(localRepo)) // For each image within the repo, push them @@ -498,24 +499,24 @@ func (srv *Server) pushRepository(out io.Writer, name string, localRepo map[stri fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) continue } - if err := srv.pushImage(out, name, elem.Id, ep, repoData.Tokens); err != nil { + if err := srv.pushImage(r, out, name, elem.Id, ep, repoData.Tokens); err != nil { // FIXME: Continue on error? return err } fmt.Fprintf(out, "Pushing tags for rev [%s] on {%s}\n", elem.Id, ep+"/users/"+name+"/"+elem.Tag) - if err := srv.registry.PushRegistryTag(name, elem.Id, elem.Tag, ep, repoData.Tokens); err != nil { + if err := r.PushRegistryTag(name, elem.Id, elem.Tag, ep, repoData.Tokens); err != nil { return err } } } - if _, err := srv.registry.PushImageJsonIndex(name, imgList, true); err != nil { + if _, err := r.PushImageJsonIndex(name, imgList, true); err != nil { return err } return nil } -func (srv *Server) pushImage(out io.Writer, remote, imgId, ep string, token []string) error { +func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgId, ep string, token []string) error { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json")) if err != nil { @@ -534,7 +535,7 @@ func (srv *Server) pushImage(out io.Writer, remote, imgId, ep string, token []st } // Send the json - if err := srv.registry.PushImageJsonRegistry(imgData, jsonRaw, ep, token); err != nil { + if err := r.PushImageJsonRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { fmt.Fprintf(out, "Image %s already uploaded ; skipping\n", imgData.Id) return nil @@ -569,20 +570,22 @@ func (srv *Server) pushImage(out io.Writer, remote, imgId, ep string, token []st } // Send the layer - if err := srv.registry.PushImageLayerRegistry(imgData.Id, utils.ProgressReader(layerData, int(layerData.Size), out, "", false), ep, token); err != nil { + if err := r.PushImageLayerRegistry(imgData.Id, utils.ProgressReader(layerData, int(layerData.Size), out, "", false), ep, token); err != nil { return err } return nil } -func (srv *Server) ImagePush(name, registry string, out io.Writer) error { +func (srv *Server) ImagePush(name, endpoint string, out io.Writer) error { out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(name) + r := registry.NewRegistry(srv.runtime.root) + if err != nil { fmt.Fprintf(out, "The push refers to a repository [%s] (len: %d)\n", name, len(srv.runtime.repositories.Repositories[name])) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.pushRepository(out, name, localRepo); err != nil { + if err := srv.pushRepository(r, out, name, localRepo); err != nil { return err } return nil @@ -591,7 +594,7 @@ func (srv *Server) ImagePush(name, registry string, out io.Writer) error { return err } fmt.Fprintf(out, "The push refers to an image: [%s]\n", name) - if err := srv.pushImage(out, name, img.Id, registry, nil); err != nil { + if err := srv.pushImage(r, out, name, img.Id, endpoint, nil); err != nil { return err } return nil @@ -871,14 +874,12 @@ func NewServer(autoRestart bool) (*Server, error) { return nil, err } srv := &Server{ - runtime: runtime, - registry: registry.NewRegistry(runtime.root), + runtime: runtime, } runtime.srv = srv return srv, nil } type Server struct { - runtime *Runtime - registry *registry.Registry + runtime *Runtime } From b76d6120ac66a853d177f7e53318ab1ee75dca92 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 28 May 2013 17:35:10 -0700 Subject: [PATCH 2/2] Update tests with new cookies for registry --- api.go | 11 +++++++++-- api_test.go | 19 +++++++++---------- auth/auth.go | 19 +++++++++++++++---- runtime_test.go | 4 +--- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/api.go b/api.go index 82fb4b4ad3..f044fe9c96 100644 --- a/api.go +++ b/api.go @@ -61,9 +61,13 @@ func getBoolParam(value string) (bool, error) { func getAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { // FIXME: Handle multiple login at once + // FIXME: return specific error code if config file missing? authConfig, err := auth.LoadConfig(srv.runtime.root) if err != nil { - return err + if err != auth.ErrConfigFileMissing { + return err + } + authConfig = &auth.AuthConfig{} } b, err := json.Marshal(&auth.AuthConfig{Username: authConfig.Username, Email: authConfig.Email}) if err != nil { @@ -82,7 +86,10 @@ func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reque authConfig, err := auth.LoadConfig(srv.runtime.root) if err != nil { - return err + if err != auth.ErrConfigFileMissing { + return err + } + authConfig = &auth.AuthConfig{} } if config.Username == authConfig.Username { config.Password = authConfig.Password diff --git a/api_test.go b/api_test.go index f94ba23fa0..f364c6c895 100644 --- a/api_test.go +++ b/api_test.go @@ -26,8 +26,7 @@ func TestGetAuth(t *testing.T) { defer nuke(runtime) srv := &Server{ - runtime: runtime, - registry: registry.NewRegistry(runtime.root), + runtime: runtime, } r := httptest.NewRecorder() @@ -56,7 +55,7 @@ func TestGetAuth(t *testing.T) { t.Fatalf("%d OK or 0 expected, received %d\n", http.StatusOK, r.Code) } - newAuthConfig := srv.registry.GetAuthConfig(false) + newAuthConfig := registry.NewRegistry(runtime.root).GetAuthConfig(false) if newAuthConfig.Username != authConfig.Username || newAuthConfig.Email != authConfig.Email { t.Fatalf("The auth configuration hasn't been set correctly") @@ -247,8 +246,7 @@ func TestGetImagesSearch(t *testing.T) { defer nuke(runtime) srv := &Server{ - runtime: runtime, - registry: registry.NewRegistry(runtime.root), + runtime: runtime, } r := httptest.NewRecorder() @@ -504,15 +502,16 @@ func TestPostAuth(t *testing.T) { defer nuke(runtime) srv := &Server{ - runtime: runtime, - registry: registry.NewRegistry(runtime.root), + runtime: runtime, } - authConfigOrig := &auth.AuthConfig{ + config := &auth.AuthConfig{ Username: "utest", Email: "utest@yopmail.com", } - srv.registry.ResetClient(authConfigOrig) + + authStr := auth.EncodeAuth(config) + auth.SaveConfig(runtime.root, authStr, config.Email) r := httptest.NewRecorder() if err := getAuth(srv, API_VERSION, r, nil, nil); err != nil { @@ -524,7 +523,7 @@ func TestPostAuth(t *testing.T) { t.Fatal(err) } - if authConfig.Username != authConfigOrig.Username || authConfig.Email != authConfigOrig.Email { + if authConfig.Username != config.Username || authConfig.Email != config.Email { t.Errorf("The retrieve auth mismatch with the one set.") } } diff --git a/auth/auth.go b/auth/auth.go index 2b99c95038..9c34604419 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "encoding/base64" "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -17,6 +18,12 @@ const CONFIGFILE = ".dockercfg" // the registry server we want to login against const INDEX_SERVER = "https://index.docker.io/v1" +//const INDEX_SERVER = "http://indexstaging-docker.dotcloud.com/" + +var ( + ErrConfigFileMissing error = errors.New("The Auth config file is missing") +) + type AuthConfig struct { Username string `json:"username"` Password string `json:"password"` @@ -75,7 +82,7 @@ func DecodeAuth(authStr string) (*AuthConfig, error) { func LoadConfig(rootPath string) (*AuthConfig, error) { confFile := path.Join(rootPath, CONFIGFILE) if _, err := os.Stat(confFile); err != nil { - return &AuthConfig{}, fmt.Errorf("The Auth config file is missing") + return nil, ErrConfigFileMissing } b, err := ioutil.ReadFile(confFile) if err != nil { @@ -97,7 +104,7 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { } // save the auth config -func saveConfig(rootPath, authStr string, email string) error { +func SaveConfig(rootPath, authStr string, email string) error { confFile := path.Join(rootPath, CONFIGFILE) if len(email) == 0 { os.Remove(confFile) @@ -161,7 +168,9 @@ func Login(authConfig *AuthConfig) (string, error) { status = "Login Succeeded\n" storeConfig = true } else if resp.StatusCode == 401 { - saveConfig(authConfig.rootPath, "", "") + if err := SaveConfig(authConfig.rootPath, "", ""); err != nil { + return "", err + } return "", fmt.Errorf("Wrong login/password, please try again") } else { return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, @@ -175,7 +184,9 @@ func Login(authConfig *AuthConfig) (string, error) { } if storeConfig { authStr := EncodeAuth(authConfig) - saveConfig(authConfig.rootPath, authStr, authConfig.Email) + if err := SaveConfig(authConfig.rootPath, authStr, authConfig.Email); err != nil { + return "", err + } } return status, nil } diff --git a/runtime_test.go b/runtime_test.go index 6c4ec5ded4..8f62fa80fc 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -2,7 +2,6 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -71,8 +70,7 @@ func init() { // Create the "Server" srv := &Server{ - runtime: runtime, - registry: registry.NewRegistry(runtime.root), + runtime: runtime, } // Retrieve the Image if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, false); err != nil {