From 793fd983ef937d2bea1edf5d8d855e2a452a4aa7 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 02:47:58 -0400 Subject: [PATCH 1/6] http utils --- utils/http.go | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 utils/http.go diff --git a/utils/http.go b/utils/http.go new file mode 100644 index 0000000000..61fdbff04e --- /dev/null +++ b/utils/http.go @@ -0,0 +1,129 @@ +package utils + +import ( + "bytes" + "io" + "net/http" + "strings" +) + +// VersionInfo is used to model entities which has a version. +// It is basically a tupple with name and version. +type VersionInfo interface { + Name() string + Version() string +} + +func validVersion(version VersionInfo) bool { + stopChars := " \t\r\n/" + if strings.ContainsAny(version.Name(), stopChars) { + return false + } + if strings.ContainsAny(version.Version(), stopChars) { + return false + } + return true +} + +// Convert versions to a string and append the string to the string base. +// +// Each VersionInfo will be converted to a string in the format of +// "product/version", where the "product" is get from the Name() method, while +// version is get from the Version() method. Several pieces of verson information +// will be concatinated and separated by space. +func appendVersions(base string, versions ...VersionInfo) string { + if len(versions) == 0 { + return base + } + + var buf bytes.Buffer + if len(base) > 0 { + buf.Write([]byte(base)) + } + + for _, v := range versions { + name := []byte(v.Name()) + version := []byte(v.Version()) + + if len(name) == 0 || len(version) == 0 { + continue + } + if !validVersion(v) { + continue + } + buf.Write([]byte(v.Name())) + buf.Write([]byte("/")) + buf.Write([]byte(v.Version())) + buf.Write([]byte(" ")) + } + return buf.String() +} + +// HTTPRequestDecorator is used to change an instance of +// http.Request. It could be used to add more header fields, +// change body, etc. +type HTTPRequestDecorator interface { + // ChangeRequest() changes the request accordingly. + // The changed request will be returned or err will be non-nil + // if an error occur. + ChangeRequest(req *http.Request) (newReq *http.Request, err error) +} + +// HTTPUserAgentDecorator appends the product/version to the user agent field +// of a request. +type HTTPUserAgentDecorator struct { + versions []VersionInfo +} + +func NewHTTPUserAgentDecorator(versions ...VersionInfo) HTTPRequestDecorator { + ret := new(HTTPUserAgentDecorator) + ret.versions = versions + return ret +} + +func (self *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) { + if req == nil { + return req, nil + } + + userAgent := appendVersions(req.UserAgent(), self.versions...) + if len(userAgent) > 0 { + req.Header.Set("User-Agent", userAgent) + } + return req, nil +} + +// HTTPRequestFactory creates an HTTP request +// and applies a list of decorators on the request. +type HTTPRequestFactory struct { + decorators []HTTPRequestDecorator +} + +func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory { + ret := new(HTTPRequestFactory) + ret.decorators = d + return ret +} + +// NewRequest() creates a new *http.Request, +// applies all decorators in the HTTPRequestFactory on the request, +// then applies decorators provided by d on the request. +func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) { + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + return nil, err + } + for _, dec := range self.decorators { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + for _, dec := range d { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + return req, err +} From 7dac26ce69b442d55122caa2897572d3ac8255fa Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:08:08 -0400 Subject: [PATCH 2/6] reqFactory in Registry --- registry/registry.go | 101 +++++++------------------------------------ 1 file changed, 15 insertions(+), 86 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 4e9dd8895f..da5c83bff1 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -100,13 +100,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } -// VersionInfo is used to model entities which has a version. -// It is basically a tupple with name and version. -type VersionInfo interface { - Name() string - Version() string -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -121,29 +114,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { return res, err } -// Set the user agent field in the header based on the versions provided -// in NewRegistry() and extra. -func (r *Registry) setUserAgent(req *http.Request, extra ...VersionInfo) { - if len(r.baseVersions)+len(extra) == 0 { - return - } - if len(extra) == 0 { - req.Header.Set("User-Agent", r.baseVersionsStr) - } else { - req.Header.Set("User-Agent", appendVersions(r.baseVersionsStr, extra...)) - } - return -} - // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -170,7 +148,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -185,12 +163,11 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -213,12 +190,11 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -239,7 +215,6 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return nil, err @@ -281,7 +256,6 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { @@ -339,7 +313,7 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } @@ -375,13 +349,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { @@ -410,14 +383,13 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr tarsumLayer := &utils.TarSum{Reader: layer} - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) @@ -435,7 +407,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { - req, err := http.NewRequest(method, urlStr, body) + req, err := r.reqFactory.NewRequest(method, urlStr, body) if err != nil { return nil, err } @@ -455,7 +427,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -500,7 +471,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -521,7 +491,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -576,7 +545,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) - req, err := http.NewRequest("GET", u, nil) + req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } @@ -628,52 +597,12 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - baseVersions []VersionInfo - baseVersionsStr string + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory } -func validVersion(version VersionInfo) bool { - stopChars := " \t\r\n/" - if strings.ContainsAny(version.Name(), stopChars) { - return false - } - if strings.ContainsAny(version.Version(), stopChars) { - return false - } - return true -} - -// Convert versions to a string and append the string to the string base. -// -// Each VersionInfo will be converted to a string in the format of -// "product/version", where the "product" is get from the Name() method, while -// version is get from the Version() method. Several pieces of verson information -// will be concatinated and separated by space. -func appendVersions(base string, versions ...VersionInfo) string { - if len(versions) == 0 { - return base - } - - var buf bytes.Buffer - if len(base) > 0 { - buf.Write([]byte(base)) - } - - for _, v := range versions { - if !validVersion(v) { - continue - } - buf.Write([]byte(v.Name())) - buf.Write([]byte("/")) - buf.Write([]byte(v.Version())) - buf.Write([]byte(" ")) - } - return buf.String() -} - -func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionInfo) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -689,7 +618,7 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...Versi if err != nil { return nil, err } - r.baseVersions = baseVersions - r.baseVersionsStr = appendVersions("", baseVersions...) + + r.reqFactory = factory return r, nil } From 6a56b7b391ac967540915c2ee8f82b23714ad84c Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:23:46 -0400 Subject: [PATCH 3/6] Server now use request factory --- server.go | 15 ++++++++++----- utils/http.go | 5 +++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/server.go b/server.go index cb7b2cf1be..b015e459f3 100644 --- a/server.go +++ b/server.go @@ -52,9 +52,9 @@ func (v *simpleVersionInfo) Version() string { // docker, go, git-commit (of the docker) and the host's kernel. // // Such information will be used on call to NewRegistry(). -func (srv *Server) versionInfos() []registry.VersionInfo { +func (srv *Server) versionInfos() []utils.VersionInfo { v := srv.DockerVersion() - ret := make([]registry.VersionInfo, 0, 4) + ret := make([]utils.VersionInfo, 0, 4) ret = append(ret, &simpleVersionInfo{"docker", v.Version}) if len(v.GoVersion) > 0 { @@ -102,7 +102,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - r, err := registry.NewRegistry(srv.runtime.root, nil, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, nil, srv.reqFactory) if err != nil { return nil, err } @@ -559,7 +559,7 @@ func (srv *Server) poolRemove(kind, key string) error { } func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) if err != nil { return err } @@ -720,7 +720,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) if err2 != nil { return err2 } @@ -1164,7 +1164,11 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( pushingPool: make(map[string]struct{}), events: make([]utils.JSONMessage, 0, 64), //only keeps the 64 last events listeners: make(map[string]chan utils.JSONMessage), + reqFactory: nil, } + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + factory := utils.NewHTTPRequestFactory(ud) + srv.reqFactory = factory runtime.srv = srv return srv, nil } @@ -1189,4 +1193,5 @@ type Server struct { pushingPool map[string]struct{} events []utils.JSONMessage listeners map[string]chan utils.JSONMessage + reqFactory *utils.HTTPRequestFactory } diff --git a/utils/http.go b/utils/http.go index 61fdbff04e..8c1e4b7a79 100644 --- a/utils/http.go +++ b/utils/http.go @@ -113,6 +113,11 @@ func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader if err != nil { return nil, err } + + // By default, a nil factory should work. + if self == nil { + return req, nil + } for _, dec := range self.decorators { req, err = dec.ChangeRequest(req) if err != nil { From 4bd287e107eab1623a0e77aeaecda77fc26e7536 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 03:30:45 -0400 Subject: [PATCH 4/6] auth with user agent --- api.go | 2 +- auth/auth.go | 5 +++-- server.go | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index b8b7897c32..77faaaf23c 100644 --- a/api.go +++ b/api.go @@ -87,7 +87,7 @@ func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Reque if err != nil { return err } - status, err := auth.Login(authConfig) + status, err := auth.Login(authConfig, srv.HTTPRequestFactory()) if err != nil { return err } diff --git a/auth/auth.go b/auth/auth.go index 6dd6ceb620..b9e1ee153b 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker/utils" "io/ioutil" "net/http" "os" @@ -140,7 +141,7 @@ func SaveConfig(configFile *ConfigFile) error { } // try to register/login to the registry server -func Login(authConfig *AuthConfig) (string, error) { +func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { client := &http.Client{} reqStatusCode := 0 var status string @@ -171,7 +172,7 @@ func Login(authConfig *AuthConfig) (string, error) { "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil) + req, err := factory.NewRequest("GET", IndexServerAddress()+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/server.go b/server.go index b015e459f3..fa6f19ad4b 100644 --- a/server.go +++ b/server.go @@ -1173,6 +1173,15 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( return srv, nil } +func (srv *Server) HTTPRequestFactory() *utils.HTTPRequestFactory { + if srv.reqFactory == nil { + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + factory := utils.NewHTTPRequestFactory(ud) + srv.reqFactory = factory + } + return srv.reqFactory +} + func (srv *Server) LogEvent(action, id string) { now := time.Now().Unix() jm := utils.JSONMessage{Status: action, ID: id, Time: now} From 5bc344ab73f73a1fb5bea0933a031ad0418cb8f8 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 04:10:26 -0400 Subject: [PATCH 5/6] factory generated from one place. --- server.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server.go b/server.go index fa6f19ad4b..d77bcdb59e 100644 --- a/server.go +++ b/server.go @@ -102,7 +102,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - r, err := registry.NewRegistry(srv.runtime.root, nil, srv.reqFactory) + r, err := registry.NewRegistry(srv.runtime.root, nil, srv.HTTPRequestFactory()) if err != nil { return nil, err } @@ -559,7 +559,7 @@ func (srv *Server) poolRemove(kind, key string) error { } func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) + r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err != nil { return err } @@ -720,7 +720,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.reqFactory) + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err2 != nil { return err2 } @@ -1166,9 +1166,6 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( listeners: make(map[string]chan utils.JSONMessage), reqFactory: nil, } - ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) - factory := utils.NewHTTPRequestFactory(ud) - srv.reqFactory = factory runtime.srv = srv return srv, nil } From 7bade49d4c661c5037de586e6f69291999038ef9 Mon Sep 17 00:00:00 2001 From: Nan Monnand Deng Date: Fri, 2 Aug 2013 14:08:16 -0400 Subject: [PATCH 6/6] update auth_test.go --- auth/auth_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth/auth_test.go b/auth/auth_test.go index d94d429da1..24a0666cf7 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -33,7 +33,7 @@ func TestLogin(t *testing.T) { os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") defer os.Setenv("DOCKER_INDEX_URL", "") authConfig := &AuthConfig{Username: "unittester", Password: "surlautrerivejetattendrai", Email: "noise+unittester@dotcloud.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func TestCreateAccount(t *testing.T) { token := hex.EncodeToString(tokenBuffer)[:12] username := "ut" + token authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestCreateAccount(t *testing.T) { t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status) } - status, err = Login(authConfig) + status, err = Login(authConfig, nil) if err == nil { t.Fatalf("Expected error but found nil instead") }