kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

Merge pull request #1290 from dotcloud/parallel_pull

Browse source 
* Runtime: Parallel pull
This commit is contained in:
Guillaume J. Charmes 2013-08-05 14:17:29 -07:00
commit e54e8fa920
8 changed files with 132 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api.go
View file

@ -386,7 +386,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht
}
sf := utils.NewStreamFormatter(version > 1.0)
if image != "" { //pull
if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil {
if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}, version > 1.3); err != nil {
if sf.Used() {
w.Write(sf.FormatError(err))
return nil

2
buildfile.go
View file

@ -56,7 +56,7 @@ func (b *buildFile) CmdFrom(name string) error {
if err != nil {
if b.runtime.graph.IsNotExist(err) {
remote, tag := utils.ParseRepositoryTag(name)
if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil {
if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil, true); err != nil {
return err
}
image, err = b.runtime.repositories.LookupImage(name)

21
commands.go
View file

@ -30,8 +30,7 @@ import (
const VERSION = "0.5.0-dev"
var (
GITCOMMIT string
AuthRequiredError = fmt.Errorf("Authentication is required.")
GITCOMMIT string
)
func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) {
@ -197,7 +196,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
// FIXME: ProgressReader shouldn't be this annoyning to use
if context != nil {
sf := utils.NewStreamFormatter(false)
body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf)
body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf)
}
// Upload the build context
v := &url.Values{}
@ -846,7 +845,7 @@ func (cli *DockerCli) CmdPush(args ...string) error {
}
if err := push(); err != nil {
if err == AuthRequiredError {
if err == fmt.Errorf("Authentication is required.") {
if err = cli.checkIfLogged("push"); err == nil {
return push()
}
@ -1569,19 +1568,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e
}
if resp.Header.Get("Content-Type") == "application/json" {
dec := json.NewDecoder(resp.Body)
for {
var jm utils.JSONMessage
if err := dec.Decode(&jm); err == io.EOF {
break
} else if err != nil {
return err
}
if jm.Error != nil && jm.Error.Code == 401 {
return AuthRequiredError
}
jm.Display(out)
}
return utils.DisplayJSONMessagesStream(resp.Body, out)
} else {
if _, err := io.Copy(out, resp.Body); err != nil {
return err

4
docs/sources/api/docker_remote_api.rst
View file

@ -40,6 +40,10 @@ You can still call an old version of the api using
What's new
----------
.. http:post:: /images/create
**New!** When pull a repo, all images are now downloaded in parallel.
.. http:get:: /containers/(id)/top
**New!** You can now use ps args with docker top, like `docker top <container_id> aux`

2
graph.go
View file

@ -159,7 +159,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti
if err != nil {
return nil, err
}
return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)"), sf), tmp.Root)
return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf), tmp.Root)
}
// Mktemp creates a temporary sub-directory inside the graph's filesystem.

2
runtime_test.go
View file

@ -101,7 +101,7 @@ func init() {
// If the unit test is not found, try to download it.
if img, err := globalRuntime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
// Retrieve the Image
if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, true); err != nil {
panic(err)
}
}

106
server.go
View file

@ -145,7 +145,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.
return "", err
}
if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), path); err != nil {
if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf), path); err != nil {
return "", err
}
// FIXME: Handle custom repo, tag comment, author
@ -153,7 +153,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.
if err != nil {
return "", err
}
out.Write(sf.FormatStatus(img.ID))
out.Write(sf.FormatStatus("", img.ID))
return img.ShortID(), nil
}
@ -420,7 +420,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin
// FIXME: Launch the getRemoteImage() in goroutines
for _, id := range history {
if !srv.runtime.graph.Exists(id) {
out.Write(sf.FormatStatus("Pulling %s metadata", id))
out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling metadata"))
imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
if err != nil {
// FIXME: Keep goging in case of error?
@ -432,13 +432,13 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin
}
// Get the layer
out.Write(sf.FormatStatus("Pulling %s fs layer", id))
out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling fs layer"))
layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
if err != nil {
return err
}
defer layer.Close()
if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), img); err != nil {
if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf), img); err != nil {
return err
}
}
@ -446,8 +446,8 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin
return nil
}
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error {
out.Write(sf.FormatStatus("Pulling repository %s", localName))
func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error {
out.Write(sf.FormatStatus("", "Pulling repository %s", localName))
repoData, err := r.GetRepositoryData(indexEp, remoteName)
if err != nil {
@ -484,30 +484,51 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName
repoData.ImgList[id].Tag = askedTag
}
for _, img := range repoData.ImgList {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
continue
errors := make(chan error)
for _, image := range repoData.ImgList {
downloadImage := func(img *registry.ImgData) {
if askedTag != "" && img.Tag != askedTag {
utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID)
errors <- nil
return
}
if img.Tag == "" {
utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
errors <- nil
return
}
out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, localName))
success := false
for _, ep := range repoData.Endpoints {
if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
success = true
break
}
if !success {
errors <- fmt.Errorf("Could not find repository on any of the indexed registries.")
}
errors <- nil
}
if img.Tag == "" {
utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
continue
}
out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName))
success := false
for _, ep := range repoData.Endpoints {
if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err))
continue
}
success = true
break
}
if !success {
return fmt.Errorf("Could not find repository on any of the indexed registries.")
if parallel {
go downloadImage(image)
} else {
downloadImage(image)
}
}
if parallel {
for i := 0; i < len(repoData.ImgList); i++ {
if err := <-errors; err != nil {
return err
}
}
}
for tag, id := range tagsList {
if askedTag != "" && tag != askedTag {
continue
@ -558,7 +579,7 @@ func (srv *Server) poolRemove(kind, key string) error {
return nil
}
func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error {
func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, parallel bool) error {
r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory())
if err != nil {
return err
@ -580,7 +601,7 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut
}
out = utils.NewWriteFlusher(out)
err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf)
err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel)
if err != nil {
if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil {
return err
@ -620,12 +641,11 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat
func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
imgList, err := srv.getImageList(localRepo)
if err != nil {
return err
}
out.Write(sf.FormatStatus("Sending image list"))
out.Write(sf.FormatStatus("", "Sending image list"))
var repoData *registry.RepositoryData
repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil)
@ -634,14 +654,14 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName
}
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo)))
out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo)))
// For each image within the repo, push them
for _, elem := range imgList {
if _, exists := repoData.ImgList[elem.ID]; exists {
out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
continue
} else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) {
out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID))
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
continue
}
if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
@ -650,7 +670,7 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName
} else {
elem.Checksum = checksum
}
out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag))
if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil {
return err
}
@ -670,7 +690,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID,
if err != nil {
return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err)
}
out.Write(sf.FormatStatus("Pushing %s", imgID))
out.Write(sf.FormatStatus("", "Pushing %s", imgID))
imgData := &registry.ImgData{
ID: imgID,
@ -679,7 +699,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID,
// Send the json
if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
if err == registry.ErrAlreadyExists {
out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID))
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID))
return "", nil
}
return "", err
@ -691,7 +711,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID,
}
// Send the layer
if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil {
if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil {
return "", err
} else {
imgData.Checksum = checksum
@ -727,7 +747,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo
if err != nil {
reposLen := len(srv.runtime.repositories.Repositories[localName])
out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen))
out.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen))
// If it fails, try to get the repository
if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists {
if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil {
@ -739,7 +759,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo
}
var token []string
out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName))
out.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName))
if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil {
return err
}
@ -762,14 +782,14 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write
u.Host = src
u.Path = ""
}
out.Write(sf.FormatStatus("Downloading from %s", u))
out.Write(sf.FormatStatus("", "Downloading from %s", u))
// Download with curl (pretty progress bar)
// If curl is not available, fallback to http.Get()
resp, err = utils.Download(u.String(), out)
if err != nil {
return err
}
archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%8v/%v (%v)"), sf)
archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf)
}
img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil)
if err != nil {
@ -781,7 +801,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write
return err
}
}
out.Write(sf.FormatStatus(img.ShortID()))
out.Write(sf.FormatStatus("", img.ShortID()))
return nil
}

73
utils/utils.go
View file

@ -94,11 +94,6 @@ func (r *progressReader) Read(p []byte) (n int, err error) {
}
r.lastUpdate = r.readProgress
}
// Send newline when complete
if err != nil {
r.output.Write(r.sf.FormatStatus(""))
}
return read, err
}
func (r *progressReader) Close() error {
@ -107,7 +102,7 @@ func (r *progressReader) Close() error {
func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader {
tpl := string(template)
if tpl == "" {
tpl = string(sf.FormatProgress("", "%8v/%v (%v)"))
tpl = string(sf.FormatProgress("", "%8v/%v (%v)", ""))
}
return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf}
}
@ -587,11 +582,14 @@ type NopFlusher struct{}
func (f *NopFlusher) Flush() {}
type WriteFlusher struct {
sync.Mutex
w io.Writer
flusher http.Flusher
}
func (wf *WriteFlusher) Write(b []byte) (n int, err error) {
wf.Lock()
defer wf.Unlock()
n, err = wf.w.Write(b)
wf.flusher.Flush()
return n, err
@ -633,17 +631,60 @@ func NewHTTPRequestError(msg string, res *http.Response) error {
}
func (jm *JSONMessage) Display(out io.Writer) error {
if jm.Error != nil {
if jm.Error.Code == 401 {
return fmt.Errorf("Authentication is required.")
}
return jm.Error
}
fmt.Fprintf(out, "%c[2K", 27)
if jm.Time != 0 {
fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0))
}
if jm.ID != "" {
fmt.Fprintf(out, "%s: ", jm.ID)
}
if jm.Progress != "" {
fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress)
} else if jm.Error != nil {
return jm.Error
} else if jm.ID != "" {
fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status)
} else {
fmt.Fprintf(out, "%s\n", jm.Status)
fmt.Fprintf(out, "%s\r", jm.Status)
}
if jm.ID == "" {
fmt.Fprintf(out, "\n")
}
return nil
}
func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error {
dec := json.NewDecoder(in)
jm := JSONMessage{}
ids := make(map[string]int)
diff := 0
for {
if err := dec.Decode(&jm); err == io.EOF {
break
} else if err != nil {
return err
}
if jm.ID != "" {
line, ok := ids[jm.ID]
if !ok {
line = len(ids)
ids[jm.ID] = line
fmt.Fprintf(out, "\n")
diff = 0
} else {
diff = len(ids) - line
}
fmt.Fprintf(out, "%c[%dA", 27, diff)
}
err := jm.Display(out)
if jm.ID != "" {
fmt.Fprintf(out, "%c[%dB", 27, diff)
}
if err != nil {
return err
}
}
return nil
}
@ -657,11 +698,11 @@ func NewStreamFormatter(json bool) *StreamFormatter {
return &StreamFormatter{json, false}
}
func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte {
func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte {
sf.used = true
str := fmt.Sprintf(format, a...)
if sf.json {
b, err := json.Marshal(&JSONMessage{Status: str})
b, err := json.Marshal(&JSONMessage{ID: id, Status: str})
if err != nil {
return sf.FormatError(err)
}
@ -685,16 +726,16 @@ func (sf *StreamFormatter) FormatError(err error) []byte {
return []byte("Error: " + err.Error() + "\r\n")
}
func (sf *StreamFormatter) FormatProgress(action, str string) []byte {
func (sf *StreamFormatter) FormatProgress(id, action, progress string) []byte {
sf.used = true
if sf.json {
b, err := json.Marshal(&JSONMessage{Status: action, Progress: str})
b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID: id})
if err != nil {
return nil
}
return b
}
return []byte(action + " " + str + "\r")
return []byte(action + " " + progress + "\r")
}
func (sf *StreamFormatter) Used() bool {