2014-08-08 02:58:58 -04:00
package graph
import (
"fmt"
"io"
2014-10-01 21:26:06 -04:00
"io/ioutil"
2014-08-08 02:58:58 -04:00
"net"
"net/url"
2014-10-01 21:26:06 -04:00
"os"
2014-08-08 02:58:58 -04:00
"strings"
"time"
2015-03-26 18:22:04 -04:00
"github.com/Sirupsen/logrus"
2015-03-11 11:17:48 -04:00
"github.com/docker/distribution/digest"
2015-04-22 08:06:58 -04:00
"github.com/docker/docker/cliconfig"
2014-08-08 02:58:58 -04:00
"github.com/docker/docker/image"
2015-02-24 03:51:46 -05:00
"github.com/docker/docker/pkg/progressreader"
2015-03-17 22:18:41 -04:00
"github.com/docker/docker/pkg/streamformatter"
2015-03-24 07:25:26 -04:00
"github.com/docker/docker/pkg/stringid"
2015-05-15 21:35:04 -04:00
"github.com/docker/docker/pkg/transport"
2014-08-08 02:58:58 -04:00
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
2015-04-15 07:43:15 -04:00
type ImagePullConfig struct {
MetaHeaders map [ string ] [ ] string
2015-04-22 08:06:58 -04:00
AuthConfig * cliconfig . AuthConfig
2015-04-15 07:43:15 -04:00
OutStream io . Writer
}
2015-04-21 17:23:48 -04:00
func ( s * TagStore ) Pull ( image string , tag string , imagePullConfig * ImagePullConfig ) error {
2014-08-08 02:58:58 -04:00
var (
2015-05-12 14:18:54 -04:00
sf = streamformatter . NewJSONStreamFormatter ( )
2014-08-08 02:58:58 -04:00
)
2014-10-03 18:24:14 -04:00
2014-10-06 21:54:52 -04:00
// Resolve the Repository name from fqn to RepositoryInfo
2015-04-15 07:43:15 -04:00
repoInfo , err := s . registryService . ResolveRepository ( image )
2014-10-06 21:54:52 -04:00
if err != nil {
2015-03-25 03:44:12 -04:00
return err
2014-10-06 21:54:52 -04:00
}
2015-04-22 17:41:24 -04:00
if err := validateRepoName ( repoInfo . LocalName ) ; err != nil {
return err
}
2015-02-26 21:23:50 -05:00
c , err := s . poolAdd ( "pull" , utils . ImageReference ( repoInfo . LocalName , tag ) )
2014-08-08 02:58:58 -04:00
if err != nil {
if c != nil {
// Another pull of the same repository is already taking place; just wait for it to finish
2015-04-15 07:43:15 -04:00
imagePullConfig . OutStream . Write ( sf . FormatStatus ( "" , "Repository %s already being pulled by another client. Waiting." , repoInfo . LocalName ) )
2014-08-08 02:58:58 -04:00
<- c
2015-03-25 03:44:12 -04:00
return nil
2014-08-08 02:58:58 -04:00
}
2015-03-25 03:44:12 -04:00
return err
2014-08-08 02:58:58 -04:00
}
2015-02-26 21:23:50 -05:00
defer s . poolRemove ( "pull" , utils . ImageReference ( repoInfo . LocalName , tag ) )
2014-08-08 02:58:58 -04:00
2015-05-15 20:48:20 -04:00
logName := repoInfo . LocalName
if tag != "" {
logName = utils . ImageReference ( logName , tag )
}
2015-05-22 21:20:54 -04:00
// Attempt pulling official content from a provided v2 mirror
if repoInfo . Index . Official {
v2mirrorEndpoint , v2mirrorRepoInfo , err := configureV2Mirror ( repoInfo , s . registryService )
if err != nil {
logrus . Errorf ( "Error configuring mirrors: %s" , err )
return err
}
2015-05-15 20:48:20 -04:00
2015-05-22 21:20:54 -04:00
if v2mirrorEndpoint != nil {
2015-05-27 18:52:51 -04:00
logrus . Debugf ( "Attempting to pull from v2 mirror: %s" , v2mirrorEndpoint . URL )
2015-05-22 21:20:54 -04:00
return s . pullFromV2Mirror ( v2mirrorEndpoint , v2mirrorRepoInfo , imagePullConfig , tag , sf , logName )
}
2015-05-15 20:48:20 -04:00
}
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "pulling image from host %q with remote name %q" , repoInfo . Index . Name , repoInfo . RemoteName )
2015-05-15 21:35:04 -04:00
endpoint , err := repoInfo . GetEndpoint ( imagePullConfig . MetaHeaders )
2014-08-08 02:58:58 -04:00
if err != nil {
2015-03-25 03:44:12 -04:00
return err
2014-08-08 02:58:58 -04:00
}
2015-05-15 21:35:04 -04:00
// TODO(tiborvass): reuse client from endpoint?
2015-05-14 10:12:54 -04:00
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
2015-05-15 21:35:04 -04:00
tr := transport . NewTransport (
2015-05-14 10:12:54 -04:00
registry . NewTransport ( registry . ReceiveTimeout , endpoint . IsSecure ) ,
2015-05-15 21:35:04 -04:00
registry . DockerHeaders ( imagePullConfig . MetaHeaders ) ... ,
)
2015-05-14 10:12:54 -04:00
client := registry . HTTPClient ( tr )
r , err := registry . NewSession ( client , imagePullConfig . AuthConfig , endpoint )
2014-08-08 02:58:58 -04:00
if err != nil {
2015-03-25 03:44:12 -04:00
return err
2014-08-08 02:58:58 -04:00
}
2015-03-03 13:59:50 -05:00
if len ( repoInfo . Index . Mirrors ) == 0 && ( repoInfo . Index . Official || endpoint . Version == registry . APIVersion2 ) {
2015-02-10 19:08:57 -05:00
if repoInfo . Official {
2015-04-20 15:48:33 -04:00
s . trustService . UpdateBase ( )
2014-10-07 14:00:17 -04:00
}
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "pulling v2 repository with local name %q" , repoInfo . LocalName )
2015-05-11 17:53:52 -04:00
if err := s . pullV2Repository ( r , imagePullConfig . OutStream , repoInfo , tag , sf ) ; err == nil {
2015-04-03 18:17:49 -04:00
s . eventsService . Log ( "pull" , logName , "" )
2015-03-25 03:44:12 -04:00
return nil
2015-02-02 18:54:14 -05:00
} else if err != registry . ErrDoesNotExist && err != ErrV2RegistryUnavailable {
2015-03-26 18:22:04 -04:00
logrus . Errorf ( "Error from V2 registry: %s" , err )
2014-10-07 14:00:17 -04:00
}
2014-12-23 16:40:06 -05:00
2015-03-26 18:22:04 -04:00
logrus . Debug ( "image does not exist on v2 registry, falling back to v1" )
2014-10-07 14:00:17 -04:00
}
2015-04-29 15:29:50 -04:00
if utils . DigestReference ( tag ) {
return fmt . Errorf ( "pulling with digest reference failed from v2 registry" )
}
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "pulling v1 repository with local name %q" , repoInfo . LocalName )
2015-05-11 17:53:52 -04:00
if err = s . pullRepository ( r , imagePullConfig . OutStream , repoInfo , tag , sf ) ; err != nil {
2015-03-25 03:44:12 -04:00
return err
2014-08-08 02:58:58 -04:00
}
2015-04-03 18:17:49 -04:00
s . eventsService . Log ( "pull" , logName , "" )
2014-10-22 09:48:02 -04:00
2015-03-25 03:44:12 -04:00
return nil
2015-05-15 20:48:20 -04:00
}
func makeMirrorRepoInfo ( repoInfo * registry . RepositoryInfo , mirror string ) * registry . RepositoryInfo {
mirrorRepo := & registry . RepositoryInfo {
RemoteName : repoInfo . RemoteName ,
LocalName : repoInfo . LocalName ,
CanonicalName : repoInfo . CanonicalName ,
Official : false ,
Index : & registry . IndexInfo {
Official : false ,
Secure : repoInfo . Index . Secure ,
Name : mirror ,
Mirrors : [ ] string { } ,
} ,
}
return mirrorRepo
}
2015-05-22 21:20:54 -04:00
func configureV2Mirror ( repoInfo * registry . RepositoryInfo , s * registry . Service ) ( * registry . Endpoint , * registry . RepositoryInfo , error ) {
mirrors := repoInfo . Index . Mirrors
2015-05-27 18:52:51 -04:00
if len ( mirrors ) == 0 {
// no mirrors configured
return nil , nil , nil
2015-05-15 20:48:20 -04:00
}
v1MirrorCount := 0
var v2MirrorEndpoint * registry . Endpoint
var v2MirrorRepoInfo * registry . RepositoryInfo
for _ , mirror := range mirrors {
mirrorRepoInfo := makeMirrorRepoInfo ( repoInfo , mirror )
endpoint , err := registry . NewEndpoint ( mirrorRepoInfo . Index , nil )
if err != nil {
logrus . Errorf ( "Unable to create endpoint for %s: %s" , mirror , err )
continue
}
if endpoint . Version == 2 {
if v2MirrorEndpoint == nil {
v2MirrorEndpoint = endpoint
v2MirrorRepoInfo = mirrorRepoInfo
} else {
// > 1 v2 mirrors given
return nil , nil , fmt . Errorf ( "multiple v2 mirrors configured" )
}
} else {
v1MirrorCount ++
}
}
if v1MirrorCount == len ( mirrors ) {
// OK, but mirrors are v1
return nil , nil , nil
}
if v2MirrorEndpoint != nil && v1MirrorCount == 0 {
// OK, 1 v2 mirror specified
return v2MirrorEndpoint , v2MirrorRepoInfo , nil
}
if v2MirrorEndpoint != nil && v1MirrorCount > 0 {
2015-06-01 18:18:56 -04:00
return nil , nil , fmt . Errorf ( "v1 and v2 mirrors configured" )
2015-05-15 20:48:20 -04:00
}
2015-06-01 18:18:56 -04:00
// No endpoint could be established with the given mirror configurations
// Fallback to pulling from the hub as per v1 behavior.
return nil , nil , nil
2015-05-15 20:48:20 -04:00
}
func ( s * TagStore ) pullFromV2Mirror ( mirrorEndpoint * registry . Endpoint , repoInfo * registry . RepositoryInfo ,
imagePullConfig * ImagePullConfig , tag string , sf * streamformatter . StreamFormatter , logName string ) error {
tr := transport . NewTransport (
registry . NewTransport ( registry . ReceiveTimeout , mirrorEndpoint . IsSecure ) ,
registry . DockerHeaders ( imagePullConfig . MetaHeaders ) ... ,
)
client := registry . HTTPClient ( tr )
2015-05-22 21:20:54 -04:00
mirrorSession , err := registry . NewSession ( client , & cliconfig . AuthConfig { } , mirrorEndpoint )
2015-05-15 20:48:20 -04:00
if err != nil {
return err
}
logrus . Debugf ( "Pulling v2 repository with local name %q from %s" , repoInfo . LocalName , mirrorEndpoint . URL )
if err := s . pullV2Repository ( mirrorSession , imagePullConfig . OutStream , repoInfo , tag , sf ) ; err != nil {
return err
}
s . eventsService . Log ( "pull" , logName , "" )
return nil
2014-08-08 02:58:58 -04:00
}
2015-05-11 17:53:52 -04:00
func ( s * TagStore ) pullRepository ( r * registry . Session , out io . Writer , repoInfo * registry . RepositoryInfo , askedTag string , sf * streamformatter . StreamFormatter ) error {
2014-10-06 21:54:52 -04:00
out . Write ( sf . FormatStatus ( "" , "Pulling repository %s" , repoInfo . CanonicalName ) )
2014-08-08 02:58:58 -04:00
2014-10-06 21:54:52 -04:00
repoData , err := r . GetRepositoryData ( repoInfo . RemoteName )
2014-08-08 02:58:58 -04:00
if err != nil {
if strings . Contains ( err . Error ( ) , "HTTP code: 404" ) {
2015-02-26 21:23:50 -05:00
return fmt . Errorf ( "Error: image %s not found" , utils . ImageReference ( repoInfo . RemoteName , askedTag ) )
2014-08-08 02:58:58 -04:00
}
2014-08-29 07:21:28 -04:00
// Unexpected HTTP error
return err
2014-08-08 02:58:58 -04:00
}
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Retrieving the tag list" )
2015-06-10 18:18:15 -04:00
tagsList := make ( map [ string ] string )
if askedTag == "" {
tagsList , err = r . GetRemoteTags ( repoData . Endpoints , repoInfo . RemoteName )
} else {
var tagId string
tagId , err = r . GetRemoteTag ( repoData . Endpoints , repoInfo . RemoteName , askedTag )
tagsList [ askedTag ] = tagId
}
2014-08-08 02:58:58 -04:00
if err != nil {
2015-06-10 18:18:15 -04:00
if err == registry . ErrRepoNotFound && askedTag != "" {
return fmt . Errorf ( "Tag %s not found in repository %s" , askedTag , repoInfo . CanonicalName )
}
2015-03-26 18:22:04 -04:00
logrus . Errorf ( "unable to get remote tags: %s" , err )
2014-08-08 02:58:58 -04:00
return err
}
for tag , id := range tagsList {
repoData . ImgList [ id ] = & registry . ImgData {
ID : id ,
Tag : tag ,
Checksum : "" ,
}
}
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Registering tags" )
2014-08-08 02:58:58 -04:00
// If no tag has been specified, pull them all
if askedTag == "" {
for tag , id := range tagsList {
repoData . ImgList [ id ] . Tag = tag
}
} else {
// Otherwise, check that the tag exists and use only that one
id , exists := tagsList [ askedTag ]
if ! exists {
2014-10-06 21:54:52 -04:00
return fmt . Errorf ( "Tag %s not found in repository %s" , askedTag , repoInfo . CanonicalName )
2014-08-08 02:58:58 -04:00
}
repoData . ImgList [ id ] . Tag = askedTag
}
errors := make ( chan error )
2014-09-23 18:53:43 -04:00
2015-03-25 21:40:23 -04:00
layersDownloaded := false
2014-08-08 02:58:58 -04:00
for _ , image := range repoData . ImgList {
downloadImage := func ( img * registry . ImgData ) {
if askedTag != "" && img . Tag != askedTag {
2015-05-11 17:53:52 -04:00
errors <- nil
2014-08-08 02:58:58 -04:00
return
}
if img . Tag == "" {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Image (id: %s) present in this repository but untagged, skipping" , img . ID )
2015-05-11 17:53:52 -04:00
errors <- nil
2014-08-08 02:58:58 -04:00
return
}
// ensure no two downloads of the same image happen at the same time
if c , err := s . poolAdd ( "pull" , "img:" + img . ID ) ; err != nil {
if c != nil {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Layer already being pulled by another client. Waiting." , nil ) )
2014-08-08 02:58:58 -04:00
<- c
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Download complete" , nil ) )
2014-08-08 02:58:58 -04:00
} else {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Image (id: %s) pull is already running, skipping: %v" , img . ID , err )
2014-08-08 02:58:58 -04:00
}
2015-05-11 17:53:52 -04:00
errors <- nil
2014-08-08 02:58:58 -04:00
return
}
defer s . poolRemove ( "pull" , "img:" + img . ID )
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , fmt . Sprintf ( "Pulling image (%s) from %s" , img . Tag , repoInfo . CanonicalName ) , nil ) )
2014-08-08 02:58:58 -04:00
success := false
2014-09-23 18:53:43 -04:00
var lastErr , err error
2015-03-25 21:40:23 -04:00
var isDownloaded bool
2014-10-06 21:54:52 -04:00
for _ , ep := range repoInfo . Index . Mirrors {
2015-05-27 17:59:57 -04:00
// Ensure endpoint is v1
ep = ep + "v1/"
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , fmt . Sprintf ( "Pulling image (%s) from %s, mirror: %s" , img . Tag , repoInfo . CanonicalName , ep ) , nil ) )
2015-03-25 21:40:23 -04:00
if isDownloaded , err = s . pullImage ( r , out , img . ID , ep , repoData . Tokens , sf ) ; err != nil {
2014-10-06 21:54:52 -04:00
// Don't report errors when pulling from mirrors.
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Error pulling image (%s) from %s, mirror: %s, %s" , img . Tag , repoInfo . CanonicalName , ep , err )
2014-10-06 21:54:52 -04:00
continue
2014-07-18 14:48:19 -04:00
}
2015-03-25 21:40:23 -04:00
layersDownloaded = layersDownloaded || isDownloaded
2014-10-06 21:54:52 -04:00
success = true
break
2014-07-18 14:48:19 -04:00
}
if ! success {
for _ , ep := range repoData . Endpoints {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , fmt . Sprintf ( "Pulling image (%s) from %s, endpoint: %s" , img . Tag , repoInfo . CanonicalName , ep ) , nil ) )
2015-03-25 21:40:23 -04:00
if isDownloaded , err = s . pullImage ( r , out , img . ID , ep , repoData . Tokens , sf ) ; err != nil {
2014-07-18 14:48:19 -04:00
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , fmt . Sprintf ( "Error pulling image (%s) from %s, endpoint: %s, %s" , img . Tag , repoInfo . CanonicalName , ep , err ) , nil ) )
2014-07-18 14:48:19 -04:00
continue
}
2015-03-25 21:40:23 -04:00
layersDownloaded = layersDownloaded || isDownloaded
2014-07-18 14:48:19 -04:00
success = true
break
2014-08-08 02:58:58 -04:00
}
}
if ! success {
2014-10-06 21:54:52 -04:00
err := fmt . Errorf ( "Error pulling image (%s) from %s, %v" , img . Tag , repoInfo . CanonicalName , lastErr )
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , err . Error ( ) , nil ) )
2015-05-11 17:53:52 -04:00
errors <- err
return
2014-08-08 02:58:58 -04:00
}
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Download complete" , nil ) )
2014-08-08 02:58:58 -04:00
2015-05-11 17:53:52 -04:00
errors <- nil
2014-08-08 02:58:58 -04:00
}
2015-05-11 17:53:52 -04:00
go downloadImage ( image )
2014-08-08 02:58:58 -04:00
}
2015-05-11 17:53:52 -04:00
var lastError error
for i := 0 ; i < len ( repoData . ImgList ) ; i ++ {
if err := <- errors ; err != nil {
lastError = err
}
}
if lastError != nil {
return lastError
2014-08-08 02:58:58 -04:00
}
2015-05-11 17:53:52 -04:00
2014-08-08 02:58:58 -04:00
for tag , id := range tagsList {
2015-02-04 18:06:34 -05:00
if askedTag != "" && tag != askedTag {
2014-08-08 02:58:58 -04:00
continue
}
2015-04-13 22:46:29 -04:00
if err := s . Tag ( repoInfo . LocalName , tag , id , true ) ; err != nil {
2014-08-08 02:58:58 -04:00
return err
}
}
2014-10-06 21:54:52 -04:00
requestedTag := repoInfo . CanonicalName
2014-09-23 18:53:43 -04:00
if len ( askedTag ) > 0 {
2015-02-26 21:23:50 -05:00
requestedTag = utils . ImageReference ( repoInfo . CanonicalName , askedTag )
2014-09-23 18:53:43 -04:00
}
2015-03-25 21:40:23 -04:00
WriteStatus ( requestedTag , out , sf , layersDownloaded )
2014-08-08 02:58:58 -04:00
return nil
}
2015-03-17 22:18:41 -04:00
func ( s * TagStore ) pullImage ( r * registry . Session , out io . Writer , imgID , endpoint string , token [ ] string , sf * streamformatter . StreamFormatter ) ( bool , error ) {
2015-05-14 10:12:54 -04:00
history , err := r . GetRemoteHistory ( imgID , endpoint )
2014-08-08 02:58:58 -04:00
if err != nil {
2014-09-23 18:53:43 -04:00
return false , err
2014-08-08 02:58:58 -04:00
}
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( imgID ) , "Pulling dependent layers" , nil ) )
2014-08-08 02:58:58 -04:00
// FIXME: Try to stream the images?
// FIXME: Launch the getRemoteImage() in goroutines
2015-03-25 21:40:23 -04:00
layersDownloaded := false
2014-08-08 02:58:58 -04:00
for i := len ( history ) - 1 ; i >= 0 ; i -- {
id := history [ i ]
// ensure no two downloads of the same layer happen at the same time
if c , err := s . poolAdd ( "pull" , "layer:" + id ) ; err != nil {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Image (id: %s) pull is already running, skipping: %v" , id , err )
2014-08-08 02:58:58 -04:00
<- c
}
defer s . poolRemove ( "pull" , "layer:" + id )
if ! s . graph . Exists ( id ) {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , "Pulling metadata" , nil ) )
2014-08-08 02:58:58 -04:00
var (
imgJSON [ ] byte
imgSize int
err error
img * image . Image
)
retries := 5
for j := 1 ; j <= retries ; j ++ {
2015-05-14 10:12:54 -04:00
imgJSON , imgSize , err = r . GetRemoteImageJSON ( id , endpoint )
2014-08-08 02:58:58 -04:00
if err != nil && j == retries {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , "Error pulling dependent layers" , nil ) )
2015-03-25 21:40:23 -04:00
return layersDownloaded , err
2014-08-08 02:58:58 -04:00
} else if err != nil {
time . Sleep ( time . Duration ( j ) * 500 * time . Millisecond )
continue
}
img , err = image . NewImgJSON ( imgJSON )
2015-03-25 21:40:23 -04:00
layersDownloaded = true
2014-08-08 02:58:58 -04:00
if err != nil && j == retries {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , "Error pulling dependent layers" , nil ) )
2015-03-25 21:40:23 -04:00
return layersDownloaded , fmt . Errorf ( "Failed to parse json: %s" , err )
2014-08-08 02:58:58 -04:00
} else if err != nil {
time . Sleep ( time . Duration ( j ) * 500 * time . Millisecond )
continue
} else {
break
}
}
for j := 1 ; j <= retries ; j ++ {
// Get the layer
status := "Pulling fs layer"
if j > 1 {
status = fmt . Sprintf ( "Pulling fs layer [retries: %d]" , j )
}
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , status , nil ) )
2015-05-14 10:12:54 -04:00
layer , err := r . GetRemoteImageLayer ( img . ID , endpoint , int64 ( imgSize ) )
2014-08-08 02:58:58 -04:00
if uerr , ok := err . ( * url . Error ) ; ok {
err = uerr . Err
}
if terr , ok := err . ( net . Error ) ; ok && terr . Timeout ( ) && j < retries {
time . Sleep ( time . Duration ( j ) * 500 * time . Millisecond )
continue
} else if err != nil {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , "Error pulling dependent layers" , nil ) )
2015-03-25 21:40:23 -04:00
return layersDownloaded , err
2014-08-08 02:58:58 -04:00
}
2015-03-25 21:40:23 -04:00
layersDownloaded = true
2014-08-08 02:58:58 -04:00
defer layer . Close ( )
2014-10-27 14:00:29 -04:00
err = s . graph . Register ( img ,
2015-02-24 03:51:46 -05:00
progressreader . New ( progressreader . Config {
In : layer ,
Out : out ,
Formatter : sf ,
Size : imgSize ,
NewLines : false ,
2015-03-24 07:25:26 -04:00
ID : stringid . TruncateID ( id ) ,
2015-02-24 03:51:46 -05:00
Action : "Downloading" ,
} ) )
2014-08-08 02:58:58 -04:00
if terr , ok := err . ( net . Error ) ; ok && terr . Timeout ( ) && j < retries {
time . Sleep ( time . Duration ( j ) * 500 * time . Millisecond )
continue
} else if err != nil {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , "Error downloading dependent layers" , nil ) )
2015-03-25 21:40:23 -04:00
return layersDownloaded , err
2014-08-08 02:58:58 -04:00
} else {
break
}
}
}
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( id ) , "Download complete" , nil ) )
2014-09-23 18:53:43 -04:00
}
2015-03-25 21:40:23 -04:00
return layersDownloaded , nil
2014-09-23 18:53:43 -04:00
}
2014-08-08 02:58:58 -04:00
2015-03-25 21:40:23 -04:00
func WriteStatus ( requestedTag string , out io . Writer , sf * streamformatter . StreamFormatter , layersDownloaded bool ) {
if layersDownloaded {
2014-09-23 18:53:43 -04:00
out . Write ( sf . FormatStatus ( "" , "Status: Downloaded newer image for %s" , requestedTag ) )
} else {
out . Write ( sf . FormatStatus ( "" , "Status: Image is up to date for %s" , requestedTag ) )
2014-08-08 02:58:58 -04:00
}
}
2014-10-01 21:26:06 -04:00
2015-05-11 17:53:52 -04:00
func ( s * TagStore ) pullV2Repository ( r * registry . Session , out io . Writer , repoInfo * registry . RepositoryInfo , tag string , sf * streamformatter . StreamFormatter ) error {
2015-01-14 19:46:31 -05:00
endpoint , err := r . V2RegistryEndpoint ( repoInfo . Index )
if err != nil {
2015-02-02 18:54:14 -05:00
if repoInfo . Index . Official {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Unable to pull from V2 registry, falling back to v1: %s" , err )
2015-02-02 18:54:14 -05:00
return ErrV2RegistryUnavailable
}
2015-01-14 19:46:31 -05:00
return fmt . Errorf ( "error getting registry endpoint: %s" , err )
}
auth , err := r . GetV2Authorization ( endpoint , repoInfo . RemoteName , true )
2015-01-13 18:48:49 -05:00
if err != nil {
return fmt . Errorf ( "error getting authorization: %s" , err )
}
2015-05-28 23:46:20 -04:00
if ! auth . CanAuthorizeV2 ( ) {
return ErrV2RegistryUnavailable
}
2014-10-08 19:14:54 -04:00
var layersDownloaded bool
2014-10-01 21:26:06 -04:00
if tag == "" {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Pulling tag list from V2 registry for %s" , repoInfo . CanonicalName )
2015-01-14 19:46:31 -05:00
tags , err := r . GetV2RemoteTags ( endpoint , repoInfo . RemoteName , auth )
2014-10-01 21:26:06 -04:00
if err != nil {
return err
}
2015-01-30 19:11:47 -05:00
if len ( tags ) == 0 {
return registry . ErrDoesNotExist
}
2014-10-01 21:26:06 -04:00
for _ , t := range tags {
2015-05-11 17:53:52 -04:00
if downloaded , err := s . pullV2Tag ( r , out , endpoint , repoInfo , t , sf , auth ) ; err != nil {
2014-10-01 21:26:06 -04:00
return err
2014-10-08 19:14:54 -04:00
} else if downloaded {
layersDownloaded = true
2014-10-01 21:26:06 -04:00
}
}
} else {
2015-05-11 17:53:52 -04:00
if downloaded , err := s . pullV2Tag ( r , out , endpoint , repoInfo , tag , sf , auth ) ; err != nil {
2014-10-01 21:26:06 -04:00
return err
2014-10-08 19:14:54 -04:00
} else if downloaded {
layersDownloaded = true
2014-10-01 21:26:06 -04:00
}
}
2014-10-06 21:54:52 -04:00
requestedTag := repoInfo . CanonicalName
2014-10-08 19:14:54 -04:00
if len ( tag ) > 0 {
2015-02-26 21:23:50 -05:00
requestedTag = utils . ImageReference ( repoInfo . CanonicalName , tag )
2014-10-08 19:14:54 -04:00
}
WriteStatus ( requestedTag , out , sf , layersDownloaded )
2014-10-01 21:26:06 -04:00
return nil
}
2015-05-11 17:53:52 -04:00
func ( s * TagStore ) pullV2Tag ( r * registry . Session , out io . Writer , endpoint * registry . Endpoint , repoInfo * registry . RepositoryInfo , tag string , sf * streamformatter . StreamFormatter , auth * registry . RequestAuthorization ) ( bool , error ) {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Pulling tag from V2 registry: %q" , tag )
2015-03-18 02:45:30 -04:00
2015-05-29 01:50:56 -04:00
remoteDigest , manifestBytes , err := r . GetV2ImageManifest ( endpoint , repoInfo . RemoteName , tag , auth )
2014-10-01 21:26:06 -04:00
if err != nil {
2014-10-08 19:14:54 -04:00
return false , err
2014-10-01 21:26:06 -04:00
}
2015-03-18 02:45:30 -04:00
// loadManifest ensures that the manifest payload has the expected digest
// if the tag is a digest reference.
2015-05-29 01:50:56 -04:00
localDigest , manifest , verified , err := s . loadManifest ( manifestBytes , tag , remoteDigest )
2014-10-01 21:26:06 -04:00
if err != nil {
2014-10-08 19:14:54 -04:00
return false , fmt . Errorf ( "error verifying manifest: %s" , err )
2014-10-01 21:26:06 -04:00
}
if verified {
2015-03-26 18:22:04 -04:00
logrus . Printf ( "Image manifest for %s has been verified" , utils . ImageReference ( repoInfo . CanonicalName , tag ) )
2014-10-01 21:26:06 -04:00
}
2015-02-05 20:46:55 -05:00
out . Write ( sf . FormatStatus ( tag , "Pulling from %s" , repoInfo . CanonicalName ) )
2015-01-20 22:19:11 -05:00
2015-05-29 01:50:56 -04:00
// downloadInfo is used to pass information from download to extractor
type downloadInfo struct {
imgJSON [ ] byte
img * image . Image
digest digest . Digest
tmpFile * os . File
length int64
downloaded bool
err chan error
}
2014-10-09 20:34:34 -04:00
downloads := make ( [ ] downloadInfo , len ( manifest . FSLayers ) )
2014-10-01 21:26:06 -04:00
2014-10-09 20:34:34 -04:00
for i := len ( manifest . FSLayers ) - 1 ; i >= 0 ; i -- {
2014-10-01 21:26:06 -04:00
var (
2014-10-09 20:34:34 -04:00
sumStr = manifest . FSLayers [ i ] . BlobSum
imgJSON = [ ] byte ( manifest . History [ i ] . V1Compatibility )
2014-10-01 21:26:06 -04:00
)
img , err := image . NewImgJSON ( imgJSON )
if err != nil {
2014-10-08 19:14:54 -04:00
return false , fmt . Errorf ( "failed to parse json: %s" , err )
2014-10-01 21:26:06 -04:00
}
downloads [ i ] . img = img
// Check if exists
if s . graph . Exists ( img . ID ) {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Image already exists: %s" , img . ID )
2014-10-01 21:26:06 -04:00
continue
}
2015-03-11 11:17:48 -04:00
dgst , err := digest . ParseDigest ( sumStr )
if err != nil {
return false , err
2014-10-01 21:26:06 -04:00
}
2015-03-11 11:17:48 -04:00
downloads [ i ] . digest = dgst
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Pulling fs layer" , nil ) )
2014-10-01 21:26:06 -04:00
downloadFunc := func ( di * downloadInfo ) error {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "pulling blob %q to V1 img %s" , sumStr , img . ID )
2014-10-01 21:26:06 -04:00
if c , err := s . poolAdd ( "pull" , "img:" + img . ID ) ; err != nil {
if c != nil {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Layer already being pulled by another client. Waiting." , nil ) )
2014-10-01 21:26:06 -04:00
<- c
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Download complete" , nil ) )
2014-10-01 21:26:06 -04:00
} else {
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Image (id: %s) pull is already running, skipping: %v" , img . ID , err )
2014-10-01 21:26:06 -04:00
}
} else {
2014-10-03 19:16:03 -04:00
defer s . poolRemove ( "pull" , "img:" + img . ID )
2014-10-01 21:26:06 -04:00
tmpFile , err := ioutil . TempFile ( "" , "GetV2ImageBlob" )
if err != nil {
return err
}
2015-03-31 18:02:27 -04:00
r , l , err := r . GetV2ImageBlobReader ( endpoint , repoInfo . RemoteName , di . digest , auth )
2014-10-01 21:26:06 -04:00
if err != nil {
return err
}
defer r . Close ( )
2014-12-23 16:40:06 -05:00
2015-03-11 11:17:48 -04:00
verifier , err := digest . NewDigestVerifier ( di . digest )
2014-12-23 16:40:06 -05:00
if err != nil {
2015-03-11 11:17:48 -04:00
return err
2014-12-23 16:40:06 -05:00
}
2015-02-24 03:51:46 -05:00
if _ , err := io . Copy ( tmpFile , progressreader . New ( progressreader . Config {
2015-03-11 11:17:48 -04:00
In : ioutil . NopCloser ( io . TeeReader ( r , verifier ) ) ,
2015-02-24 03:51:46 -05:00
Out : out ,
Formatter : sf ,
Size : int ( l ) ,
NewLines : false ,
2015-03-24 07:25:26 -04:00
ID : stringid . TruncateID ( img . ID ) ,
2015-02-24 03:51:46 -05:00
Action : "Downloading" ,
} ) ) ; err != nil {
2015-03-02 20:40:10 -05:00
return fmt . Errorf ( "unable to copy v2 image blob data: %s" , err )
}
2014-12-23 16:40:06 -05:00
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Verifying Checksum" , nil ) )
2014-12-23 16:40:06 -05:00
2015-03-11 11:17:48 -04:00
if ! verifier . Verified ( ) {
2015-05-29 01:50:56 -04:00
return fmt . Errorf ( "image layer digest verification failed for %q" , di . digest )
2014-12-23 16:40:06 -05:00
}
2014-10-01 21:26:06 -04:00
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( img . ID ) , "Download complete" , nil ) )
2014-10-01 21:26:06 -04:00
2015-03-26 18:22:04 -04:00
logrus . Debugf ( "Downloaded %s to tempfile %s" , img . ID , tmpFile . Name ( ) )
2014-10-01 21:26:06 -04:00
di . tmpFile = tmpFile
di . length = l
di . downloaded = true
}
di . imgJSON = imgJSON
return nil
}
2015-05-11 17:53:52 -04:00
downloads [ i ] . err = make ( chan error )
go func ( di * downloadInfo ) {
di . err <- downloadFunc ( di )
} ( & downloads [ i ] )
2014-10-01 21:26:06 -04:00
}
2015-03-16 15:22:00 -04:00
var tagUpdated bool
2014-10-01 21:26:06 -04:00
for i := len ( downloads ) - 1 ; i >= 0 ; i -- {
d := & downloads [ i ]
if d . err != nil {
2015-04-26 12:50:25 -04:00
if err := <- d . err ; err != nil {
2014-10-08 19:14:54 -04:00
return false , err
2014-10-01 21:26:06 -04:00
}
}
if d . downloaded {
// if tmpFile is empty assume download and extracted elsewhere
defer os . Remove ( d . tmpFile . Name ( ) )
defer d . tmpFile . Close ( )
d . tmpFile . Seek ( 0 , 0 )
if d . tmpFile != nil {
2014-10-27 14:00:29 -04:00
err = s . graph . Register ( d . img ,
2015-02-24 03:51:46 -05:00
progressreader . New ( progressreader . Config {
In : d . tmpFile ,
Out : out ,
Formatter : sf ,
Size : int ( d . length ) ,
2015-03-24 07:25:26 -04:00
ID : stringid . TruncateID ( d . img . ID ) ,
2015-02-24 03:51:46 -05:00
Action : "Extracting" ,
} ) )
2014-10-01 21:26:06 -04:00
if err != nil {
2014-10-08 19:14:54 -04:00
return false , err
2014-10-01 21:26:06 -04:00
}
2015-06-15 14:06:21 -04:00
if err := s . graph . SetDigest ( d . img . ID , d . digest ) ; err != nil {
return false , err
}
2014-10-01 21:26:06 -04:00
// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
}
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( d . img . ID ) , "Pull complete" , nil ) )
2015-03-16 15:22:00 -04:00
tagUpdated = true
2014-10-01 21:26:06 -04:00
} else {
2015-03-24 07:25:26 -04:00
out . Write ( sf . FormatProgress ( stringid . TruncateID ( d . img . ID ) , "Already exists" , nil ) )
2014-10-01 21:26:06 -04:00
}
}
2015-03-16 15:22:00 -04:00
// Check for new tag if no layers downloaded
if ! tagUpdated {
repo , err := s . Get ( repoInfo . LocalName )
if err != nil {
return false , err
}
if repo != nil {
if _ , exists := repo [ tag ] ; ! exists {
tagUpdated = true
}
2015-03-26 05:27:10 -04:00
} else {
tagUpdated = true
2015-03-16 15:22:00 -04:00
}
}
if verified && tagUpdated {
2015-02-26 21:23:50 -05:00
out . Write ( sf . FormatStatus ( utils . ImageReference ( repoInfo . CanonicalName , tag ) , "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security." ) )
2015-02-05 20:46:55 -05:00
}
2015-01-20 22:19:11 -05:00
2015-05-29 01:50:56 -04:00
if localDigest != remoteDigest { // this is not a verification check.
// NOTE(stevvooe): This is a very defensive branch and should never
// happen, since all manifest digest implementations use the same
// algorithm.
logrus . WithFields (
logrus . Fields {
"local" : localDigest ,
"remote" : remoteDigest ,
} ) . Debugf ( "local digest does not match remote" )
out . Write ( sf . FormatStatus ( "" , "Remote Digest: %s" , remoteDigest ) )
2015-02-26 21:23:50 -05:00
}
2015-05-29 01:50:56 -04:00
out . Write ( sf . FormatStatus ( "" , "Digest: %s" , localDigest ) )
2015-05-29 17:20:05 -04:00
if tag == localDigest . String ( ) {
// TODO(stevvooe): Ideally, we should always set the digest so we can
// use the digest whether we pull by it or not. Unfortunately, the tag
// store treats the digest as a separate tag, meaning there may be an
// untagged digest image that would seem to be dangling by a user.
if err = s . SetDigest ( repoInfo . LocalName , localDigest . String ( ) , downloads [ 0 ] . img . ID ) ; err != nil {
2015-02-26 21:23:50 -05:00
return false , err
}
}
2015-05-29 01:50:56 -04:00
if ! utils . DigestReference ( tag ) {
2015-02-26 21:23:50 -05:00
// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
2015-04-13 22:46:29 -04:00
if err = s . Tag ( repoInfo . LocalName , tag , downloads [ 0 ] . img . ID , true ) ; err != nil {
2015-02-26 21:23:50 -05:00
return false , err
}
2014-10-01 21:26:06 -04:00
}
2015-03-16 15:22:00 -04:00
return tagUpdated , nil
2014-10-01 21:26:06 -04:00
}