2015-11-18 22:18:44 +00:00
package distribution
import (
"errors"
"fmt"
"io"
2015-11-14 00:59:01 +00:00
"io/ioutil"
2015-11-18 22:18:44 +00:00
"net"
"net/url"
2016-01-26 02:20:18 +00:00
"os"
2015-11-18 22:18:44 +00:00
"strings"
"time"
2017-01-26 00:54:18 +00:00
"github.com/docker/distribution/reference"
2017-09-13 19:49:04 +00:00
"github.com/docker/distribution/registry/client/auth"
2015-11-18 22:18:44 +00:00
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/distribution/metadata"
2015-11-14 00:59:01 +00:00
"github.com/docker/docker/distribution/xfer"
2016-01-04 18:36:01 +00:00
"github.com/docker/docker/dockerversion"
2015-11-18 22:18:44 +00:00
"github.com/docker/docker/image"
"github.com/docker/docker/image/v1"
"github.com/docker/docker/layer"
2015-11-14 00:59:01 +00:00
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress"
2015-11-18 22:18:44 +00:00
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
2017-07-26 21:42:13 +00:00
"github.com/sirupsen/logrus"
2015-11-14 00:59:01 +00:00
"golang.org/x/net/context"
2015-11-18 22:18:44 +00:00
)
type v1Puller struct {
v1IDService * metadata . V1IDService
endpoint registry . APIEndpoint
config * ImagePullConfig
repoInfo * registry . RepositoryInfo
session * registry . Session
}
2017-08-08 19:43:48 +00:00
func ( p * v1Puller ) Pull ( ctx context . Context , ref reference . Named , platform string ) error {
2015-12-04 21:55:15 +00:00
if _ , isCanonical := ref . ( reference . Canonical ) ; isCanonical {
2015-11-18 22:18:44 +00:00
// Allowing fallback, because HTTPS v1 is before HTTP v2
2016-02-11 22:08:49 +00:00
return fallbackError { err : ErrNoSupport { Err : errors . New ( "Cannot pull by digest with v1 registry" ) } }
2015-11-18 22:18:44 +00:00
}
tlsConfig , err := p . config . RegistryService . TLSConfig ( p . repoInfo . Index . Name )
if err != nil {
2015-12-04 21:42:33 +00:00
return err
2015-11-18 22:18:44 +00:00
}
// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
tr := transport . NewTransport (
// TODO(tiborvass): was ReceiveTimeout
registry . NewTransport ( tlsConfig ) ,
2016-03-18 21:42:40 +00:00
registry . DockerHeaders ( dockerversion . DockerUserAgent ( ctx ) , p . config . MetaHeaders ) ... ,
2015-11-18 22:18:44 +00:00
)
client := registry . HTTPClient ( tr )
2017-07-19 14:20:13 +00:00
v1Endpoint := p . endpoint . ToV1Endpoint ( dockerversion . DockerUserAgent ( ctx ) , p . config . MetaHeaders )
2015-11-18 22:18:44 +00:00
p . session , err = registry . NewSession ( client , p . config . AuthConfig , v1Endpoint )
if err != nil {
// TODO(dmcgowan): Check if should fallback
logrus . Debugf ( "Fallback from error: %s" , err )
2015-12-04 21:42:33 +00:00
return fallbackError { err : err }
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
if err := p . pullRepository ( ctx , ref ) ; err != nil {
2015-11-18 22:18:44 +00:00
// TODO(dmcgowan): Check if should fallback
2015-12-04 21:42:33 +00:00
return err
2015-11-18 22:18:44 +00:00
}
2017-01-26 00:54:18 +00:00
progress . Message ( p . config . ProgressOutput , "" , p . repoInfo . Name . Name ( ) + ": this image was pulled from a legacy registry. Important: This registry version will not be supported in future versions of docker." )
2015-11-18 22:18:44 +00:00
2015-12-04 21:42:33 +00:00
return nil
2015-11-18 22:18:44 +00:00
}
2017-09-13 19:49:04 +00:00
// Note use auth.Scope rather than reference.Named due to this warning causing Jenkins CI to fail:
// warning: ref can be github.com/docker/docker/vendor/github.com/docker/distribution/registry/client/auth.Scope (interfacer)
func ( p * v1Puller ) pullRepository ( ctx context . Context , ref auth . Scope ) error {
2017-01-26 00:54:18 +00:00
progress . Message ( p . config . ProgressOutput , "" , "Pulling repository " + p . repoInfo . Name . Name ( ) )
2015-11-18 22:18:44 +00:00
2016-03-29 00:16:56 +00:00
tagged , isTagged := ref . ( reference . NamedTagged )
2017-01-26 00:54:18 +00:00
repoData , err := p . session . GetRepositoryData ( p . repoInfo . Name )
2015-11-18 22:18:44 +00:00
if err != nil {
if strings . Contains ( err . Error ( ) , "HTTP code: 404" ) {
2016-03-29 00:16:56 +00:00
if isTagged {
2017-01-26 00:54:18 +00:00
return fmt . Errorf ( "Error: image %s:%s not found" , reference . Path ( p . repoInfo . Name ) , tagged . Tag ( ) )
2016-03-29 00:16:56 +00:00
}
2017-01-26 00:54:18 +00:00
return fmt . Errorf ( "Error: image %s not found" , reference . Path ( p . repoInfo . Name ) )
2015-11-18 22:18:44 +00:00
}
// Unexpected HTTP error
return err
}
2016-06-11 20:16:55 +00:00
logrus . Debug ( "Retrieving the tag list" )
2015-11-18 22:18:44 +00:00
var tagsList map [ string ] string
if ! isTagged {
2017-01-26 00:54:18 +00:00
tagsList , err = p . session . GetRemoteTags ( repoData . Endpoints , p . repoInfo . Name )
2015-11-18 22:18:44 +00:00
} else {
var tagID string
tagsList = make ( map [ string ] string )
2017-01-26 00:54:18 +00:00
tagID , err = p . session . GetRemoteTag ( repoData . Endpoints , p . repoInfo . Name , tagged . Tag ( ) )
2015-11-18 22:18:44 +00:00
if err == registry . ErrRepoNotFound {
2017-01-26 00:54:18 +00:00
return fmt . Errorf ( "Tag %s not found in repository %s" , tagged . Tag ( ) , p . repoInfo . Name . Name ( ) )
2015-11-18 22:18:44 +00:00
}
tagsList [ tagged . Tag ( ) ] = tagID
}
if err != nil {
logrus . Errorf ( "unable to get remote tags: %s" , err )
return err
}
for tag , id := range tagsList {
repoData . ImgList [ id ] = & registry . ImgData {
ID : id ,
Tag : tag ,
Checksum : "" ,
}
}
layersDownloaded := false
for _ , imgData := range repoData . ImgList {
if isTagged && imgData . Tag != tagged . Tag ( ) {
continue
}
2015-11-14 00:59:01 +00:00
err := p . downloadImage ( ctx , repoData , imgData , & layersDownloaded )
if err != nil {
return err
2015-11-18 22:18:44 +00:00
}
}
2017-01-26 00:54:18 +00:00
writeStatus ( reference . FamiliarString ( ref ) , p . config . ProgressOutput , layersDownloaded )
2015-11-18 22:18:44 +00:00
return nil
}
2015-11-14 00:59:01 +00:00
func ( p * v1Puller ) downloadImage ( ctx context . Context , repoData * registry . RepositoryData , img * registry . ImgData , layersDownloaded * bool ) error {
2015-11-18 22:18:44 +00:00
if img . Tag == "" {
logrus . Debugf ( "Image (id: %s) present in this repository but untagged, skipping" , img . ID )
2015-11-14 00:59:01 +00:00
return nil
2015-11-18 22:18:44 +00:00
}
2017-01-26 00:54:18 +00:00
localNameRef , err := reference . WithTag ( p . repoInfo . Name , img . Tag )
2015-11-18 22:18:44 +00:00
if err != nil {
retErr := fmt . Errorf ( "Image (id: %s) has invalid tag: %s" , img . ID , img . Tag )
logrus . Debug ( retErr . Error ( ) )
2015-11-14 00:59:01 +00:00
return retErr
2015-11-18 22:18:44 +00:00
}
if err := v1 . ValidateID ( img . ID ) ; err != nil {
2015-11-14 00:59:01 +00:00
return err
2015-11-18 22:18:44 +00:00
}
2017-01-26 00:54:18 +00:00
progress . Updatef ( p . config . ProgressOutput , stringid . TruncateID ( img . ID ) , "Pulling image (%s) from %s" , img . Tag , p . repoInfo . Name . Name ( ) )
2015-11-18 22:18:44 +00:00
success := false
var lastErr error
for _ , ep := range p . repoInfo . Index . Mirrors {
ep += "v1/"
2017-01-26 00:54:18 +00:00
progress . Updatef ( p . config . ProgressOutput , stringid . TruncateID ( img . ID ) , fmt . Sprintf ( "Pulling image (%s) from %s, mirror: %s" , img . Tag , p . repoInfo . Name . Name ( ) , ep ) )
2015-11-14 00:59:01 +00:00
if err = p . pullImage ( ctx , img . ID , ep , localNameRef , layersDownloaded ) ; err != nil {
2015-11-18 22:18:44 +00:00
// Don't report errors when pulling from mirrors.
2017-01-26 00:54:18 +00:00
logrus . Debugf ( "Error pulling image (%s) from %s, mirror: %s, %s" , img . Tag , p . repoInfo . Name . Name ( ) , ep , err )
2015-11-18 22:18:44 +00:00
continue
}
success = true
break
}
if ! success {
for _ , ep := range repoData . Endpoints {
2017-01-26 00:54:18 +00:00
progress . Updatef ( p . config . ProgressOutput , stringid . TruncateID ( img . ID ) , "Pulling image (%s) from %s, endpoint: %s" , img . Tag , p . repoInfo . Name . Name ( ) , ep )
2015-11-14 00:59:01 +00:00
if err = p . pullImage ( ctx , img . ID , ep , localNameRef , layersDownloaded ) ; err != nil {
2015-11-18 22:18:44 +00:00
// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
// As the error is also given to the output stream the user will see the error.
lastErr = err
2017-01-26 00:54:18 +00:00
progress . Updatef ( p . config . ProgressOutput , stringid . TruncateID ( img . ID ) , "Error pulling image (%s) from %s, endpoint: %s, %s" , img . Tag , p . repoInfo . Name . Name ( ) , ep , err )
2015-11-18 22:18:44 +00:00
continue
}
success = true
break
}
}
if ! success {
2017-01-26 00:54:18 +00:00
err := fmt . Errorf ( "Error pulling image (%s) from %s, %v" , img . Tag , p . repoInfo . Name . Name ( ) , lastErr )
2015-11-14 00:59:01 +00:00
progress . Update ( p . config . ProgressOutput , stringid . TruncateID ( img . ID ) , err . Error ( ) )
return err
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
return nil
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
func ( p * v1Puller ) pullImage ( ctx context . Context , v1ID , endpoint string , localNameRef reference . Named , layersDownloaded * bool ) ( err error ) {
2015-11-18 22:18:44 +00:00
var history [ ] string
history , err = p . session . GetRemoteHistory ( v1ID , endpoint )
if err != nil {
2015-11-14 00:59:01 +00:00
return err
2015-11-18 22:18:44 +00:00
}
if len ( history ) < 1 {
2015-11-14 00:59:01 +00:00
return fmt . Errorf ( "empty history for image %s" , v1ID )
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
progress . Update ( p . config . ProgressOutput , stringid . TruncateID ( v1ID ) , "Pulling dependent layers" )
2015-11-18 22:18:44 +00:00
var (
2015-11-14 00:59:01 +00:00
descriptors [ ] xfer . DownloadDescriptor
newHistory [ ] image . History
imgJSON [ ] byte
imgSize int64
2015-11-18 22:18:44 +00:00
)
// Iterate over layers, in order from bottom-most to top-most. Download
2015-11-14 00:59:01 +00:00
// config for all layers and create descriptors.
for i := len ( history ) - 1 ; i >= 0 ; i -- {
2015-11-18 22:18:44 +00:00
v1LayerID := history [ i ]
2015-11-14 00:59:01 +00:00
imgJSON , imgSize , err = p . downloadLayerConfig ( v1LayerID , endpoint )
2015-11-18 22:18:44 +00:00
if err != nil {
2015-11-14 00:59:01 +00:00
return err
2015-11-18 22:18:44 +00:00
}
// Create a new-style config from the legacy configs
h , err := v1 . HistoryFromConfig ( imgJSON , false )
if err != nil {
2015-11-14 00:59:01 +00:00
return err
2015-11-18 22:18:44 +00:00
}
newHistory = append ( newHistory , h )
2015-11-14 00:59:01 +00:00
layerDescriptor := & v1LayerDescriptor {
v1LayerID : v1LayerID ,
indexName : p . repoInfo . Index . Name ,
endpoint : endpoint ,
v1IDService : p . v1IDService ,
layersDownloaded : layersDownloaded ,
layerSize : imgSize ,
session : p . session ,
}
descriptors = append ( descriptors , layerDescriptor )
2015-11-18 22:18:44 +00:00
}
rootFS := image . NewRootFS ( )
2017-04-25 23:45:42 +00:00
resultRootFS , release , err := p . config . DownloadManager . Download ( ctx , * rootFS , "" , descriptors , p . config . ProgressOutput )
2015-11-14 00:59:01 +00:00
if err != nil {
return err
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
defer release ( )
2015-11-18 22:18:44 +00:00
2015-11-14 00:59:01 +00:00
config , err := v1 . MakeConfigFromV1Config ( imgJSON , & resultRootFS , newHistory )
2015-11-18 22:18:44 +00:00
if err != nil {
2015-11-14 00:59:01 +00:00
return err
2015-11-18 22:18:44 +00:00
}
2016-12-16 19:19:05 +00:00
imageID , err := p . config . ImageStore . Put ( config )
2015-11-18 22:18:44 +00:00
if err != nil {
2015-11-14 00:59:01 +00:00
return err
2015-11-18 22:18:44 +00:00
}
2016-12-16 19:19:05 +00:00
if p . config . ReferenceStore != nil {
if err := p . config . ReferenceStore . AddTag ( localNameRef , imageID , true ) ; err != nil {
return err
}
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
return nil
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
func ( p * v1Puller ) downloadLayerConfig ( v1LayerID , endpoint string ) ( imgJSON [ ] byte , imgSize int64 , err error ) {
progress . Update ( p . config . ProgressOutput , stringid . TruncateID ( v1LayerID ) , "Pulling metadata" )
2015-11-18 22:18:44 +00:00
retries := 5
for j := 1 ; j <= retries ; j ++ {
imgJSON , imgSize , err := p . session . GetRemoteImageJSON ( v1LayerID , endpoint )
if err != nil && j == retries {
2015-11-14 00:59:01 +00:00
progress . Update ( p . config . ProgressOutput , stringid . TruncateID ( v1LayerID ) , "Error pulling layer metadata" )
2015-11-18 22:18:44 +00:00
return nil , 0 , err
} else if err != nil {
time . Sleep ( time . Duration ( j ) * 500 * time . Millisecond )
continue
}
return imgJSON , imgSize , nil
}
// not reached
return nil , 0 , nil
}
2015-11-14 00:59:01 +00:00
type v1LayerDescriptor struct {
v1LayerID string
indexName string
endpoint string
v1IDService * metadata . V1IDService
layersDownloaded * bool
layerSize int64
session * registry . Session
2016-01-26 02:20:18 +00:00
tmpFile * os . File
2015-11-14 00:59:01 +00:00
}
2015-11-18 22:18:44 +00:00
2015-11-14 00:59:01 +00:00
func ( ld * v1LayerDescriptor ) Key ( ) string {
return "v1:" + ld . v1LayerID
}
2015-11-18 22:18:44 +00:00
2015-11-14 00:59:01 +00:00
func ( ld * v1LayerDescriptor ) ID ( ) string {
return stringid . TruncateID ( ld . v1LayerID )
}
func ( ld * v1LayerDescriptor ) DiffID ( ) ( layer . DiffID , error ) {
return ld . v1IDService . Get ( ld . v1LayerID , ld . indexName )
}
func ( ld * v1LayerDescriptor ) Download ( ctx context . Context , progressOutput progress . Output ) ( io . ReadCloser , int64 , error ) {
progress . Update ( progressOutput , ld . ID ( ) , "Pulling fs layer" )
layerReader , err := ld . session . GetRemoteImageLayer ( ld . v1LayerID , ld . endpoint , ld . layerSize )
if err != nil {
progress . Update ( progressOutput , ld . ID ( ) , "Error pulling dependent layers" )
2015-11-18 22:18:44 +00:00
if uerr , ok := err . ( * url . Error ) ; ok {
err = uerr . Err
}
2015-11-14 00:59:01 +00:00
if terr , ok := err . ( net . Error ) ; ok && terr . Timeout ( ) {
return nil , 0 , err
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
return nil , 0 , xfer . DoNotRetry { Err : err }
}
* ld . layersDownloaded = true
2015-11-18 22:18:44 +00:00
2016-01-26 02:20:18 +00:00
ld . tmpFile , err = ioutil . TempFile ( "" , "GetImageBlob" )
2015-11-14 00:59:01 +00:00
if err != nil {
layerReader . Close ( )
return nil , 0 , err
}
2015-11-18 22:18:44 +00:00
2015-11-14 00:59:01 +00:00
reader := progress . NewProgressReader ( ioutils . NewCancelReadCloser ( ctx , layerReader ) , progressOutput , ld . layerSize , ld . ID ( ) , "Downloading" )
defer reader . Close ( )
2015-11-18 22:18:44 +00:00
2016-01-26 02:20:18 +00:00
_ , err = io . Copy ( ld . tmpFile , reader )
2015-11-14 00:59:01 +00:00
if err != nil {
2016-01-26 02:20:18 +00:00
ld . Close ( )
2015-11-14 00:59:01 +00:00
return nil , 0 , err
2015-11-18 22:18:44 +00:00
}
2015-11-14 00:59:01 +00:00
progress . Update ( progressOutput , ld . ID ( ) , "Download complete" )
2016-01-26 02:20:18 +00:00
logrus . Debugf ( "Downloaded %s to tempfile %s" , ld . ID ( ) , ld . tmpFile . Name ( ) )
2015-11-14 00:59:01 +00:00
2016-01-26 02:20:18 +00:00
ld . tmpFile . Seek ( 0 , 0 )
2016-03-31 02:26:12 +00:00
// hand off the temporary file to the download manager, so it will only
// be closed once
tmpFile := ld . tmpFile
ld . tmpFile = nil
return ioutils . NewReadCloserWrapper ( tmpFile , func ( ) error {
tmpFile . Close ( )
err := os . RemoveAll ( tmpFile . Name ( ) )
if err != nil {
logrus . Errorf ( "Failed to remove temp file: %s" , tmpFile . Name ( ) )
}
return err
} ) , ld . layerSize , nil
2016-01-26 02:20:18 +00:00
}
func ( ld * v1LayerDescriptor ) Close ( ) {
if ld . tmpFile != nil {
ld . tmpFile . Close ( )
if err := os . RemoveAll ( ld . tmpFile . Name ( ) ) ; err != nil {
logrus . Errorf ( "Failed to remove temp file: %s" , ld . tmpFile . Name ( ) )
}
ld . tmpFile = nil
}
2015-11-14 00:59:01 +00:00
}
func ( ld * v1LayerDescriptor ) Registered ( diffID layer . DiffID ) {
// Cache mapping from this layer's DiffID to the blobsum
ld . v1IDService . Set ( ld . v1LayerID , ld . indexName , diffID )
2015-11-18 22:18:44 +00:00
}