2014-03-07 21:04:38 -05:00
package graph
2013-03-18 03:15:35 -04:00
import (
2015-03-30 14:19:12 -04:00
"compress/gzip"
2015-06-05 18:31:10 -04:00
"encoding/json"
2015-06-05 21:07:41 -04:00
"errors"
2013-03-18 03:15:35 -04:00
"fmt"
2013-05-06 14:06:44 -04:00
"io"
2013-03-18 03:15:35 -04:00
"io/ioutil"
"os"
"path/filepath"
2013-12-20 11:20:08 -05:00
"runtime"
2015-06-05 18:31:10 -04:00
"strconv"
2013-03-26 08:28:17 -04:00
"strings"
2015-04-07 22:29:29 -04:00
"sync"
2013-03-18 03:15:35 -04:00
"time"
2014-05-19 18:04:51 -04:00
2015-03-26 18:22:04 -04:00
"github.com/Sirupsen/logrus"
2015-03-30 14:19:12 -04:00
"github.com/docker/distribution/digest"
2014-07-24 18:19:50 -04:00
"github.com/docker/docker/daemon/graphdriver"
2015-11-09 13:32:46 -05:00
"github.com/docker/docker/dockerversion"
2014-07-24 18:19:50 -04:00
"github.com/docker/docker/image"
2014-09-30 02:23:36 -04:00
"github.com/docker/docker/pkg/archive"
2015-10-08 11:51:41 -04:00
"github.com/docker/docker/pkg/idtools"
2015-11-07 21:23:12 -05:00
"github.com/docker/docker/pkg/locker"
2015-02-24 03:51:46 -05:00
"github.com/docker/docker/pkg/progressreader"
2015-03-17 22:18:41 -04:00
"github.com/docker/docker/pkg/streamformatter"
2015-03-24 07:25:26 -04:00
"github.com/docker/docker/pkg/stringid"
2014-07-24 18:19:50 -04:00
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/runconfig"
2015-07-20 21:32:14 -04:00
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
2013-03-18 03:15:35 -04:00
)
2015-08-26 17:58:56 -04:00
// v1Descriptor is a non-content-addressable image descriptor
type v1Descriptor struct {
img * image . Image
}
// ID returns the image ID specified in the image structure.
func ( img v1Descriptor ) ID ( ) string {
return img . img . ID
}
// Parent returns the parent ID specified in the image structure.
func ( img v1Descriptor ) Parent ( ) string {
return img . img . Parent
}
// MarshalConfig renders the image structure into JSON.
func ( img v1Descriptor ) MarshalConfig ( ) ( [ ] byte , error ) {
return json . Marshal ( img . img )
}
2015-04-07 22:29:29 -04:00
// The type is used to protect pulling or building related image
// layers from deleteing when filtered by dangling=true
// The key of layers is the images ID which is pulling or building
// The value of layers is a slice which hold layer IDs referenced to
// pulling or building images
type retainedLayers struct {
layerHolders map [ string ] map [ string ] struct { } // map[layerID]map[sessionID]
sync . Mutex
}
func ( r * retainedLayers ) Add ( sessionID string , layerIDs [ ] string ) {
r . Lock ( )
defer r . Unlock ( )
for _ , layerID := range layerIDs {
if r . layerHolders [ layerID ] == nil {
r . layerHolders [ layerID ] = map [ string ] struct { } { }
}
r . layerHolders [ layerID ] [ sessionID ] = struct { } { }
}
}
func ( r * retainedLayers ) Delete ( sessionID string , layerIDs [ ] string ) {
r . Lock ( )
defer r . Unlock ( )
for _ , layerID := range layerIDs {
holders , ok := r . layerHolders [ layerID ]
if ! ok {
continue
}
delete ( holders , sessionID )
if len ( holders ) == 0 {
delete ( r . layerHolders , layerID ) // Delete any empty reference set.
}
}
}
func ( r * retainedLayers ) Exists ( layerID string ) bool {
r . Lock ( )
_ , exists := r . layerHolders [ layerID ]
r . Unlock ( )
return exists
}
2013-03-30 03:22:24 -04:00
// A Graph is a store for versioned filesystem images and the relationship between them.
2013-03-18 03:15:35 -04:00
type Graph struct {
2015-07-24 20:49:43 -04:00
root string
idIndex * truncindex . TruncIndex
driver graphdriver . Driver
2015-10-11 03:32:52 -04:00
imagesMutex sync . Mutex
2015-11-07 21:23:12 -05:00
imageMutex locker . Locker // protect images in driver.
2015-07-24 20:49:43 -04:00
retained * retainedLayers
tarSplitDisabled bool
2015-10-08 11:51:41 -04:00
uidMaps [ ] idtools . IDMap
gidMaps [ ] idtools . IDMap
2015-10-11 03:32:52 -04:00
// access to parentRefs must be protected with imageMutex locking the image id
// on the key of the map (e.g. imageMutex.Lock(img.ID), parentRefs[img.ID]...)
parentRefs map [ string ] int
2013-03-18 03:15:35 -04:00
}
2015-06-23 19:44:07 -04:00
// file names for ./graph/<ID>/
const (
2015-08-26 17:58:56 -04:00
jsonFileName = "json"
layersizeFileName = "layersize"
digestFileName = "checksum"
tarDataFileName = "tar-data.json.gz"
v1CompatibilityFileName = "v1Compatibility"
parentFileName = "parent"
2015-06-23 19:44:07 -04:00
)
2015-06-05 21:07:41 -04:00
var (
2015-11-08 15:30:31 -05:00
// errDigestNotSet is used when request the digest for a layer
2015-06-05 21:07:41 -04:00
// but the layer has no digest value or content to compute the
// the digest.
2015-11-08 15:30:31 -05:00
errDigestNotSet = errors . New ( "digest is not set for layer" )
2015-06-05 21:07:41 -04:00
)
2013-03-30 03:22:24 -04:00
// NewGraph instantiates a new graph at the given root path in the filesystem.
2013-03-30 00:13:59 -04:00
// `root` will be created if it doesn't exist.
2015-10-08 11:51:41 -04:00
func NewGraph ( root string , driver graphdriver . Driver , uidMaps , gidMaps [ ] idtools . IDMap ) ( * Graph , error ) {
2013-03-18 03:15:35 -04:00
abspath , err := filepath . Abs ( root )
if err != nil {
return nil , err
}
2015-10-08 11:51:41 -04:00
rootUID , rootGID , err := idtools . GetRootUIDGID ( uidMaps , gidMaps )
if err != nil {
return nil , err
}
2013-03-18 03:15:35 -04:00
// Create the root directory if it doesn't exists
2015-10-08 11:51:41 -04:00
if err := idtools . MkdirAllAs ( root , 0700 , rootUID , rootGID ) ; err != nil && ! os . IsExist ( err ) {
2013-03-18 03:15:35 -04:00
return nil , err
}
2013-11-04 18:22:34 -05:00
2013-04-01 01:11:55 -04:00
graph := & Graph {
2015-10-09 13:13:45 -04:00
root : abspath ,
idIndex : truncindex . NewTruncIndex ( [ ] string { } ) ,
driver : driver ,
retained : & retainedLayers { layerHolders : make ( map [ string ] map [ string ] struct { } ) } ,
uidMaps : uidMaps ,
gidMaps : gidMaps ,
parentRefs : make ( map [ string ] int ) ,
2013-04-01 01:11:55 -04:00
}
2015-07-24 20:49:43 -04:00
// Windows does not currently support tarsplit functionality.
if runtime . GOOS == "windows" {
graph . tarSplitDisabled = true
}
2013-04-01 01:11:55 -04:00
if err := graph . restore ( ) ; err != nil {
return nil , err
}
return graph , nil
}
2015-04-07 22:29:29 -04:00
// IsHeld returns whether the given layerID is being used by an ongoing pull or build.
func ( graph * Graph ) IsHeld ( layerID string ) bool {
return graph . retained . Exists ( layerID )
}
2013-04-01 01:11:55 -04:00
func ( graph * Graph ) restore ( ) error {
2015-06-05 18:31:10 -04:00
dir , err := ioutil . ReadDir ( graph . root )
2013-04-01 01:11:55 -04:00
if err != nil {
return err
}
2014-04-11 16:39:58 -04:00
var ids = [ ] string { }
2013-04-01 01:11:55 -04:00
for _ , v := range dir {
id := v . Name ( )
2013-11-19 05:32:08 -05:00
if graph . driver . Exists ( id ) {
2015-10-11 03:32:52 -04:00
img , err := graph . loadImage ( id )
2015-10-09 13:13:45 -04:00
if err != nil {
2015-11-04 13:20:12 -05:00
logrus . Warnf ( "ignoring image %s, it could not be restored: %v" , id , err )
2015-11-02 19:05:45 -05:00
continue
2015-10-09 13:13:45 -04:00
}
graph . imageMutex . Lock ( img . Parent )
graph . parentRefs [ img . Parent ] ++
graph . imageMutex . Unlock ( img . Parent )
2014-04-11 16:39:58 -04:00
ids = append ( ids , id )
2013-11-19 05:32:08 -05:00
}
2013-04-01 01:11:55 -04:00
}
2015-06-11 14:29:29 -04:00
2014-06-24 13:19:15 -04:00
graph . idIndex = truncindex . NewTruncIndex ( ids )
2015-06-10 18:18:51 -04:00
logrus . Debugf ( "Restored %d elements" , len ( ids ) )
2013-04-01 01:11:55 -04:00
return nil
2013-03-18 03:15:35 -04:00
}
2015-07-29 19:45:47 -04:00
// IsNotExist detects whether an image exists by parsing the incoming error
// message.
2015-03-27 21:07:20 -04:00
func ( graph * Graph ) IsNotExist ( err error , id string ) bool {
2015-07-29 19:45:47 -04:00
// FIXME: Implement error subclass instead of looking at the error text
// Note: This is the way golang implements os.IsNotExists on Plan9
2015-03-27 21:07:20 -04:00
return err != nil && ( strings . Contains ( strings . ToLower ( err . Error ( ) ) , "does not exist" ) || strings . Contains ( strings . ToLower ( err . Error ( ) ) , "no such" ) ) && strings . Contains ( err . Error ( ) , id )
2013-03-26 08:28:17 -04:00
}
2013-03-30 00:13:59 -04:00
// Exists returns true if an image is registered at the given id.
// If the image doesn't exist or if an error is encountered, false is returned.
2013-03-18 03:15:35 -04:00
func ( graph * Graph ) Exists ( id string ) bool {
if _ , err := graph . Get ( id ) ; err != nil {
return false
}
return true
}
2013-03-30 00:13:59 -04:00
// Get returns the image with the given id, or an error if the image doesn't exist.
2015-07-20 13:57:15 -04:00
func ( graph * Graph ) Get ( name string ) ( * image . Image , error ) {
2013-04-01 01:11:55 -04:00
id , err := graph . idIndex . Get ( name )
if err != nil {
2015-11-03 20:19:18 -05:00
if err == truncindex . ErrNotExist {
return nil , fmt . Errorf ( "image %s does not exist" , name )
}
return nil , err
2013-04-01 01:11:55 -04:00
}
2015-06-05 18:31:10 -04:00
img , err := graph . loadImage ( id )
2013-03-18 03:15:35 -04:00
if err != nil {
return nil , err
}
2013-06-04 14:00:22 -04:00
if img . ID != id {
return nil , fmt . Errorf ( "Image stored at '%s' has wrong id '%s'" , id , img . ID )
2013-03-18 03:15:35 -04:00
}
2013-12-05 17:03:23 -05:00
if img . Size < 0 {
2014-09-10 23:30:52 -04:00
size , err := graph . driver . DiffSize ( img . ID , img . Parent )
2014-01-07 14:34:19 -05:00
if err != nil {
2014-09-10 23:30:52 -04:00
return nil , fmt . Errorf ( "unable to calculate size of image id %q: %s" , img . ID , err )
2013-05-13 09:10:26 -04:00
}
2013-12-05 17:03:23 -05:00
2013-11-07 15:34:01 -05:00
img . Size = size
2015-07-23 17:19:58 -04:00
if err := graph . saveSize ( graph . imageRoot ( id ) , img . Size ) ; err != nil {
2013-05-13 09:10:26 -04:00
return nil , err
}
}
2013-03-18 03:15:35 -04:00
return img , nil
}
2013-03-30 00:13:59 -04:00
// Create creates a new image and registers it in the graph.
2015-08-20 17:20:07 -04:00
func ( graph * Graph ) Create ( layerData io . Reader , containerID , containerImage , comment , author string , containerConfig , config * runconfig . Config ) ( * image . Image , error ) {
2015-07-20 13:57:15 -04:00
img := & image . Image {
2015-03-24 07:25:26 -04:00
ID : stringid . GenerateRandomID ( ) ,
2013-04-04 21:38:43 -04:00
Comment : comment ,
2013-11-21 19:41:41 -05:00
Created : time . Now ( ) . UTC ( ) ,
2015-11-09 13:32:46 -05:00
DockerVersion : dockerversion . Version ,
2013-04-17 22:58:17 -04:00
Author : author ,
2013-04-25 19:48:31 -04:00
Config : config ,
2013-12-20 11:20:08 -05:00
Architecture : runtime . GOARCH ,
OS : runtime . GOOS ,
2013-03-18 03:15:35 -04:00
}
2014-05-19 18:04:51 -04:00
2014-03-07 21:04:38 -05:00
if containerID != "" {
img . Parent = containerImage
img . Container = containerID
img . ContainerConfig = * containerConfig
2013-03-22 00:13:27 -04:00
}
2014-05-19 18:04:51 -04:00
2015-08-26 17:58:56 -04:00
if err := graph . Register ( v1Descriptor { img } , layerData ) ; err != nil {
2013-03-18 03:15:35 -04:00
return nil , err
}
return img , nil
}
2013-03-30 00:13:59 -04:00
// Register imports a pre-existing image into the graph.
2015-08-20 13:25:06 -04:00
// Returns nil if the image is already registered.
2015-08-26 17:58:56 -04:00
func ( graph * Graph ) Register ( im image . Descriptor , layerData io . Reader ) ( err error ) {
imgID := im . ID ( )
2015-07-17 10:01:52 -04:00
2015-08-26 17:58:56 -04:00
if err := image . ValidateID ( imgID ) ; err != nil {
2015-06-04 15:58:58 -04:00
return err
}
2015-10-11 03:32:52 -04:00
// this is needed cause pull_v2 attemptIDReuse could deadlock
graph . imagesMutex . Lock ( )
defer graph . imagesMutex . Unlock ( )
2015-06-04 15:58:58 -04:00
// We need this entire operation to be atomic within the engine. Note that
// this doesn't mean Register is fully safe yet.
2015-08-26 17:58:56 -04:00
graph . imageMutex . Lock ( imgID )
defer graph . imageMutex . Unlock ( imgID )
return graph . register ( im , layerData )
}
func ( graph * Graph ) register ( im image . Descriptor , layerData io . Reader ) ( err error ) {
imgID := im . ID ( )
2015-06-04 15:58:58 -04:00
2015-08-20 13:25:06 -04:00
// Skip register if image is already registered
2015-08-26 17:58:56 -04:00
if graph . Exists ( imgID ) {
2015-08-20 13:25:06 -04:00
return nil
}
2015-07-17 10:01:52 -04:00
// The returned `error` must be named in this function's signature so that
// `err` is not shadowed in this deferred cleanup.
2013-11-25 23:48:34 -05:00
defer func ( ) {
// If any error occurs, remove the new dir from the driver.
// Don't check for errors since the dir might not have been created.
if err != nil {
2015-08-26 17:58:56 -04:00
graph . driver . Remove ( imgID )
2013-11-25 23:48:34 -05:00
}
} ( )
2015-06-04 15:58:58 -04:00
2013-11-19 05:32:08 -05:00
// Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph.
// This is common when you switch from one graph driver to another
2015-08-26 17:58:56 -04:00
if err := os . RemoveAll ( graph . imageRoot ( imgID ) ) ; err != nil && ! os . IsNotExist ( err ) {
2013-11-19 05:32:08 -05:00
return err
}
2013-11-25 23:48:34 -05:00
// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
// (the graph is the source of truth).
// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
// (FIXME: make that mandatory for drivers).
2015-08-26 17:58:56 -04:00
graph . driver . Remove ( imgID )
2013-11-25 23:48:34 -05:00
2015-09-15 08:55:24 -04:00
tmp , err := graph . mktemp ( )
2013-03-18 03:15:35 -04:00
if err != nil {
2015-10-11 03:32:52 -04:00
return err
2013-03-18 03:15:35 -04:00
}
2015-10-11 03:32:52 -04:00
defer os . RemoveAll ( tmp )
2013-11-07 15:34:01 -05:00
2015-08-26 17:58:56 -04:00
parent := im . Parent ( )
2013-11-07 15:34:01 -05:00
// Create root filesystem in the driver
2015-10-23 03:07:06 -04:00
if err := createRootFilesystemInDriver ( graph , imgID , parent ) ; err != nil {
2015-06-11 14:29:29 -04:00
return err
2013-11-07 15:34:01 -05:00
}
2015-06-11 14:29:29 -04:00
2014-09-10 23:30:52 -04:00
// Apply the diff/layer
2015-08-26 17:58:56 -04:00
config , err := im . MarshalConfig ( )
if err != nil {
return err
}
if err := graph . storeImage ( imgID , parent , config , layerData , tmp ) ; err != nil {
2013-03-18 03:15:35 -04:00
return err
}
// Commit
2015-08-26 17:58:56 -04:00
if err := os . Rename ( tmp , graph . imageRoot ( imgID ) ) ; err != nil {
2013-03-18 03:15:35 -04:00
return err
}
2015-10-09 13:13:45 -04:00
2015-10-11 03:32:52 -04:00
graph . idIndex . Add ( imgID )
2015-10-09 13:13:45 -04:00
2015-10-11 03:32:52 -04:00
graph . imageMutex . Lock ( parent )
graph . parentRefs [ parent ] ++
graph . imageMutex . Unlock ( parent )
2015-10-09 13:13:45 -04:00
2013-03-18 03:15:35 -04:00
return nil
}
2015-10-23 03:07:06 -04:00
func createRootFilesystemInDriver ( graph * Graph , id , parent string ) error {
2015-08-26 17:58:56 -04:00
if err := graph . driver . Create ( id , parent ) ; err != nil {
return fmt . Errorf ( "Driver %s failed to create image rootfs %s: %s" , graph . driver , id , err )
2015-07-24 20:49:43 -04:00
}
return nil
}
2013-04-21 17:23:55 -04:00
// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
// The archive is stored on disk and will be automatically deleted as soon as has been read.
2013-04-21 18:29:26 -04:00
// If output is not nil, a human-readable progress bar will be written to it.
2015-11-08 15:30:31 -05:00
func ( graph * Graph ) tempLayerArchive ( id string , sf * streamformatter . StreamFormatter , output io . Writer ) ( * archive . TempArchive , error ) {
2013-04-21 17:23:55 -04:00
image , err := graph . Get ( id )
if err != nil {
return nil , err
}
2015-09-15 08:55:24 -04:00
tmp , err := graph . mktemp ( )
2013-04-21 17:23:55 -04:00
if err != nil {
return nil , err
}
2015-10-11 03:32:52 -04:00
defer os . RemoveAll ( tmp )
2015-11-08 15:30:31 -05:00
a , err := graph . tarLayer ( image )
2013-04-21 17:23:55 -04:00
if err != nil {
return nil , err
}
2015-02-24 03:51:46 -05:00
progressReader := progressreader . New ( progressreader . Config {
In : a ,
Out : output ,
Formatter : sf ,
Size : 0 ,
NewLines : false ,
2015-03-24 07:25:26 -04:00
ID : stringid . TruncateID ( id ) ,
2015-02-24 03:51:46 -05:00
Action : "Buffering to disk" ,
} )
defer progressReader . Close ( )
return archive . NewTempArchive ( progressReader , tmp )
2013-04-21 17:23:55 -04:00
}
2015-06-05 18:31:10 -04:00
// mktemp creates a temporary sub-directory inside the graph's filesystem.
2015-09-15 08:55:24 -04:00
func ( graph * Graph ) mktemp ( ) ( string , error ) {
2015-07-28 20:19:17 -04:00
dir := filepath . Join ( graph . root , "_tmp" , stringid . GenerateNonCryptoID ( ) )
2015-10-08 11:51:41 -04:00
rootUID , rootGID , err := idtools . GetRootUIDGID ( graph . uidMaps , graph . gidMaps )
if err != nil {
return "" , err
}
if err := idtools . MkdirAllAs ( dir , 0700 , rootUID , rootGID ) ; err != nil {
2013-11-08 19:53:58 -05:00
return "" , err
2013-03-18 03:15:35 -04:00
}
2013-11-07 15:34:01 -05:00
return dir , nil
2013-03-18 03:15:35 -04:00
}
2013-03-30 00:13:59 -04:00
// Delete atomically removes an image from the graph.
2013-04-01 19:04:44 -04:00
func ( graph * Graph ) Delete ( name string ) error {
id , err := graph . idIndex . Get ( name )
if err != nil {
return err
}
2015-10-09 13:13:45 -04:00
img , err := graph . Get ( id )
if err != nil {
return err
}
2013-04-01 01:11:55 -04:00
graph . idIndex . Delete ( id )
2015-10-11 03:32:52 -04:00
tmp , err := graph . mktemp ( )
if err != nil {
tmp = graph . imageRoot ( id )
} else {
2015-06-05 18:31:10 -04:00
if err := os . Rename ( graph . imageRoot ( id ) , tmp ) ; err != nil {
2015-04-26 12:50:25 -04:00
// On err make tmp point to old dir and cleanup unused tmp dir
2014-09-22 10:47:20 -04:00
os . RemoveAll ( tmp )
2015-06-05 18:31:10 -04:00
tmp = graph . imageRoot ( id )
2014-09-22 10:47:20 -04:00
}
2013-03-18 03:15:35 -04:00
}
2013-11-07 15:34:01 -05:00
// Remove rootfs data from the driver
graph . driver . Remove ( id )
2015-10-09 13:13:45 -04:00
graph . imageMutex . Lock ( img . Parent )
graph . parentRefs [ img . Parent ] --
if graph . parentRefs [ img . Parent ] == 0 {
delete ( graph . parentRefs , img . Parent )
}
graph . imageMutex . Unlock ( img . Parent )
2013-11-07 15:34:01 -05:00
// Remove the trashed image directory
2013-04-04 01:14:28 -04:00
return os . RemoveAll ( tmp )
2013-03-18 03:15:35 -04:00
}
2013-03-30 03:22:24 -04:00
// Map returns a list of all images in the graph, addressable by ID.
2015-07-20 13:57:15 -04:00
func ( graph * Graph ) Map ( ) map [ string ] * image . Image {
images := make ( map [ string ] * image . Image )
graph . walkAll ( func ( image * image . Image ) {
2013-08-31 23:31:21 -04:00
images [ image . ID ] = image
} )
2015-06-19 11:01:39 -04:00
return images
2013-03-21 20:35:49 -04:00
}
2013-08-31 23:34:51 -04:00
// walkAll iterates over each image in the graph, and passes it to a handler.
2013-03-30 00:13:59 -04:00
// The walking order is undetermined.
2015-07-20 13:57:15 -04:00
func ( graph * Graph ) walkAll ( handler func ( * image . Image ) ) {
2015-06-19 11:01:39 -04:00
graph . idIndex . Iterate ( func ( id string ) {
2015-10-09 13:13:45 -04:00
img , err := graph . Get ( id )
if err != nil {
2015-06-19 11:01:39 -04:00
return
2015-10-09 13:13:45 -04:00
}
if handler != nil {
2013-03-23 20:03:30 -04:00
handler ( img )
}
2015-06-19 11:01:39 -04:00
} )
2013-03-23 20:03:30 -04:00
}
2013-03-30 00:13:59 -04:00
// ByParent returns a lookup table of images by their parent.
2015-08-13 07:58:17 -04:00
// If an image of key ID has 3 children images, then the value for key ID
2013-03-30 00:13:59 -04:00
// will be a list of 3 images.
// If an image has no children, it will not have an entry in the table.
2015-07-20 13:57:15 -04:00
func ( graph * Graph ) ByParent ( ) map [ string ] [ ] * image . Image {
byParent := make ( map [ string ] [ ] * image . Image )
graph . walkAll ( func ( img * image . Image ) {
2014-03-07 20:36:47 -05:00
parent , err := graph . Get ( img . Parent )
2013-03-23 20:03:30 -04:00
if err != nil {
return
}
2013-06-04 14:00:22 -04:00
if children , exists := byParent [ parent . ID ] ; exists {
2014-03-07 20:36:47 -05:00
byParent [ parent . ID ] = append ( children , img )
2013-08-11 03:37:16 -04:00
} else {
2015-07-20 13:57:15 -04:00
byParent [ parent . ID ] = [ ] * image . Image { img }
2013-03-18 03:15:35 -04:00
}
2013-03-23 20:03:30 -04:00
} )
2015-06-19 11:01:39 -04:00
return byParent
2013-03-23 20:03:30 -04:00
}
2015-08-15 03:30:25 -04:00
// HasChildren returns whether the given image has any child images.
2015-10-09 13:13:45 -04:00
func ( graph * Graph ) HasChildren ( imgID string ) bool {
graph . imageMutex . Lock ( imgID )
count := graph . parentRefs [ imgID ]
graph . imageMutex . Unlock ( imgID )
return count > 0
2015-08-15 03:30:25 -04:00
}
2015-07-29 19:45:47 -04:00
// Retain keeps the images and layers that are in the pulling chain so that
// they are not deleted. If not retained, they may be deleted by rmi.
2015-04-07 22:29:29 -04:00
func ( graph * Graph ) Retain ( sessionID string , layerIDs ... string ) {
graph . retained . Add ( sessionID , layerIDs )
}
2015-07-29 19:45:47 -04:00
// Release removes the referenced image ID from the provided set of layers.
2015-04-07 22:29:29 -04:00
func ( graph * Graph ) Release ( sessionID string , layerIDs ... string ) {
graph . retained . Delete ( sessionID , layerIDs )
}
2015-11-08 15:30:31 -05:00
// heads returns all heads in the graph, keyed by id.
2013-03-30 00:13:59 -04:00
// A head is an image which is not the parent of another image in the graph.
2015-11-08 15:30:31 -05:00
func ( graph * Graph ) heads ( ) map [ string ] * image . Image {
2015-07-20 13:57:15 -04:00
heads := make ( map [ string ] * image . Image )
graph . walkAll ( func ( image * image . Image ) {
2015-10-09 13:14:45 -04:00
// if it has no children, then it's not a parent, so it's an head
2015-10-09 13:13:45 -04:00
if ! graph . HasChildren ( image . ID ) {
2013-06-04 14:00:22 -04:00
heads [ image . ID ] = image
2013-03-23 20:03:30 -04:00
}
} )
2015-06-19 11:01:39 -04:00
return heads
2013-03-18 03:15:35 -04:00
}
2015-11-08 15:30:31 -05:00
// tarLayer returns a tar archive of the image's filesystem layer.
func ( graph * Graph ) tarLayer ( img * image . Image ) ( arch io . ReadCloser , err error ) {
2015-07-24 20:49:43 -04:00
rdr , err := graph . assembleTarLayer ( img )
if err != nil {
2015-11-08 15:30:31 -05:00
logrus . Debugf ( "[graph] tarLayer with traditional differ: %s" , img . ID )
2015-07-24 20:49:43 -04:00
return graph . driver . Diff ( img . ID , img . Parent )
}
return rdr , nil
}
2015-06-05 18:31:10 -04:00
func ( graph * Graph ) imageRoot ( id string ) string {
return filepath . Join ( graph . root , id )
2013-03-18 03:15:35 -04:00
}
2013-11-25 23:04:57 -05:00
2015-06-05 18:31:10 -04:00
// loadImage fetches the image with the given id from the graph.
2015-07-20 13:57:15 -04:00
func ( graph * Graph ) loadImage ( id string ) ( * image . Image , error ) {
2015-06-05 18:31:10 -04:00
root := graph . imageRoot ( id )
// Open the JSON file to decode by streaming
jsonSource , err := os . Open ( jsonPath ( root ) )
if err != nil {
return nil , err
}
defer jsonSource . Close ( )
2015-07-20 13:57:15 -04:00
img := & image . Image { }
2015-06-05 18:31:10 -04:00
dec := json . NewDecoder ( jsonSource )
// Decode the JSON data
if err := dec . Decode ( img ) ; err != nil {
return nil , err
}
2015-08-26 17:58:56 -04:00
if img . ID == "" {
img . ID = id
}
if img . Parent == "" && img . ParentID != "" && img . ParentID . Validate ( ) == nil {
img . Parent = img . ParentID . Hex ( )
}
// compatibilityID for parent
parent , err := ioutil . ReadFile ( filepath . Join ( root , parentFileName ) )
if err == nil && len ( parent ) > 0 {
img . Parent = string ( parent )
}
2015-06-05 18:31:10 -04:00
if err := image . ValidateID ( img . ID ) ; err != nil {
return nil , err
}
2015-06-23 19:44:07 -04:00
if buf , err := ioutil . ReadFile ( filepath . Join ( root , layersizeFileName ) ) ; err != nil {
2015-06-05 18:31:10 -04:00
if ! os . IsNotExist ( err ) {
return nil , err
}
// If the layersize file does not exist then set the size to a negative number
// because a layer size of 0 (zero) is valid
img . Size = - 1
} else {
// Using Atoi here instead would temporarily convert the size to a machine
// dependent integer type, which causes images larger than 2^31 bytes to
// display negative sizes on 32-bit machines:
size , err := strconv . ParseInt ( string ( buf ) , 10 , 64 )
if err != nil {
return nil , err
}
img . Size = int64 ( size )
}
return img , nil
}
// saveSize stores the `size` in the provided graph `img` directory `root`.
2015-07-23 17:19:58 -04:00
func ( graph * Graph ) saveSize ( root string , size int64 ) error {
if err := ioutil . WriteFile ( filepath . Join ( root , layersizeFileName ) , [ ] byte ( strconv . FormatInt ( size , 10 ) ) , 0600 ) ; err != nil {
2015-06-23 19:44:07 -04:00
return fmt . Errorf ( "Error storing image size in %s/%s: %s" , root , layersizeFileName , err )
2015-06-05 18:31:10 -04:00
}
return nil
}
2015-11-08 15:30:31 -05:00
// setLayerDigestWithLock sets the digest for the image layer to the provided value.
func ( graph * Graph ) setLayerDigestWithLock ( id string , dgst digest . Digest ) error {
2015-08-20 13:25:06 -04:00
graph . imageMutex . Lock ( id )
defer graph . imageMutex . Unlock ( id )
2015-08-26 17:58:56 -04:00
return graph . setLayerDigest ( id , dgst )
}
func ( graph * Graph ) setLayerDigest ( id string , dgst digest . Digest ) error {
2015-06-05 18:31:10 -04:00
root := graph . imageRoot ( id )
2015-06-23 19:44:07 -04:00
if err := ioutil . WriteFile ( filepath . Join ( root , digestFileName ) , [ ] byte ( dgst . String ( ) ) , 0600 ) ; err != nil {
return fmt . Errorf ( "Error storing digest in %s/%s: %s" , root , digestFileName , err )
2015-06-05 18:31:10 -04:00
}
return nil
}
2015-11-08 15:30:31 -05:00
// getLayerDigestWithLock gets the digest for the provide image layer id.
func ( graph * Graph ) getLayerDigestWithLock ( id string ) ( digest . Digest , error ) {
2015-08-20 13:25:06 -04:00
graph . imageMutex . Lock ( id )
defer graph . imageMutex . Unlock ( id )
2015-08-26 17:58:56 -04:00
return graph . getLayerDigest ( id )
}
func ( graph * Graph ) getLayerDigest ( id string ) ( digest . Digest , error ) {
2015-06-05 18:31:10 -04:00
root := graph . imageRoot ( id )
2015-06-23 19:44:07 -04:00
cs , err := ioutil . ReadFile ( filepath . Join ( root , digestFileName ) )
2015-06-05 18:31:10 -04:00
if err != nil {
if os . IsNotExist ( err ) {
2015-11-08 15:30:31 -05:00
return "" , errDigestNotSet
2015-06-05 18:31:10 -04:00
}
return "" , err
}
2015-06-05 21:07:41 -04:00
return digest . ParseDigest ( string ( cs ) )
2015-06-05 18:31:10 -04:00
}
2015-11-08 15:30:31 -05:00
// setV1CompatibilityConfig stores the v1Compatibility JSON data associated
2015-08-26 17:58:56 -04:00
// with the image in the manifest to the disk
func ( graph * Graph ) setV1CompatibilityConfig ( id string , data [ ] byte ) error {
root := graph . imageRoot ( id )
return ioutil . WriteFile ( filepath . Join ( root , v1CompatibilityFileName ) , data , 0600 )
}
2015-11-08 15:30:31 -05:00
// getV1CompatibilityConfig reads the v1Compatibility JSON data for the image
2015-08-26 17:58:56 -04:00
// from the disk
func ( graph * Graph ) getV1CompatibilityConfig ( id string ) ( [ ] byte , error ) {
root := graph . imageRoot ( id )
return ioutil . ReadFile ( filepath . Join ( root , v1CompatibilityFileName ) )
}
2015-11-08 15:30:31 -05:00
// generateV1CompatibilityChain makes sure v1Compatibility JSON data exists
2015-08-26 17:58:56 -04:00
// for the image. If it doesn't it generates and stores it for the image and
// all of it's parents based on the image config JSON.
2015-11-08 15:30:31 -05:00
func ( graph * Graph ) generateV1CompatibilityChain ( id string ) ( [ ] byte , error ) {
2015-08-26 17:58:56 -04:00
graph . imageMutex . Lock ( id )
defer graph . imageMutex . Unlock ( id )
if v1config , err := graph . getV1CompatibilityConfig ( id ) ; err == nil {
return v1config , nil
}
// generate new, store it to disk
img , err := graph . Get ( id )
if err != nil {
return nil , err
}
digestPrefix := string ( digest . Canonical ) + ":"
img . ID = strings . TrimPrefix ( img . ID , digestPrefix )
if img . Parent != "" {
2015-11-08 15:30:31 -05:00
parentConfig , err := graph . generateV1CompatibilityChain ( img . Parent )
2015-08-26 17:58:56 -04:00
if err != nil {
return nil , err
}
var parent struct { ID string }
err = json . Unmarshal ( parentConfig , & parent )
if err != nil {
return nil , err
}
img . Parent = parent . ID
}
json , err := json . Marshal ( img )
if err != nil {
return nil , err
}
if err := graph . setV1CompatibilityConfig ( id , json ) ; err != nil {
return nil , err
}
return json , nil
}
2015-06-05 18:31:10 -04:00
func jsonPath ( root string ) string {
2015-06-23 19:44:07 -04:00
return filepath . Join ( root , jsonFileName )
2013-03-18 03:15:35 -04:00
}
2015-07-20 21:32:14 -04:00
2015-07-24 20:49:43 -04:00
// storeImage stores file system layer data for the given image to the
// graph's storage driver. Image metadata is stored in a file
// at the specified root directory.
2015-08-26 17:58:56 -04:00
func ( graph * Graph ) storeImage ( id , parent string , config [ ] byte , layerData io . Reader , root string ) ( err error ) {
var size int64
2015-07-24 20:49:43 -04:00
// Store the layer. If layerData is not nil, unpack it into the new layer
if layerData != nil {
2015-08-26 17:58:56 -04:00
if size , err = graph . disassembleAndApplyTarLayer ( id , parent , layerData , root ) ; err != nil {
2015-07-24 20:49:43 -04:00
return err
}
2015-07-20 21:32:14 -04:00
}
2015-08-26 17:58:56 -04:00
if err := graph . saveSize ( root , size ) ; err != nil {
2015-07-20 21:32:14 -04:00
return err
}
2015-08-26 17:58:56 -04:00
if err := ioutil . WriteFile ( jsonPath ( root ) , config , 0600 ) ; err != nil {
2015-07-20 21:32:14 -04:00
return err
}
2015-08-26 17:58:56 -04:00
// If image is pointing to a parent via CompatibilityID write the reference to disk
img , err := image . NewImgJSON ( config )
if err != nil {
return err
}
2015-07-24 20:49:43 -04:00
2015-08-26 17:58:56 -04:00
if img . ParentID . Validate ( ) == nil && parent != img . ParentID . Hex ( ) {
if err := ioutil . WriteFile ( filepath . Join ( root , parentFileName ) , [ ] byte ( parent ) , 0600 ) ; err != nil {
return err
}
}
return nil
2015-07-24 20:49:43 -04:00
}
2015-08-26 17:58:56 -04:00
func ( graph * Graph ) disassembleAndApplyTarLayer ( id , parent string , layerData io . Reader , root string ) ( size int64 , err error ) {
2015-08-20 17:20:07 -04:00
var ar io . Reader
2015-07-24 20:49:43 -04:00
if graph . tarSplitDisabled {
ar = layerData
} else {
// this is saving the tar-split metadata
mf , err := os . OpenFile ( filepath . Join ( root , tarDataFileName ) , os . O_CREATE | os . O_WRONLY | os . O_TRUNC , os . FileMode ( 0600 ) )
if err != nil {
2015-08-26 17:58:56 -04:00
return 0 , err
2015-07-24 20:49:43 -04:00
}
mfz := gzip . NewWriter ( mf )
metaPacker := storage . NewJSONPacker ( mfz )
defer mf . Close ( )
defer mfz . Close ( )
inflatedLayerData , err := archive . DecompressStream ( layerData )
if err != nil {
2015-08-26 17:58:56 -04:00
return 0 , err
2015-07-24 20:49:43 -04:00
}
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr , err := asm . NewInputTarStream ( inflatedLayerData , metaPacker , nil )
if err != nil {
2015-08-26 17:58:56 -04:00
return 0 , err
2015-07-24 20:49:43 -04:00
}
ar = archive . Reader ( rdr )
}
2015-08-26 17:58:56 -04:00
if size , err = graph . driver . ApplyDiff ( id , parent , ar ) ; err != nil {
return 0 , err
2015-07-20 21:32:14 -04:00
}
2015-08-26 17:58:56 -04:00
return
2015-07-20 21:32:14 -04:00
}
2015-08-20 17:20:07 -04:00
func ( graph * Graph ) assembleTarLayer ( img * image . Image ) ( io . ReadCloser , error ) {
2015-07-20 21:32:14 -04:00
root := graph . imageRoot ( img . ID )
mFileName := filepath . Join ( root , tarDataFileName )
mf , err := os . Open ( mFileName )
if err != nil {
if ! os . IsNotExist ( err ) {
logrus . Errorf ( "failed to open %q: %s" , mFileName , err )
}
return nil , err
}
pR , pW := io . Pipe ( )
// this will need to be in a goroutine, as we are returning the stream of a
// tar archive, but can not close the metadata reader early (when this
// function returns)...
go func ( ) {
defer mf . Close ( )
// let's reassemble!
logrus . Debugf ( "[graph] TarLayer with reassembly: %s" , img . ID )
mfz , err := gzip . NewReader ( mf )
if err != nil {
pW . CloseWithError ( fmt . Errorf ( "[graph] error with %s: %s" , mFileName , err ) )
return
}
defer mfz . Close ( )
// get our relative path to the container
fsLayer , err := graph . driver . Get ( img . ID , "" )
if err != nil {
pW . CloseWithError ( err )
return
}
defer graph . driver . Put ( img . ID )
metaUnpacker := storage . NewJSONUnpacker ( mfz )
fileGetter := storage . NewPathFileGetter ( fsLayer )
logrus . Debugf ( "[graph] %s is at %q" , img . ID , fsLayer )
ots := asm . NewOutputTarStream ( fileGetter , metaUnpacker )
defer ots . Close ( )
if _ , err := io . Copy ( pW , ots ) ; err != nil {
pW . CloseWithError ( err )
return
}
pW . Close ( )
} ( )
return pR , nil
}