2014-03-08 02:04:38 +00:00
package graph
2013-03-18 07:15:35 +00:00
import (
2015-03-30 18:19:12 +00:00
"compress/gzip"
"crypto/sha256"
2013-03-18 07:15:35 +00:00
"fmt"
2013-05-06 18:06:44 +00:00
"io"
2013-03-18 07:15:35 +00:00
"io/ioutil"
"os"
"path"
"path/filepath"
2013-12-20 16:20:08 +00:00
"runtime"
2013-03-26 12:28:17 +00:00
"strings"
2013-11-13 00:59:37 +00:00
"syscall"
2013-03-18 07:15:35 +00:00
"time"
2014-05-19 22:04:51 +00:00
2015-03-26 22:22:04 +00:00
"github.com/Sirupsen/logrus"
2015-03-30 18:19:12 +00:00
"github.com/docker/distribution/digest"
2015-02-04 21:22:38 +00:00
"github.com/docker/docker/autogen/dockerversion"
2014-07-24 22:19:50 +00:00
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/image"
2014-09-30 06:23:36 +00:00
"github.com/docker/docker/pkg/archive"
2015-02-24 08:51:46 +00:00
"github.com/docker/docker/pkg/progressreader"
2015-03-18 02:18:41 +00:00
"github.com/docker/docker/pkg/streamformatter"
2015-03-24 11:25:26 +00:00
"github.com/docker/docker/pkg/stringid"
2014-07-24 22:19:50 +00:00
"github.com/docker/docker/pkg/truncindex"
"github.com/docker/docker/runconfig"
2013-03-18 07:15:35 +00:00
)
2013-03-30 07:22:24 +00:00
// A Graph is a store for versioned filesystem images and the relationship between them.
2013-03-18 07:15:35 +00:00
type Graph struct {
2013-07-17 19:13:22 +00:00
Root string
2014-06-24 17:19:15 +00:00
idIndex * truncindex . TruncIndex
2013-11-04 23:22:34 +00:00
driver graphdriver . Driver
2013-03-18 07:15:35 +00:00
}
2013-03-30 07:22:24 +00:00
// NewGraph instantiates a new graph at the given root path in the filesystem.
2013-03-30 04:13:59 +00:00
// `root` will be created if it doesn't exist.
2013-11-07 20:34:01 +00:00
func NewGraph ( root string , driver graphdriver . Driver ) ( * Graph , error ) {
2013-03-18 07:15:35 +00:00
abspath , err := filepath . Abs ( root )
if err != nil {
return nil , err
}
// Create the root directory if it doesn't exists
2013-05-08 23:22:12 +00:00
if err := os . MkdirAll ( root , 0700 ) ; err != nil && ! os . IsExist ( err ) {
2013-03-18 07:15:35 +00:00
return nil , err
}
2013-11-04 23:22:34 +00:00
2013-04-01 05:11:55 +00:00
graph := & Graph {
2013-07-17 19:13:22 +00:00
Root : abspath ,
2014-06-24 17:19:15 +00:00
idIndex : truncindex . NewTruncIndex ( [ ] string { } ) ,
2013-11-04 23:22:34 +00:00
driver : driver ,
2013-04-01 05:11:55 +00:00
}
if err := graph . restore ( ) ; err != nil {
return nil , err
}
return graph , nil
}
func ( graph * Graph ) restore ( ) error {
dir , err := ioutil . ReadDir ( graph . Root )
if err != nil {
return err
}
2014-04-11 20:39:58 +00:00
var ids = [ ] string { }
2013-04-01 05:11:55 +00:00
for _ , v := range dir {
id := v . Name ( )
2013-11-19 10:32:08 +00:00
if graph . driver . Exists ( id ) {
2014-04-11 20:39:58 +00:00
ids = append ( ids , id )
2013-11-19 10:32:08 +00:00
}
2013-04-01 05:11:55 +00:00
}
2014-06-24 17:19:15 +00:00
graph . idIndex = truncindex . NewTruncIndex ( ids )
2015-03-26 22:22:04 +00:00
logrus . Debugf ( "Restored %d elements" , len ( dir ) )
2013-04-01 05:11:55 +00:00
return nil
2013-03-18 07:15:35 +00:00
}
2013-03-26 12:28:17 +00:00
// FIXME: Implement error subclass instead of looking at the error text
// Note: This is the way golang implements os.IsNotExists on Plan9
2015-03-28 01:07:20 +00:00
func ( graph * Graph ) IsNotExist ( err error , id string ) bool {
return err != nil && ( strings . Contains ( strings . ToLower ( err . Error ( ) ) , "does not exist" ) || strings . Contains ( strings . ToLower ( err . Error ( ) ) , "no such" ) ) && strings . Contains ( err . Error ( ) , id )
2013-03-26 12:28:17 +00:00
}
2013-03-30 04:13:59 +00:00
// Exists returns true if an image is registered at the given id.
// If the image doesn't exist or if an error is encountered, false is returned.
2013-03-18 07:15:35 +00:00
func ( graph * Graph ) Exists ( id string ) bool {
if _ , err := graph . Get ( id ) ; err != nil {
return false
}
return true
}
2013-03-30 04:13:59 +00:00
// Get returns the image with the given id, or an error if the image doesn't exist.
2014-03-08 01:36:47 +00:00
func ( graph * Graph ) Get ( name string ) ( * image . Image , error ) {
2013-04-01 05:11:55 +00:00
id , err := graph . idIndex . Get ( name )
if err != nil {
2014-12-11 17:57:23 +00:00
return nil , fmt . Errorf ( "could not find image: %v" , err )
2013-04-01 05:11:55 +00:00
}
2014-03-08 01:36:47 +00:00
img , err := image . LoadImage ( graph . ImageRoot ( id ) )
2013-03-18 07:15:35 +00:00
if err != nil {
return nil , err
}
2013-06-04 18:00:22 +00:00
if img . ID != id {
return nil , fmt . Errorf ( "Image stored at '%s' has wrong id '%s'" , id , img . ID )
2013-03-18 07:15:35 +00:00
}
2014-03-08 01:36:47 +00:00
img . SetGraph ( graph )
2013-12-05 22:03:23 +00:00
if img . Size < 0 {
2014-09-11 03:30:52 +00:00
size , err := graph . driver . DiffSize ( img . ID , img . Parent )
2014-01-07 19:34:19 +00:00
if err != nil {
2014-09-11 03:30:52 +00:00
return nil , fmt . Errorf ( "unable to calculate size of image id %q: %s" , img . ID , err )
2013-05-13 13:10:26 +00:00
}
2013-12-05 22:03:23 +00:00
2013-11-07 20:34:01 +00:00
img . Size = size
2014-03-08 01:36:47 +00:00
if err := img . SaveSize ( graph . ImageRoot ( id ) ) ; err != nil {
2013-05-13 13:10:26 +00:00
return nil , err
}
}
2013-03-18 07:15:35 +00:00
return img , nil
}
2013-03-30 04:13:59 +00:00
// Create creates a new image and registers it in the graph.
2014-03-08 02:04:38 +00:00
func ( graph * Graph ) Create ( layerData archive . ArchiveReader , containerID , containerImage , comment , author string , containerConfig , config * runconfig . Config ) ( * image . Image , error ) {
2014-03-08 01:36:47 +00:00
img := & image . Image {
2015-03-24 11:25:26 +00:00
ID : stringid . GenerateRandomID ( ) ,
2013-04-05 01:38:43 +00:00
Comment : comment ,
2013-11-22 00:41:41 +00:00
Created : time . Now ( ) . UTC ( ) ,
2014-02-12 00:26:54 +00:00
DockerVersion : dockerversion . VERSION ,
2013-04-18 02:58:17 +00:00
Author : author ,
2013-04-25 23:48:31 +00:00
Config : config ,
2013-12-20 16:20:08 +00:00
Architecture : runtime . GOARCH ,
OS : runtime . GOOS ,
2013-03-18 07:15:35 +00:00
}
2014-05-19 22:04:51 +00:00
2014-03-08 02:04:38 +00:00
if containerID != "" {
img . Parent = containerImage
img . Container = containerID
img . ContainerConfig = * containerConfig
2013-03-22 04:13:27 +00:00
}
2014-05-19 22:04:51 +00:00
2014-10-27 18:00:29 +00:00
if err := graph . Register ( img , layerData ) ; err != nil {
2013-03-18 07:15:35 +00:00
return nil , err
}
return img , nil
}
2013-03-30 04:13:59 +00:00
// Register imports a pre-existing image into the graph.
2014-10-27 18:00:29 +00:00
func ( graph * Graph ) Register ( img * image . Image , layerData archive . ArchiveReader ) ( err error ) {
2013-11-26 04:48:34 +00:00
defer func ( ) {
// If any error occurs, remove the new dir from the driver.
// Don't check for errors since the dir might not have been created.
// FIXME: this leaves a possible race condition.
if err != nil {
graph . driver . Remove ( img . ID )
}
} ( )
2015-03-29 21:17:23 +00:00
if err := image . ValidateID ( img . ID ) ; err != nil {
2013-03-18 07:15:35 +00:00
return err
}
// (This is a convenience to save time. Race conditions are taken care of by os.Rename)
2013-06-04 18:00:22 +00:00
if graph . Exists ( img . ID ) {
return fmt . Errorf ( "Image %s already exists" , img . ID )
2013-03-18 07:15:35 +00:00
}
2013-11-19 10:32:08 +00:00
// Ensure that the image root does not exist on the filesystem
// when it is not registered in the graph.
// This is common when you switch from one graph driver to another
2014-03-08 01:36:47 +00:00
if err := os . RemoveAll ( graph . ImageRoot ( img . ID ) ) ; err != nil && ! os . IsNotExist ( err ) {
2013-11-19 10:32:08 +00:00
return err
}
2013-11-26 04:48:34 +00:00
// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
// (the graph is the source of truth).
// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
// (FIXME: make that mandatory for drivers).
graph . driver . Remove ( img . ID )
2013-04-17 23:35:22 +00:00
tmp , err := graph . Mktemp ( "" )
2013-03-18 07:15:35 +00:00
defer os . RemoveAll ( tmp )
if err != nil {
return fmt . Errorf ( "Mktemp failed: %s" , err )
}
2013-11-07 20:34:01 +00:00
// Create root filesystem in the driver
2014-04-17 23:47:27 +00:00
if err := graph . driver . Create ( img . ID , img . Parent ) ; err != nil {
2013-11-07 20:34:01 +00:00
return fmt . Errorf ( "Driver %s failed to create image rootfs %s: %s" , graph . driver , img . ID , err )
}
2014-09-11 03:30:52 +00:00
// Apply the diff/layer
2014-03-08 01:36:47 +00:00
img . SetGraph ( graph )
2014-10-27 18:00:29 +00:00
if err := image . StoreImage ( img , layerData , tmp ) ; err != nil {
2013-03-18 07:15:35 +00:00
return err
}
// Commit
2014-03-08 01:36:47 +00:00
if err := os . Rename ( tmp , graph . ImageRoot ( img . ID ) ) ; err != nil {
2013-03-18 07:15:35 +00:00
return err
}
2013-06-04 18:00:22 +00:00
graph . idIndex . Add ( img . ID )
2013-03-18 07:15:35 +00:00
return nil
}
2013-04-21 21:23:55 +00:00
// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
// The archive is stored on disk and will be automatically deleted as soon as has been read.
2013-04-21 22:29:26 +00:00
// If output is not nil, a human-readable progress bar will be written to it.
// FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
2015-03-18 02:18:41 +00:00
func ( graph * Graph ) TempLayerArchive ( id string , sf * streamformatter . StreamFormatter , output io . Writer ) ( * archive . TempArchive , error ) {
2013-04-21 21:23:55 +00:00
image , err := graph . Get ( id )
if err != nil {
return nil , err
}
2013-11-07 20:34:01 +00:00
tmp , err := graph . Mktemp ( "" )
2013-04-21 21:23:55 +00:00
if err != nil {
return nil , err
}
2013-11-20 23:37:26 +00:00
a , err := image . TarLayer ( )
2013-04-21 21:23:55 +00:00
if err != nil {
return nil , err
}
2015-02-24 08:51:46 +00:00
progressReader := progressreader . New ( progressreader . Config {
In : a ,
Out : output ,
Formatter : sf ,
Size : 0 ,
NewLines : false ,
2015-03-24 11:25:26 +00:00
ID : stringid . TruncateID ( id ) ,
2015-02-24 08:51:46 +00:00
Action : "Buffering to disk" ,
} )
defer progressReader . Close ( )
return archive . NewTempArchive ( progressReader , tmp )
2013-04-21 21:23:55 +00:00
}
2013-03-30 04:13:59 +00:00
// Mktemp creates a temporary sub-directory inside the graph's filesystem.
2013-03-18 07:15:35 +00:00
func ( graph * Graph ) Mktemp ( id string ) ( string , error ) {
2015-03-24 11:25:26 +00:00
dir := path . Join ( graph . Root , "_tmp" , stringid . GenerateRandomID ( ) )
2013-11-09 00:53:58 +00:00
if err := os . MkdirAll ( dir , 0700 ) ; err != nil {
return "" , err
2013-03-18 07:15:35 +00:00
}
2013-11-07 20:34:01 +00:00
return dir , nil
2013-03-18 07:15:35 +00:00
}
2015-01-28 02:10:28 +00:00
func ( graph * Graph ) newTempFile ( ) ( * os . File , error ) {
tmp , err := graph . Mktemp ( "" )
if err != nil {
return nil , err
}
return ioutil . TempFile ( tmp , "" )
}
2015-03-30 18:19:12 +00:00
func bufferToFile ( f * os . File , src io . Reader ) ( int64 , digest . Digest , error ) {
var (
h = sha256 . New ( )
w = gzip . NewWriter ( io . MultiWriter ( f , h ) )
)
_ , err := io . Copy ( w , src )
w . Close ( )
2015-01-28 02:10:28 +00:00
if err != nil {
2015-03-30 18:19:12 +00:00
return 0 , "" , err
2015-01-28 02:10:28 +00:00
}
if err = f . Sync ( ) ; err != nil {
2015-03-30 18:19:12 +00:00
return 0 , "" , err
}
n , err := f . Seek ( 0 , os . SEEK_CUR )
if err != nil {
return 0 , "" , err
2015-01-28 02:10:28 +00:00
}
if _ , err := f . Seek ( 0 , 0 ) ; err != nil {
2015-03-30 18:19:12 +00:00
return 0 , "" , err
2015-01-28 02:10:28 +00:00
}
2015-03-30 18:19:12 +00:00
return n , digest . NewDigest ( "sha256" , h ) , nil
2015-01-28 02:10:28 +00:00
}
2013-11-07 20:34:01 +00:00
// setupInitLayer populates a directory with mountpoints suitable
2013-06-14 23:56:08 +00:00
// for bind-mounting dockerinit into the container. The mountpoint is simply an
// empty file at /.dockerinit
//
// This extra layer is used by all containers as the top-most ro layer. It protects
// the container from unwanted side-effects on the rw layer.
2014-03-08 02:04:38 +00:00
func SetupInitLayer ( initLayer string ) error {
2013-08-08 18:25:02 +00:00
for pth , typ := range map [ string ] string {
"/dev/pts" : "dir" ,
"/dev/shm" : "dir" ,
"/proc" : "dir" ,
"/sys" : "dir" ,
"/.dockerinit" : "file" ,
2013-08-13 22:40:23 +00:00
"/.dockerenv" : "file" ,
2013-08-08 18:25:02 +00:00
"/etc/resolv.conf" : "file" ,
2013-09-09 19:40:25 +00:00
"/etc/hosts" : "file" ,
"/etc/hostname" : "file" ,
2014-02-28 01:20:10 +00:00
"/dev/console" : "file" ,
2014-03-19 18:52:38 +00:00
"/etc/mtab" : "/proc/mounts" ,
2013-08-08 18:25:02 +00:00
} {
2013-11-13 00:59:37 +00:00
parts := strings . Split ( pth , "/" )
prev := "/"
for _ , p := range parts [ 1 : ] {
prev = path . Join ( prev , p )
syscall . Unlink ( path . Join ( initLayer , prev ) )
}
2013-08-08 18:25:02 +00:00
if _ , err := os . Stat ( path . Join ( initLayer , pth ) ) ; err != nil {
if os . IsNotExist ( err ) {
2014-04-09 17:13:54 +00:00
if err := os . MkdirAll ( path . Join ( initLayer , path . Dir ( pth ) ) , 0755 ) ; err != nil {
return err
}
2013-08-08 18:25:02 +00:00
switch typ {
case "dir" :
if err := os . MkdirAll ( path . Join ( initLayer , pth ) , 0755 ) ; err != nil {
2013-11-07 20:34:01 +00:00
return err
2013-08-08 18:25:02 +00:00
}
case "file" :
2013-11-18 23:35:56 +00:00
f , err := os . OpenFile ( path . Join ( initLayer , pth ) , os . O_CREATE , 0755 )
if err != nil {
2013-11-07 20:34:01 +00:00
return err
2013-08-08 18:25:02 +00:00
}
2013-11-18 23:35:56 +00:00
f . Close ( )
2014-03-19 18:52:38 +00:00
default :
if err := os . Symlink ( typ , path . Join ( initLayer , pth ) ) ; err != nil {
return err
}
2013-08-08 18:25:02 +00:00
}
} else {
2013-11-07 20:34:01 +00:00
return err
2013-08-08 18:25:02 +00:00
}
}
2013-06-14 23:56:08 +00:00
}
2013-08-08 18:25:02 +00:00
2013-06-14 23:56:08 +00:00
// Layer is ready to use, if it wasn't before.
2013-11-07 20:34:01 +00:00
return nil
2013-04-21 21:23:55 +00:00
}
2013-03-30 07:22:24 +00:00
// Check if given error is "not empty".
// Note: this is the way golang does it internally with os.IsNotExists.
2013-03-26 10:33:47 +00:00
func isNotEmpty ( err error ) bool {
switch pe := err . ( type ) {
case nil :
return false
case * os . PathError :
err = pe . Err
case * os . LinkError :
err = pe . Err
}
return strings . Contains ( err . Error ( ) , " not empty" )
}
2013-03-30 04:13:59 +00:00
// Delete atomically removes an image from the graph.
2013-04-01 23:04:44 +00:00
func ( graph * Graph ) Delete ( name string ) error {
id , err := graph . idIndex . Get ( name )
if err != nil {
return err
}
2013-04-04 05:14:28 +00:00
tmp , err := graph . Mktemp ( "" )
2013-04-01 05:11:55 +00:00
graph . idIndex . Delete ( id )
2014-09-22 14:47:20 +00:00
if err == nil {
err = os . Rename ( graph . ImageRoot ( id ) , tmp )
// On err make tmp point to old dir and cleanup unused tmp dir
if err != nil {
os . RemoveAll ( tmp )
tmp = graph . ImageRoot ( id )
}
} else {
// On err make tmp point to old dir for cleanup
tmp = graph . ImageRoot ( id )
2013-03-18 07:15:35 +00:00
}
2013-11-07 20:34:01 +00:00
// Remove rootfs data from the driver
graph . driver . Remove ( id )
// Remove the trashed image directory
2013-04-04 05:14:28 +00:00
return os . RemoveAll ( tmp )
2013-03-18 07:15:35 +00:00
}
2013-03-30 07:22:24 +00:00
// Map returns a list of all images in the graph, addressable by ID.
2014-03-08 01:36:47 +00:00
func ( graph * Graph ) Map ( ) ( map [ string ] * image . Image , error ) {
images := make ( map [ string ] * image . Image )
err := graph . walkAll ( func ( image * image . Image ) {
2013-09-01 03:31:21 +00:00
images [ image . ID ] = image
} )
2013-03-22 00:35:49 +00:00
if err != nil {
return nil , err
}
return images , nil
}
2013-09-01 03:34:51 +00:00
// walkAll iterates over each image in the graph, and passes it to a handler.
2013-03-30 04:13:59 +00:00
// The walking order is undetermined.
2014-03-08 01:36:47 +00:00
func ( graph * Graph ) walkAll ( handler func ( * image . Image ) ) error {
2013-03-18 07:15:35 +00:00
files , err := ioutil . ReadDir ( graph . Root )
if err != nil {
2013-03-24 00:03:30 +00:00
return err
2013-03-18 07:15:35 +00:00
}
for _ , st := range files {
if img , err := graph . Get ( st . Name ( ) ) ; err != nil {
// Skip image
continue
2013-03-24 00:03:30 +00:00
} else if handler != nil {
handler ( img )
}
}
return nil
}
2013-03-30 04:13:59 +00:00
// ByParent returns a lookup table of images by their parent.
// If an image of id ID has 3 children images, then the value for key ID
// will be a list of 3 images.
// If an image has no children, it will not have an entry in the table.
2014-03-08 01:36:47 +00:00
func ( graph * Graph ) ByParent ( ) ( map [ string ] [ ] * image . Image , error ) {
byParent := make ( map [ string ] [ ] * image . Image )
err := graph . walkAll ( func ( img * image . Image ) {
parent , err := graph . Get ( img . Parent )
2013-03-24 00:03:30 +00:00
if err != nil {
return
}
2013-06-04 18:00:22 +00:00
if children , exists := byParent [ parent . ID ] ; exists {
2014-03-08 01:36:47 +00:00
byParent [ parent . ID ] = append ( children , img )
2013-08-11 07:37:16 +00:00
} else {
2014-03-08 01:36:47 +00:00
byParent [ parent . ID ] = [ ] * image . Image { img }
2013-03-18 07:15:35 +00:00
}
2013-03-24 00:03:30 +00:00
} )
return byParent , err
}
2013-03-30 04:13:59 +00:00
// Heads returns all heads in the graph, keyed by id.
// A head is an image which is not the parent of another image in the graph.
2014-03-08 01:36:47 +00:00
func ( graph * Graph ) Heads ( ) ( map [ string ] * image . Image , error ) {
heads := make ( map [ string ] * image . Image )
2013-03-24 00:03:30 +00:00
byParent , err := graph . ByParent ( )
if err != nil {
return nil , err
2013-03-18 07:15:35 +00:00
}
2014-03-08 01:36:47 +00:00
err = graph . walkAll ( func ( image * image . Image ) {
2013-03-24 00:03:30 +00:00
// If it's not in the byParent lookup table, then
// it's not a parent -> so it's a head!
2013-06-04 18:00:22 +00:00
if _ , exists := byParent [ image . ID ] ; ! exists {
heads [ image . ID ] = image
2013-03-24 00:03:30 +00:00
}
} )
return heads , err
2013-03-18 07:15:35 +00:00
}
2014-03-08 01:36:47 +00:00
func ( graph * Graph ) ImageRoot ( id string ) string {
2013-03-18 07:15:35 +00:00
return path . Join ( graph . Root , id )
}
2013-11-26 04:04:57 +00:00
func ( graph * Graph ) Driver ( ) graphdriver . Driver {
return graph . driver
}