package fs import ( "database/sql" "fmt" "github.com/dotcloud/docker/future" _ "github.com/mattn/go-sqlite3" "github.com/shykes/gorp" //Forked to implement CreateTablesOpts "io" "io/ioutil" "os" "path" "path/filepath" "regexp" "strings" "syscall" "time" ) type Store struct { Root string db *sql.DB orm *gorp.DbMap layers *LayerStore } type Archive io.Reader func New(root string) (*Store, error) { isNewStore := true if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) { return nil, err } db, err := sql.Open("sqlite3", path.Join(root, "db")) if err != nil { return nil, err } orm := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}} orm.AddTableWithName(Image{}, "images").SetKeys(false, "Id") orm.AddTableWithName(Path{}, "paths").SetKeys(false, "Path", "Image") orm.AddTableWithName(Mountpoint{}, "mountpoints").SetKeys(false, "Root") orm.AddTableWithName(Tag{}, "tags").SetKeys(false, "TagName") if isNewStore { if err := orm.CreateTablesOpts(true); err != nil { return nil, err } } layers, err := NewLayerStore(path.Join(root, "layers")) if err != nil { return nil, err } return &Store{ Root: root, db: db, orm: orm, layers: layers, }, nil } func (store *Store) imageList(src []interface{}) []*Image { var images []*Image for _, i := range src { img := i.(*Image) img.store = store images = append(images, img) } return images } func (store *Store) Images() ([]*Image, error) { images, err := store.orm.Select(Image{}, "select * from images") if err != nil { return nil, err } return store.imageList(images), nil } func (store *Store) Paths() ([]string, error) { var paths []string rows, err := store.db.Query("select distinct Path from paths order by Path") if err != nil { return nil, err } for rows.Next() { var path string if err := rows.Scan(&path); err != nil { return nil, err } paths = append(paths, path) } return paths, nil } func (store *Store) RemoveInPath(pth string) error { images, err := store.List(pth) if err != nil { return err } for _, img := range images { if err = store.Remove(img); err != nil { return err } } return nil } // DeleteMatch deletes all images whose name matches `pattern` func (store *Store) RemoveRegexp(pattern string) error { // Retrieve all the paths paths, err := store.Paths() if err != nil { return err } // Check the pattern on each elements for _, pth := range paths { if match, err := regexp.MatchString(pattern, pth); err != nil { return err } else if match { // If there is a match, remove it if err := store.RemoveInPath(pth); err != nil { return nil } } } return nil } func (store *Store) Remove(img *Image) error { _, err := store.orm.Delete(img) return err } func (store *Store) List(pth string) ([]*Image, error) { pth = path.Clean(pth) images, err := store.orm.Select(Image{}, "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc", pth) if err != nil { return nil, err } return store.imageList(images), nil } func (store *Store) Find(pth string) (*Image, error) { pth = path.Clean(pth) img, err := store.Get(pth) if err != nil { return nil, err } else if img != nil { return img, nil } var q string var args []interface{} // FIXME: this breaks if the path contains a ':' // If format is path:rev if parts := strings.SplitN(pth, ":", 2); len(parts) == 2 { q = "select Images.* from images, paths where Path=? and images.Id=? and paths.Image=images.Id" args = []interface{}{parts[0], parts[1]} // If format is path:rev } else { q = "select images.* from images, paths where Path=? and paths.Image=images.Id order by images.Created desc limit 1" args = []interface{}{parts[0]} } images, err := store.orm.Select(Image{}, q, args...) if err != nil { return nil, err } else if len(images) < 1 { return nil, nil } img = images[0].(*Image) img.store = store return img, nil } func (store *Store) Get(id string) (*Image, error) { img, err := store.orm.Get(Image{}, id) if img == nil { return nil, err } res := img.(*Image) res.store = store return res, err } func (store *Store) Create(layerData Archive, parent *Image, pth, comment string) (*Image, error) { // FIXME: actually do something with the layer... img := &Image{ Id: future.RandomId(), Comment: comment, Created: time.Now().Unix(), store: store, } if parent != nil { img.Parent = parent.Id } // FIXME: Archive should contain compression info. For now we only support uncompressed. err := store.Register(layerData, img, pth) return img, err } func (store *Store) Register(layerData Archive, img *Image, pth string) error { img.store = store _, err := store.layers.AddLayer(img.Id, layerData) if err != nil { return fmt.Errorf("Could not add layer: %s", err) } pathObj := &Path{ Path: path.Clean(pth), Image: img.Id, } trans, err := store.orm.Begin() if err != nil { return fmt.Errorf("Could not begin transaction: %s", err) } if err := trans.Insert(img); err != nil { return fmt.Errorf("Could not insert image info: %s", err) } if err := trans.Insert(pathObj); err != nil { return fmt.Errorf("Could not insert path info: %s", err) } if err := trans.Commit(); err != nil { return fmt.Errorf("Could not commit transaction: %s", err) } return nil } func (store *Store) Layers() []string { return store.layers.List() } type Image struct { Id string Parent string Comment string Created int64 store *Store `db:"-"` } func (image *Image) Copy(pth string) (*Image, error) { if err := image.store.orm.Insert(&Path{Path: pth, Image: image.Id}); err != nil { return nil, err } return image, nil } type Mountpoint struct { Image string Root string Rw string Store *Store `db:"-"` } func (image *Image) Mountpoint(root, rw string) (*Mountpoint, error) { mountpoint := &Mountpoint{ Root: path.Clean(root), Rw: path.Clean(rw), Image: image.Id, Store: image.store, } if err := image.store.orm.Insert(mountpoint); err != nil { return nil, err } return mountpoint, nil } func (image *Image) layers() ([]string, error) { var list []string var err error currentImg := image for currentImg != nil { if layer := image.store.layers.Get(currentImg.Id); layer != "" { list = append(list, layer) } else { return list, fmt.Errorf("Layer not found for image %s", image.Id) } currentImg, err = currentImg.store.Get(currentImg.Parent) if err != nil { return list, fmt.Errorf("Error while getting parent image: %v", err) } } if len(list) == 0 { return nil, fmt.Errorf("No layer found for image %s\n", image.Id) } return list, nil } func (image *Image) Mountpoints() ([]*Mountpoint, error) { var mountpoints []*Mountpoint res, err := image.store.orm.Select(Mountpoint{}, "select * from mountpoints where Image=?", image.Id) if err != nil { return nil, err } for _, mp := range res { mountpoints = append(mountpoints, mp.(*Mountpoint)) } return mountpoints, nil } func (image *Image) Mount(root, rw string) (*Mountpoint, error) { var mountpoint *Mountpoint if mp, err := image.store.FetchMountpoint(root, rw); err != nil { return nil, err } else if mp == nil { mountpoint, err = image.Mountpoint(root, rw) if err != nil { return nil, fmt.Errorf("Could not create mountpoint: %s", err) } else if mountpoint == nil { return nil, fmt.Errorf("No mountpoint created") } } else { mountpoint = mp } if err := mountpoint.createFolders(); err != nil { return nil, err } // FIXME: Now mount the layers rwBranch := fmt.Sprintf("%v=rw", mountpoint.Rw) roBranches := "" layers, err := image.layers() if err != nil { return nil, err } for _, layer := range layers { roBranches += fmt.Sprintf("%v=ro:", layer) } branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches) if err := mount("none", mountpoint.Root, "aufs", 0, branches); err != nil { return mountpoint, err } if !mountpoint.Mounted() { return mountpoint, fmt.Errorf("Mount failed") } // FIXME: Create tests for deletion // FIXME: move this part to change.go, maybe refactor // fs.Change() to avoid the fake mountpoint // Retrieve the changeset from the parent and apply it to the container // - Retrieve the changes changes, err := image.store.Changes(&Mountpoint{ Image: image.Id, Root: layers[0], Rw: layers[0], Store: image.store}) if err != nil { return nil, err } // Iterate on changes for _, c := range changes { // If there is a delete if c.Kind == ChangeDelete { // Make sure the directory exists file_path, file_name := path.Dir(c.Path), path.Base(c.Path) if err := os.MkdirAll(path.Join(mountpoint.Rw, file_path), 0755); err != nil { return nil, err } // And create the whiteout (we just need to create empty file, discard the return) if _, err := os.Create(path.Join(path.Join(mountpoint.Rw, file_path), ".wh."+path.Base(file_name))); err != nil { return nil, err } } } return mountpoint, nil } func (mp *Mountpoint) EnsureMounted() error { if mp.Mounted() { return nil } img, err := mp.Store.Get(mp.Image) if err != nil { return err } _, err = img.Mount(mp.Root, mp.Rw) return err } func (mp *Mountpoint) createFolders() error { if err := os.Mkdir(mp.Root, 0755); err != nil && !os.IsExist(err) { return err } if err := os.Mkdir(mp.Rw, 0755); err != nil && !os.IsExist(err) { return err } return nil } func (mp *Mountpoint) Mounted() bool { root, err := os.Stat(mp.Root) if err != nil { if os.IsNotExist(err) { return false } panic(err) } parent, err := os.Stat(filepath.Join(mp.Root, "..")) if err != nil { panic(err) } rootSt := root.Sys().(*syscall.Stat_t) parentSt := parent.Sys().(*syscall.Stat_t) return rootSt.Dev != parentSt.Dev } func (mp *Mountpoint) Umount() error { if !mp.Mounted() { return fmt.Errorf("Mountpoint doesn't seem to be mounted") } if err := syscall.Unmount(mp.Root, 0); err != nil { return fmt.Errorf("Unmount syscall failed: %v", err) } if mp.Mounted() { return fmt.Errorf("Umount: Filesystem still mounted after calling umount(%v)", mp.Root) } // Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint // for some time. We'll just keep retrying until it succeeds. for retries := 0; retries < 1000; retries++ { err := os.Remove(mp.Root) if err == nil { // rm mntpoint succeeded return nil } if os.IsNotExist(err) { // mntpoint doesn't exist anymore. Success. return nil } // fmt.Printf("(%v) Remove %v returned: %v\n", retries, mp.Root, err) time.Sleep(10 * time.Millisecond) } return fmt.Errorf("Umount: Failed to umount %v", mp.Root) } func (mp *Mountpoint) Deregister() error { if mp.Mounted() { return fmt.Errorf("Mountpoint is currently mounted, can't deregister") } _, err := mp.Store.orm.Delete(mp) return err } func (store *Store) FetchMountpoint(root, rw string) (*Mountpoint, error) { res, err := store.orm.Select(Mountpoint{}, "select * from mountpoints where Root=? and Rw=?", root, rw) if err != nil { return nil, err } else if len(res) < 1 || res[0] == nil { return nil, nil } mp := res[0].(*Mountpoint) mp.Store = store return mp, nil } // OpenFile opens the named file for reading. func (mp *Mountpoint) OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) { if err := mp.EnsureMounted(); err != nil { return nil, err } return os.OpenFile(filepath.Join(mp.Root, path), flag, perm) } // ReadDir reads the directory named by dirname, relative to the Mountpoint's root, // and returns a list of sorted directory entries func (mp *Mountpoint) ReadDir(dirname string) ([]os.FileInfo, error) { if err := mp.EnsureMounted(); err != nil { return nil, err } return ioutil.ReadDir(filepath.Join(mp.Root, dirname)) } func (store *Store) AddTag(imageId, tagName string) error { if image, err := store.Get(imageId); err != nil { return err } else if image == nil { return fmt.Errorf("No image with ID %s", imageId) } err2 := store.orm.Insert(&Tag{ TagName: tagName, Image: imageId, }) return err2 } func (store *Store) GetByTag(tagName string) (*Image, error) { res, err := store.orm.Get(Tag{}, tagName) if err != nil { return nil, err } else if res == nil { return nil, fmt.Errorf("No image associated to tag \"%s\"", tagName) } tag := res.(*Tag) img, err2 := store.Get(tag.Image) if err2 != nil { return nil, err2 } else if img == nil { return nil, fmt.Errorf("Tag was found but image seems to be inexistent.") } return img, nil } type Path struct { Path string Image string } type Tag struct { TagName string Image string }