package layer import ( "compress/gzip" "encoding/json" "errors" "fmt" "io" "io/ioutil" "os" "path/filepath" "regexp" "strconv" "strings" "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/pkg/ioutils" ) var ( stringIDRegexp = regexp.MustCompile(`^[a-f0-9]{64}(-init)?$`) supportedAlgorithms = []digest.Algorithm{ digest.SHA256, // digest.SHA384, // Currently not used // digest.SHA512, // Currently not used } ) type fileMetadataStore struct { root string } type fileMetadataTransaction struct { store *fileMetadataStore ws *ioutils.AtomicWriteSet } // NewFSMetadataStore returns an instance of a metadata store // which is backed by files on disk using the provided root // as the root of metadata files. func NewFSMetadataStore(root string) (MetadataStore, error) { if err := os.MkdirAll(root, 0700); err != nil { return nil, err } return &fileMetadataStore{ root: root, }, nil } func (fms *fileMetadataStore) getLayerDirectory(layer ChainID) string { dgst := digest.Digest(layer) return filepath.Join(fms.root, string(dgst.Algorithm()), dgst.Hex()) } func (fms *fileMetadataStore) getLayerFilename(layer ChainID, filename string) string { return filepath.Join(fms.getLayerDirectory(layer), filename) } func (fms *fileMetadataStore) getMountDirectory(mount string) string { return filepath.Join(fms.root, "mounts", mount) } func (fms *fileMetadataStore) getMountFilename(mount, filename string) string { return filepath.Join(fms.getMountDirectory(mount), filename) } func (fms *fileMetadataStore) StartTransaction() (MetadataTransaction, error) { tmpDir := filepath.Join(fms.root, "tmp") if err := os.MkdirAll(tmpDir, 0755); err != nil { return nil, err } ws, err := ioutils.NewAtomicWriteSet(tmpDir) if err != nil { return nil, err } return &fileMetadataTransaction{ store: fms, ws: ws, }, nil } func (fm *fileMetadataTransaction) SetSize(size int64) error { content := fmt.Sprintf("%d", size) return fm.ws.WriteFile("size", []byte(content), 0644) } func (fm *fileMetadataTransaction) SetParent(parent ChainID) error { return fm.ws.WriteFile("parent", []byte(digest.Digest(parent).String()), 0644) } func (fm *fileMetadataTransaction) SetDiffID(diff DiffID) error { return fm.ws.WriteFile("diff", []byte(digest.Digest(diff).String()), 0644) } func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error { return fm.ws.WriteFile("cache-id", []byte(cacheID), 0644) } func (fm *fileMetadataTransaction) SetDescriptor(ref distribution.Descriptor) error { jsonRef, err := json.Marshal(ref) if err != nil { return err } return fm.ws.WriteFile("descriptor.json", jsonRef, 0644) } func (fm *fileMetadataTransaction) TarSplitWriter(compressInput bool) (io.WriteCloser, error) { f, err := fm.ws.FileWriter("tar-split.json.gz", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return nil, err } var wc io.WriteCloser if compressInput { wc = gzip.NewWriter(f) } else { wc = f } return ioutils.NewWriteCloserWrapper(wc, func() error { wc.Close() return f.Close() }), nil } func (fm *fileMetadataTransaction) Commit(layer ChainID) error { finalDir := fm.store.getLayerDirectory(layer) if err := os.MkdirAll(filepath.Dir(finalDir), 0755); err != nil { return err } return fm.ws.Commit(finalDir) } func (fm *fileMetadataTransaction) Cancel() error { return fm.ws.Cancel() } func (fm *fileMetadataTransaction) String() string { return fm.ws.String() } func (fms *fileMetadataStore) GetSize(layer ChainID) (int64, error) { content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "size")) if err != nil { return 0, err } size, err := strconv.ParseInt(string(content), 10, 64) if err != nil { return 0, err } return size, nil } func (fms *fileMetadataStore) GetParent(layer ChainID) (ChainID, error) { content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "parent")) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } dgst, err := digest.ParseDigest(strings.TrimSpace(string(content))) if err != nil { return "", err } return ChainID(dgst), nil } func (fms *fileMetadataStore) GetDiffID(layer ChainID) (DiffID, error) { content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "diff")) if err != nil { return "", err } dgst, err := digest.ParseDigest(strings.TrimSpace(string(content))) if err != nil { return "", err } return DiffID(dgst), nil } func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) { contentBytes, err := ioutil.ReadFile(fms.getLayerFilename(layer, "cache-id")) if err != nil { return "", err } content := strings.TrimSpace(string(contentBytes)) if !stringIDRegexp.MatchString(content) { return "", errors.New("invalid cache id value") } return content, nil } func (fms *fileMetadataStore) GetDescriptor(layer ChainID) (distribution.Descriptor, error) { content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "descriptor.json")) if err != nil { if os.IsNotExist(err) { // only return empty descriptor to represent what is stored return distribution.Descriptor{}, nil } return distribution.Descriptor{}, err } var ref distribution.Descriptor err = json.Unmarshal(content, &ref) if err != nil { return distribution.Descriptor{}, err } return ref, err } func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) { fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz")) if err != nil { return nil, err } f, err := gzip.NewReader(fz) if err != nil { return nil, err } return ioutils.NewReadCloserWrapper(f, func() error { f.Close() return fz.Close() }), nil } func (fms *fileMetadataStore) SetMountID(mount string, mountID string) error { if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil { return err } return ioutil.WriteFile(fms.getMountFilename(mount, "mount-id"), []byte(mountID), 0644) } func (fms *fileMetadataStore) SetInitID(mount string, init string) error { if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil { return err } return ioutil.WriteFile(fms.getMountFilename(mount, "init-id"), []byte(init), 0644) } func (fms *fileMetadataStore) SetMountParent(mount string, parent ChainID) error { if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil { return err } return ioutil.WriteFile(fms.getMountFilename(mount, "parent"), []byte(digest.Digest(parent).String()), 0644) } func (fms *fileMetadataStore) GetMountID(mount string) (string, error) { contentBytes, err := ioutil.ReadFile(fms.getMountFilename(mount, "mount-id")) if err != nil { return "", err } content := strings.TrimSpace(string(contentBytes)) if !stringIDRegexp.MatchString(content) { return "", errors.New("invalid mount id value") } return content, nil } func (fms *fileMetadataStore) GetInitID(mount string) (string, error) { contentBytes, err := ioutil.ReadFile(fms.getMountFilename(mount, "init-id")) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } content := strings.TrimSpace(string(contentBytes)) if !stringIDRegexp.MatchString(content) { return "", errors.New("invalid init id value") } return content, nil } func (fms *fileMetadataStore) GetMountParent(mount string) (ChainID, error) { content, err := ioutil.ReadFile(fms.getMountFilename(mount, "parent")) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } dgst, err := digest.ParseDigest(strings.TrimSpace(string(content))) if err != nil { return "", err } return ChainID(dgst), nil } func (fms *fileMetadataStore) List() ([]ChainID, []string, error) { var ids []ChainID for _, algorithm := range supportedAlgorithms { fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, string(algorithm))) if err != nil { if os.IsNotExist(err) { continue } return nil, nil, err } for _, fi := range fileInfos { if fi.IsDir() && fi.Name() != "mounts" { dgst := digest.NewDigestFromHex(string(algorithm), fi.Name()) if err := dgst.Validate(); err != nil { logrus.Debugf("Ignoring invalid digest %s:%s", algorithm, fi.Name()) } else { ids = append(ids, ChainID(dgst)) } } } } fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, "mounts")) if err != nil { if os.IsNotExist(err) { return ids, []string{}, nil } return nil, nil, err } var mounts []string for _, fi := range fileInfos { if fi.IsDir() { mounts = append(mounts, fi.Name()) } } return ids, mounts, nil } func (fms *fileMetadataStore) Remove(layer ChainID) error { return os.RemoveAll(fms.getLayerDirectory(layer)) } func (fms *fileMetadataStore) RemoveMount(mount string) error { return os.RemoveAll(fms.getMountDirectory(mount)) }