Merge pull request #43833 from thaJeztah/compute_sharedsize

containerd integration: compute image's shared size

daemon/containerd/image_list.go

@@ -4,10 +4,10 @@ import (
 	"context"
 
 	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
 )
 
@@ -22,6 +22,10 @@ var acceptedImageFilterTags = map[string]bool{
 // Images returns a filtered list of images.
 //
 // TODO(thaJeztah): sort the results by created (descending); see https://github.com/moby/moby/issues/43848
+// TODO(thaJeztah): implement opts.ContainerCount (used for docker system df); see https://github.com/moby/moby/issues/43853
+// TODO(thaJeztah): add labels to results; see https://github.com/moby/moby/issues/43852
+// TODO(thaJeztah): verify behavior of `RepoDigests` and `RepoTags` for images without (untagged) or multiple tags; see https://github.com/moby/moby/issues/43861
+// TODO(thaJeztah): verify "Size" vs "VirtualSize" in images; see https://github.com/moby/moby/issues/43862
 func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) ([]*types.ImageSummary, error) {
 	if err := opts.Filters.Validate(acceptedImageFilterTags); err != nil {
 		return nil, err
@@ -38,37 +42,83 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions)
 	}
 
 	snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter)
+	sizeCache := make(map[digest.Digest]int64)
+	snapshotSizeFn := func(d digest.Digest) (int64, error) {
+		if s, ok := sizeCache[d]; ok {
+			return s, nil
+		}
+		usage, err := snapshotter.Usage(ctx, d.String())
+		if err != nil {
+			return 0, err
+		}
+		sizeCache[d] = usage.Size
+		return usage.Size, nil
+	}
 
-	var ret []*types.ImageSummary
-	for _, img := range imgs {
+	var (
+		summaries = make([]*types.ImageSummary, 0, len(imgs))
+		root      []*[]digest.Digest
+		layers    map[digest.Digest]int
+	)
+	if opts.SharedSize {
+		root = make([]*[]digest.Digest, len(imgs))
+		layers = make(map[digest.Digest]int)
+	}
+	for n, img := range imgs {
 		if !filter(img) {
 			continue
 		}
 
+		diffIDs, err := img.RootFS(ctx)
+		if err != nil {
+			return nil, err
+		}
+		chainIDs := identity.ChainIDs(diffIDs)
+		if opts.SharedSize {
+			root[n] = &chainIDs
+			for _, id := range chainIDs {
+				layers[id] = layers[id] + 1
+			}
+		}
+
 		size, err := img.Size(ctx)
 		if err != nil {
 			return nil, err
 		}
 
-		virtualSize, err := computeVirtualSize(ctx, img, snapshotter)
+		virtualSize, err := computeVirtualSize(chainIDs, snapshotSizeFn)
 		if err != nil {
 			return nil, err
 		}
 
-		ret = append(ret, &types.ImageSummary{
-			RepoDigests: []string{img.Name() + "@" + img.Target().Digest.String()}, // "hello-world@sha256:bfea6278a0a267fad2634554f4f0c6f31981eea41c553fdf5a83e95a41d40c38"},
-			RepoTags:    []string{img.Name()},
-			Containers:  -1,
+		summaries = append(summaries, &types.ImageSummary{
 			ParentID:    "",
-			SharedSize:  -1,
-			VirtualSize: virtualSize,
 			ID:          img.Target().Digest.String(),
 			Created:     img.Metadata().CreatedAt.Unix(),
+			RepoDigests: []string{img.Name() + "@" + img.Target().Digest.String()}, // "hello-world@sha256:bfea6278a0a267fad2634554f4f0c6f31981eea41c553fdf5a83e95a41d40c38"},
+			RepoTags:    []string{img.Name()},
 			Size:        size,
+			VirtualSize: virtualSize,
+			// -1 indicates that the value has not been set (avoids ambiguity
+			// between 0 (default) and "not set". We cannot use a pointer (nil)
+			// for this, as the JSON representation uses "omitempty", which would
+			// consider both "0" and "nil" to be "empty".
+			SharedSize: -1,
+			Containers: -1,
 		})
 	}
 
-	return ret, nil
+	if opts.SharedSize {
+		for n, chainIDs := range root {
+			sharedSize, err := computeSharedSize(*chainIDs, layers, snapshotSizeFn)
+			if err != nil {
+				return nil, err
+			}
+			summaries[n].SharedSize = sharedSize
+		}
+	}
+
+	return summaries, nil
 }
 
 type imageFilterFunc func(image containerd.Image) bool
@@ -131,18 +181,29 @@ func (i *ImageService) setupFilters(ctx context.Context, imageFilters filters.Ar
 	}, nil
 }
 
-func computeVirtualSize(ctx context.Context, image containerd.Image, snapshotter snapshots.Snapshotter) (int64, error) {
+func computeVirtualSize(chainIDs []digest.Digest, sizeFn func(d digest.Digest) (int64, error)) (int64, error) {
 	var virtualSize int64
-	diffIDs, err := image.RootFS(ctx)
-	if err != nil {
-		return virtualSize, err
-	}
-	for _, chainID := range identity.ChainIDs(diffIDs) {
-		usage, err := snapshotter.Usage(ctx, chainID.String())
+	for _, chainID := range chainIDs {
+		size, err := sizeFn(chainID)
 		if err != nil {
 			return virtualSize, err
 		}
-		virtualSize += usage.Size
+		virtualSize += size
 	}
 	return virtualSize, nil
 }
+
+func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, sizeFn func(d digest.Digest) (int64, error)) (int64, error) {
+	var sharedSize int64
+	for _, chainID := range chainIDs {
+		if layers[chainID] == 1 {
+			continue
+		}
+		size, err := sizeFn(chainID)
+		if err != nil {
+			return 0, err
+		}
+		sharedSize += size
+	}
+	return sharedSize, nil
+}