Merge pull request #29427 from yongtang/29325-cleanup-btrfs-quota-groups

Remove btrfs quota groups after containers destroyed

daemon/graphdriver/btrfs/btrfs.go

@@ -16,13 +16,18 @@ import "C"
 
 import (
 	"fmt"
+	"io/ioutil"
+	"math"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"unsafe"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
@@ -119,6 +124,7 @@ type Driver struct {
 	gidMaps      []idtools.IDMap
 	options      btrfsOptions
 	quotaEnabled bool
+	once         sync.Once
 }
 
 // String prints the name of the driver (btrfs).
@@ -237,7 +243,7 @@ func isSubvolume(p string) (bool, error) {
 	return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil
 }
 
-func subvolDelete(dirpath, name string) error {
+func subvolDelete(dirpath, name string, quotaEnabled bool) error {
 	dir, err := openDir(dirpath)
 	if err != nil {
 		return err
@@ -265,7 +271,7 @@ func subvolDelete(dirpath, name string) error {
 				return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err)
 			}
 			if sv {
-				if err := subvolDelete(path.Dir(p), f.Name()); err != nil {
+				if err := subvolDelete(path.Dir(p), f.Name(), quotaEnabled); err != nil {
 					return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err)
 				}
 			}
@@ -276,6 +282,21 @@ func subvolDelete(dirpath, name string) error {
 		return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err)
 	}
 
+	if quotaEnabled {
+		if qgroupid, err := subvolLookupQgroup(fullPath); err == nil {
+			var args C.struct_btrfs_ioctl_qgroup_create_args
+			args.qgroupid = C.__u64(qgroupid)
+
+			_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_CREATE,
+				uintptr(unsafe.Pointer(&args)))
+			if errno != 0 {
+				logrus.Errorf("Failed to delete btrfs qgroup %v for %s: %v", qgroupid, fullPath, errno.Error())
+			}
+		} else {
+			logrus.Errorf("Failed to lookup btrfs qgroup for %s: %v", fullPath, err.Error())
+		}
+	}
+
 	// all subvolumes have been removed
 	// now remove the one originally passed in
 	for i, c := range []byte(name) {
@@ -289,13 +310,23 @@ func subvolDelete(dirpath, name string) error {
 	return nil
 }
 
-func (d *Driver) subvolEnableQuota() error {
-	if d.quotaEnabled {
-		return nil
-	}
+func (d *Driver) updateQuotaStatus() {
+	d.once.Do(func() {
+		if !d.quotaEnabled {
 			// In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed
-	if _, err := subvolLookupQgroup(d.home); err == nil {
+			if err := subvolQgroupStatus(d.home); err != nil {
+				// quota is still not enabled
+				return
+			}
 			d.quotaEnabled = true
+		}
+	})
+}
+
+func (d *Driver) subvolEnableQuota() error {
+	d.updateQuotaStatus()
+
+	if d.quotaEnabled {
 		return nil
 	}
 
@@ -319,14 +350,11 @@ func (d *Driver) subvolEnableQuota() error {
 }
 
 func (d *Driver) subvolDisableQuota() error {
+	d.updateQuotaStatus()
+
 	if !d.quotaEnabled {
-		// In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed
-		if _, err := subvolLookupQgroup(d.home); err != nil {
-			// quota is still not enabled
 		return nil
 	}
-		d.quotaEnabled = true
-	}
 
 	dir, err := openDir(d.home)
 	if err != nil {
@@ -348,14 +376,11 @@ func (d *Driver) subvolDisableQuota() error {
 }
 
 func (d *Driver) subvolRescanQuota() error {
+	d.updateQuotaStatus()
+
 	if !d.quotaEnabled {
-		// In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed
-		if _, err := subvolLookupQgroup(d.home); err != nil {
-			// quota is still not enabled
 		return nil
 	}
-		d.quotaEnabled = true
-	}
 
 	dir, err := openDir(d.home)
 	if err != nil {
@@ -392,6 +417,38 @@ func subvolLimitQgroup(path string, size uint64) error {
 	return nil
 }
 
+// subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path
+// with search key of BTRFS_QGROUP_STATUS_KEY.
+// In case qgroup is enabled, the retuned key type will match BTRFS_QGROUP_STATUS_KEY.
+// For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035
+func subvolQgroupStatus(path string) error {
+	dir, err := openDir(path)
+	if err != nil {
+		return err
+	}
+	defer closeDir(dir)
+
+	var args C.struct_btrfs_ioctl_search_args
+	args.key.tree_id = C.BTRFS_QUOTA_TREE_OBJECTID
+	args.key.min_type = C.BTRFS_QGROUP_STATUS_KEY
+	args.key.max_type = C.BTRFS_QGROUP_STATUS_KEY
+	args.key.max_objectid = C.__u64(math.MaxUint64)
+	args.key.max_offset = C.__u64(math.MaxUint64)
+	args.key.max_transid = C.__u64(math.MaxUint64)
+	args.key.nr_items = 4096
+
+	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_TREE_SEARCH,
+		uintptr(unsafe.Pointer(&args)))
+	if errno != 0 {
+		return fmt.Errorf("Failed to search qgroup for %s: %v", path, errno.Error())
+	}
+	sh := (*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&args.buf))
+	if sh._type != C.BTRFS_QGROUP_STATUS_KEY {
+		return fmt.Errorf("Invalid qgroup search header type for %s: %v", path, sh._type)
+	}
+	return nil
+}
+
 func subvolLookupQgroup(path string) (uint64, error) {
 	dir, err := openDir(path)
 	if err != nil {
@@ -422,6 +479,14 @@ func (d *Driver) subvolumesDirID(id string) string {
 	return path.Join(d.subvolumesDir(), id)
 }
 
+func (d *Driver) quotasDir() string {
+	return path.Join(d.home, "quotas")
+}
+
+func (d *Driver) quotasDirID(id string) string {
+	return path.Join(d.quotasDir(), id)
+}
+
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
 func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
@@ -430,6 +495,7 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts
 
 // Create the filesystem with given id.
 func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
+	quotas := path.Join(d.home, "quotas")
 	subvolumes := path.Join(d.home, "subvolumes")
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
@@ -466,9 +532,16 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 		if err := d.parseStorageOpt(storageOpt, driver); err != nil {
 			return err
 		}
+
 		if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil {
 			return err
 		}
+		if err := idtools.MkdirAllAs(quotas, 0700, rootUID, rootGID); err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil {
+			return err
+		}
 	}
 
 	// if we have a remapped root (user namespaces enabled), change the created snapshot
@@ -533,7 +606,19 @@ func (d *Driver) Remove(id string) error {
 	if _, err := os.Stat(dir); err != nil {
 		return err
 	}
-	if err := subvolDelete(d.subvolumesDir(), id); err != nil {
+	quotasDir := d.quotasDirID(id)
+	if _, err := os.Stat(quotasDir); err == nil {
+		if err := os.Remove(quotasDir); err != nil {
+			return err
+		}
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+
+	// Call updateQuotaStatus() to invoke status update
+	d.updateQuotaStatus()
+
+	if err := subvolDelete(d.subvolumesDir(), id, d.quotaEnabled); err != nil {
 		return err
 	}
 	if err := system.EnsureRemoveAll(dir); err != nil {
@@ -557,6 +642,17 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 		return "", fmt.Errorf("%s: not a directory", dir)
 	}
 
+	if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil {
+		if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace {
+			if err := d.subvolEnableQuota(); err != nil {
+				return "", err
+			}
+			if err := subvolLimitQgroup(dir, size); err != nil {
+				return "", err
+			}
+		}
+	}
+
 	return dir, nil
 }