From 6966dc0aa9134c518babcbf1f02684cae5374843 Mon Sep 17 00:00:00 2001 From: Sargun Dhillon Date: Sat, 2 Sep 2017 21:25:36 -0700 Subject: [PATCH] Add tests to project quotas and detection mechanism This adds a mechanism (read-only) to check for project quota support in a standard way. This mechanism is leveraged by the tests, which test for the following: 1. Can we get a quota controller? 2. Can we set the quota for a particular directory? 3. Is the quota being over-enforced? 4. Is the quota being under-enforced? 5. Can we retrieve the quota? Signed-off-by: Sargun Dhillon --- daemon/graphdriver/quota/projectquota.go | 53 +++++- daemon/graphdriver/quota/projectquota_test.go | 161 ++++++++++++++++++ 2 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 daemon/graphdriver/quota/projectquota_test.go diff --git a/daemon/graphdriver/quota/projectquota.go b/daemon/graphdriver/quota/projectquota.go index 0e70515434..84e391aa89 100644 --- a/daemon/graphdriver/quota/projectquota.go +++ b/daemon/graphdriver/quota/projectquota.go @@ -47,6 +47,8 @@ struct fsxattr { #ifndef Q_XGETPQUOTA #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) #endif + +const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA); */ import "C" import ( @@ -56,10 +58,15 @@ import ( "path/filepath" "unsafe" + "errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) +// ErrQuotaNotSupported indicates if were found the FS does not have projects quotas available +var ErrQuotaNotSupported = errors.New("Filesystem does not support or has not enabled quotas") + // Quota limit params - currently we only control blocks hard limit type Quota struct { Size uint64 @@ -96,6 +103,24 @@ type Control struct { // project ids. // func NewControl(basePath string) (*Control, error) { + // + // create backing filesystem device node + // + backingFsBlockDev, err := makeBackingFsDev(basePath) + if err != nil { + return nil, err + } + + // check if we can call quotactl with project quotas + // as a mechanism to determine (early) if we have support + hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev) + if err != nil { + return nil, err + } + if !hasQuotaSupport { + return nil, ErrQuotaNotSupported + } + // // Get project id of parent dir as minimal id to be used by driver // @@ -105,14 +130,6 @@ func NewControl(basePath string) (*Control, error) { } minProjectID++ - // - // create backing filesystem device node - // - backingFsBlockDev, err := makeBackingFsDev(basePath) - if err != nil { - return nil, err - } - // // Test if filesystem supports project quotas by trying to set // a quota on the first available project id @@ -335,3 +352,23 @@ func makeBackingFsDev(home string) (string, error) { return backingFsBlockDev, nil } + +func hasQuotaSupport(backingFsBlockDev string) (bool, error) { + var cs = C.CString(backingFsBlockDev) + defer free(cs) + var qstat C.fs_quota_stat_t + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0) + if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 { + return true, nil + } + + switch errno { + // These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota) + case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM: + default: + return false, nil + } + + return false, errno +} diff --git a/daemon/graphdriver/quota/projectquota_test.go b/daemon/graphdriver/quota/projectquota_test.go new file mode 100644 index 0000000000..2b47a58db7 --- /dev/null +++ b/daemon/graphdriver/quota/projectquota_test.go @@ -0,0 +1,161 @@ +// +build linux + +package quota + +import ( + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// 10MB +const testQuotaSize = 10 * 1024 * 1024 +const imageSize = 64 * 1024 * 1024 + +func TestBlockDev(t *testing.T) { + mkfs, err := exec.LookPath("mkfs.xfs") + if err != nil { + t.Fatal("mkfs.xfs not installed") + } + + // create a sparse image + imageFile, err := ioutil.TempFile("", "xfs-image") + if err != nil { + t.Fatal(err) + } + imageFileName := imageFile.Name() + defer os.Remove(imageFileName) + if _, err = imageFile.Seek(imageSize-1, 0); err != nil { + t.Fatal(err) + } + if _, err = imageFile.Write([]byte{0}); err != nil { + t.Fatal(err) + } + if err = imageFile.Close(); err != nil { + t.Fatal(err) + } + + // The reason for disabling these options is sometimes people run with a newer userspace + // than kernelspace + out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput() + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatal(err) + } + + runTest(t, "testBlockDevQuotaDisabled", wrapMountTest(imageFileName, false, testBlockDevQuotaDisabled)) + runTest(t, "testBlockDevQuotaEnabled", wrapMountTest(imageFileName, true, testBlockDevQuotaEnabled)) + runTest(t, "testSmallerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testSmallerThanQuota))) + runTest(t, "testBiggerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testBiggerThanQuota))) + runTest(t, "testRetrieveQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testRetrieveQuota))) +} + +func runTest(t *testing.T, testName string, testFunc func(*testing.T)) { + if success := t.Run(testName, testFunc); !success { + out, _ := exec.Command("dmesg").CombinedOutput() + t.Log(string(out)) + } +} + +func wrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev string)) func(*testing.T) { + return func(t *testing.T) { + mountOptions := "loop" + + if enableQuota { + mountOptions = mountOptions + ",prjquota" + } + + // create a mountPoint + mountPoint, err := ioutil.TempDir("", "xfs-mountPoint") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(mountPoint) + + out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput() + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatal("mount failed") + } + + defer func() { + if err := unix.Unmount(mountPoint, 0); err != nil { + t.Fatal(err) + } + }() + + backingFsDev, err := makeBackingFsDev(mountPoint) + require.NoError(t, err) + + testFunc(t, mountPoint, backingFsDev) + } +} + +func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev string) { + hasSupport, err := hasQuotaSupport(backingFsDev) + require.NoError(t, err) + assert.False(t, hasSupport) +} + +func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev string) { + hasSupport, err := hasQuotaSupport(backingFsDev) + require.NoError(t, err) + assert.True(t, hasSupport) +} + +func wrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev string) { + return func(t *testing.T, mountPoint, backingFsDev string) { + testDir, err := ioutil.TempDir(mountPoint, "per-test") + require.NoError(t, err) + defer os.RemoveAll(testDir) + + ctrl, err := NewControl(testDir) + require.NoError(t, err) + + testSubDir, err := ioutil.TempDir(testDir, "quota-test") + require.NoError(t, err) + testFunc(t, ctrl, mountPoint, testDir, testSubDir) + } + +} + +func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota") + require.NoError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644)) + require.NoError(t, os.Remove(smallerThanQuotaFile)) +} + +func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + // Make sure the quota is being enforced + // TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise + // we're able to violate quota without issue + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + + biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota") + err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644) + require.Error(t, err) + if err == io.ErrShortWrite { + require.NoError(t, os.Remove(biggerThanQuotaFile)) + } +} + +func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + // Validate that we can retrieve quota + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + + var q Quota + require.NoError(t, ctrl.GetQuota(testSubDir, &q)) + assert.EqualValues(t, testQuotaSize, q.Size) +}