Merge pull request #12190 from rhvgoyal/deferred-removal

Devicemapper: Provide deferred device removal capability

daemon/graphdriver/devmapper/README.md

@@ -252,3 +252,23 @@ Here is the list of supported options:
     > Otherwise, set this flag for migrating existing Docker daemons to a
     > daemon with a supported environment.
 
+ *  `dm.use_deferred_removal`
+
+    Enables use of deferred device removal if libdm and kernel driver
+    support the mechanism.
+
+    Deferred device removal means that if device is busy when devices is
+    being removed/deactivated, then a deferred removal is scheduled on
+    device. And devices automatically goes away when last user of device
+    exits.
+
+    For example, when contianer exits, its associated thin device is
+    removed. If that devices has leaked into some other mount namespace
+    can can't be removed now, container exit will still be successful
+    and this option will just schedule device for deferred removal and
+    will not wait in a loop trying to remove a busy device.
+
+    Example use:
+
+    ``docker -d --storage-opt dm.use_deferred_device_removal=true``
+

daemon/graphdriver/devmapper/deviceset.go

@@ -37,7 +37,9 @@ var (
 	// We retry device removal so many a times that even error messages
 	// will fill up console during normal operation. So only log Fatal
 	// messages by default.
-	DMLogLevel int = devicemapper.LogLevelFatal
+	DMLogLevel                   int  = devicemapper.LogLevelFatal
+	DriverDeferredRemovalSupport bool = false
+	EnableDeferredRemoval        bool = false
 )
 
 const deviceSetMetaFile string = "deviceset-metadata"
@@ -103,6 +105,7 @@ type DeviceSet struct {
 	thinPoolDevice        string
 	Transaction           `json:"-"`
 	overrideUdevSyncCheck bool
+	deferredRemove        bool // use deferred removal
 }
 
 type DiskUsage struct {
@@ -112,15 +115,16 @@ type DiskUsage struct {
 }
 
 type Status struct {
-	PoolName          string
-	DataFile          string // actual block device for data
-	DataLoopback      string // loopback file, if used
-	MetadataFile      string // actual block device for metadata
-	MetadataLoopback  string // loopback file, if used
-	Data              DiskUsage
-	Metadata          DiskUsage
-	SectorSize        uint64
-	UdevSyncSupported bool
+	PoolName              string
+	DataFile              string // actual block device for data
+	DataLoopback          string // loopback file, if used
+	MetadataFile          string // actual block device for metadata
+	MetadataLoopback      string // loopback file, if used
+	Data                  DiskUsage
+	Metadata              DiskUsage
+	SectorSize            uint64
+	UdevSyncSupported     bool
+	DeferredRemoveEnabled bool
 }
 
 type DevStatus struct {
@@ -434,6 +438,12 @@ func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, trans
 func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error {
 	logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
 
+	// Make sure deferred removal on device is canceled, if one was
+	// scheduled.
+	if err := devices.cancelDeferredRemoval(info); err != nil {
+		return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err)
+	}
+
 	if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
 		return nil
 	}
@@ -960,16 +970,67 @@ func (devices *DeviceSet) closeTransaction() error {
 	return nil
 }
 
+func determineDriverCapabilities(version string) error {
+	/*
+	 * Driver version 4.27.0 and greater support deferred activation
+	 * feature.
+	 */
+
+	logrus.Debugf("devicemapper: driver version is %s", version)
+
+	versionSplit := strings.Split(version, ".")
+	major, err := strconv.Atoi(versionSplit[0])
+	if err != nil {
+		return graphdriver.ErrNotSupported
+	}
+
+	if major > 4 {
+		DriverDeferredRemovalSupport = true
+		return nil
+	}
+
+	if major < 4 {
+		return nil
+	}
+
+	minor, err := strconv.Atoi(versionSplit[1])
+	if err != nil {
+		return graphdriver.ErrNotSupported
+	}
+
+	/*
+	 * If major is 4 and minor is 27, then there is no need to
+	 * check for patch level as it can not be less than 0.
+	 */
+	if minor >= 27 {
+		DriverDeferredRemovalSupport = true
+		return nil
+	}
+
+	return nil
+}
+
 func (devices *DeviceSet) initDevmapper(doInit bool) error {
 	// give ourselves to libdm as a log handler
 	devicemapper.LogInit(devices)
 
-	_, err := devicemapper.GetDriverVersion()
+	version, err := devicemapper.GetDriverVersion()
 	if err != nil {
 		// Can't even get driver version, assume not supported
 		return graphdriver.ErrNotSupported
 	}
 
+	if err := determineDriverCapabilities(version); err != nil {
+		return graphdriver.ErrNotSupported
+	}
+
+	// If user asked for deferred removal and both library and driver
+	// supports deferred removal use it.
+	if EnableDeferredRemoval && DriverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport == true {
+		logrus.Debugf("devmapper: Deferred removal support enabled.")
+		devices.deferredRemove = true
+	}
+
 	// https://github.com/docker/docker/issues/4036
 	if supported := devicemapper.UdevSetSyncSupport(true); !supported {
 		logrus.Errorf("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/cli/#daemon-storage-driver-option")
@@ -1233,12 +1294,20 @@ func (devices *DeviceSet) deactivateDevice(info *DevInfo) error {
 	if err != nil {
 		return err
 	}
-	if devinfo.Exists != 0 {
+
+	if devinfo.Exists == 0 {
+		return nil
+	}
+
+	if devices.deferredRemove {
+		if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil {
+			return err
+		}
+	} else {
 		if err := devices.removeDevice(info.Name()); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }
 
@@ -1268,6 +1337,45 @@ func (devices *DeviceSet) removeDevice(devname string) error {
 	return err
 }
 
+func (devices *DeviceSet) cancelDeferredRemoval(info *DevInfo) error {
+	if !devices.deferredRemove {
+		return nil
+	}
+
+	logrus.Debugf("[devmapper] cancelDeferredRemoval START(%s)", info.Name())
+	defer logrus.Debugf("[devmapper] cancelDeferredRemoval END(%s)", info.Name)
+
+	devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
+
+	if devinfo != nil && devinfo.DeferredRemove == 0 {
+		return nil
+	}
+
+	// Cancel deferred remove
+	for i := 0; i < 100; i++ {
+		err = devicemapper.CancelDeferredRemove(info.Name())
+		if err == nil {
+			break
+		}
+
+		if err == devicemapper.ErrEnxio {
+			// Device is probably already gone. Return success.
+			return nil
+		}
+
+		if err != devicemapper.ErrBusy {
+			return err
+		}
+
+		// If we see EBUSY it may be a transient error,
+		// sleep a bit a retry a few times.
+		devices.Unlock()
+		time.Sleep(100 * time.Millisecond)
+		devices.Lock()
+	}
+	return err
+}
+
 func (devices *DeviceSet) Shutdown() error {
 	logrus.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix)
 	logrus.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
@@ -1556,6 +1664,7 @@ func (devices *DeviceSet) Status() *Status {
 	status.MetadataFile = devices.MetadataDevicePath()
 	status.MetadataLoopback = devices.metadataLoopFile
 	status.UdevSyncSupported = devicemapper.UdevSyncSupported()
+	status.DeferredRemoveEnabled = devices.deferredRemove
 
 	totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
 	if err == nil {
@@ -1666,6 +1775,13 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error
 			if err != nil {
 				return nil, err
 			}
+
+		case "dm.use_deferred_removal":
+			EnableDeferredRemoval, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
+
 		default:
 			return nil, fmt.Errorf("Unknown option %s\n", key)
 		}

daemon/graphdriver/devmapper/driver.go

@@ -77,6 +77,7 @@ func (d *Driver) Status() [][2]string {
 		{"Metadata Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Total)))},
 		{"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))},
 		{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
+		{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
 	}
 	if len(s.DataLoopback) > 0 {
 		status = append(status, [2]string{"Data loop file", s.DataLoopback})

hack/make.sh

@@ -109,6 +109,15 @@ if \
 	DOCKER_BUILDTAGS+=' btrfs_noversion'
 fi
 
+# test whether "libdevmapper.h" is new enough to support deferred remove
+# functionality.
+if \
+	command -v gcc &> /dev/null \
+	&& ! ( echo -e  '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }'| gcc -ldevmapper -xc - &> /dev/null ) \
+; then
+       DOCKER_BUILDTAGS+=' libdm_no_deferred_remove'
+fi
+
 # Use these flags when compiling the tests and final binary
 
 IAMSTATIC='true'

pkg/devicemapper/devmapper.go

@@ -55,6 +55,7 @@ var (
 	ErrTaskGetDeps            = errors.New("dm_task_get_deps failed")
 	ErrTaskGetInfo            = errors.New("dm_task_get_info failed")
 	ErrTaskGetDriverVersion   = errors.New("dm_task_get_driver_version failed")
+	ErrTaskDeferredRemove     = errors.New("dm_task_deferred_remove failed")
 	ErrTaskSetCookie          = errors.New("dm_task_set_cookie failed")
 	ErrNilCookie              = errors.New("cookie ptr can't be nil")
 	ErrAttachLoopbackDevice   = errors.New("loopback mounting failed")
@@ -69,9 +70,11 @@ var (
 	ErrLoopbackSetCapacity    = errors.New("Unable set loopback capacity")
 	ErrBusy                   = errors.New("Device is Busy")
 	ErrDeviceIdExists         = errors.New("Device Id Exists")
+	ErrEnxio                  = errors.New("No such device or address")
 
 	dmSawBusy  bool
 	dmSawExist bool
+	dmSawEnxio bool // No Such Device or Address
 )
 
 type (
@@ -84,16 +87,17 @@ type (
 		Device []uint64
 	}
 	Info struct {
-		Exists        int
-		Suspended     int
-		LiveTable     int
-		InactiveTable int
-		OpenCount     int32
-		EventNr       uint32
-		Major         uint32
-		Minor         uint32
-		ReadOnly      int
-		TargetCount   int32
+		Exists         int
+		Suspended      int
+		LiveTable      int
+		InactiveTable  int
+		OpenCount      int32
+		EventNr        uint32
+		Major          uint32
+		Minor          uint32
+		ReadOnly       int
+		TargetCount    int32
+		DeferredRemove int
 	}
 	TaskType    int
 	AddNodeType int
@@ -219,6 +223,14 @@ func (t *Task) GetInfo() (*Info, error) {
 	return info, nil
 }
 
+func (t *Task) GetInfoWithDeferred() (*Info, error) {
+	info := &Info{}
+	if res := DmTaskGetInfoWithDeferred(t.unmanaged, info); res != 1 {
+		return nil, ErrTaskGetInfo
+	}
+	return info, nil
+}
+
 func (t *Task) GetDriverVersion() (string, error) {
 	res := DmTaskGetDriverVersion(t.unmanaged)
 	if res == "" {
@@ -371,6 +383,55 @@ func RemoveDevice(name string) error {
 	return nil
 }
 
+func RemoveDeviceDeferred(name string) error {
+	logrus.Debugf("[devmapper] RemoveDeviceDeferred START(%s)", name)
+	defer logrus.Debugf("[devmapper] RemoveDeviceDeferred END(%s)", name)
+	task, err := TaskCreateNamed(DeviceRemove, name)
+	if task == nil {
+		return err
+	}
+
+	if err := DmTaskDeferredRemove(task.unmanaged); err != 1 {
+		return ErrTaskDeferredRemove
+	}
+
+	if err = task.Run(); err != nil {
+		return fmt.Errorf("Error running RemoveDeviceDeferred %s", err)
+	}
+
+	return nil
+}
+
+// Useful helper for cleanup
+func CancelDeferredRemove(deviceName string) error {
+	task, err := TaskCreateNamed(DeviceTargetMsg, deviceName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector %s", err)
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("@cancel_deferred_remove")); err != nil {
+		return fmt.Errorf("Can't set message %s", err)
+	}
+
+	dmSawBusy = false
+	dmSawEnxio = false
+	if err := task.Run(); err != nil {
+		// A device might be being deleted already
+		if dmSawBusy {
+			return ErrBusy
+		} else if dmSawEnxio {
+			return ErrEnxio
+		}
+		return fmt.Errorf("Error running CancelDeferredRemove %s", err)
+
+	}
+	return nil
+}
+
 func GetBlockDeviceSize(file *os.File) (uint64, error) {
 	size, err := ioctlBlkGetSize64(file.Fd())
 	if err != nil {
@@ -479,6 +540,17 @@ func GetInfo(name string) (*Info, error) {
 	return task.GetInfo()
 }
 
+func GetInfoWithDeferred(name string) (*Info, error) {
+	task, err := TaskCreateNamed(DeviceInfo, name)
+	if task == nil {
+		return nil, err
+	}
+	if err := task.Run(); err != nil {
+		return nil, err
+	}
+	return task.GetInfoWithDeferred()
+}
+
 func GetDriverVersion() (string, error) {
 	task := TaskCreate(DeviceVersion)
 	if task == nil {

pkg/devicemapper/devmapper_log.go

@@ -22,6 +22,10 @@ func DevmapperLogCallback(level C.int, file *C.char, line C.int, dm_errno_or_cla
 		if strings.Contains(msg, "File exists") {
 			dmSawExist = true
 		}
+
+		if strings.Contains(msg, "No such device or address") {
+			dmSawEnxio = true
+		}
 	}
 
 	if dmLogger != nil {

pkg/devicemapper/devmapper_wrapper.go

@@ -90,28 +90,30 @@ const (
 )
 
 var (
-	DmGetLibraryVersion    = dmGetLibraryVersionFct
-	DmGetNextTarget        = dmGetNextTargetFct
-	DmLogInitVerbose       = dmLogInitVerboseFct
-	DmSetDevDir            = dmSetDevDirFct
-	DmTaskAddTarget        = dmTaskAddTargetFct
-	DmTaskCreate           = dmTaskCreateFct
-	DmTaskDestroy          = dmTaskDestroyFct
-	DmTaskGetDeps          = dmTaskGetDepsFct
-	DmTaskGetInfo          = dmTaskGetInfoFct
-	DmTaskGetDriverVersion = dmTaskGetDriverVersionFct
-	DmTaskRun              = dmTaskRunFct
-	DmTaskSetAddNode       = dmTaskSetAddNodeFct
-	DmTaskSetCookie        = dmTaskSetCookieFct
-	DmTaskSetMessage       = dmTaskSetMessageFct
-	DmTaskSetName          = dmTaskSetNameFct
-	DmTaskSetRo            = dmTaskSetRoFct
-	DmTaskSetSector        = dmTaskSetSectorFct
-	DmUdevWait             = dmUdevWaitFct
-	DmUdevSetSyncSupport   = dmUdevSetSyncSupportFct
-	DmUdevGetSyncSupport   = dmUdevGetSyncSupportFct
-	DmCookieSupported      = dmCookieSupportedFct
-	LogWithErrnoInit       = logWithErrnoInitFct
+	DmGetLibraryVersion       = dmGetLibraryVersionFct
+	DmGetNextTarget           = dmGetNextTargetFct
+	DmLogInitVerbose          = dmLogInitVerboseFct
+	DmSetDevDir               = dmSetDevDirFct
+	DmTaskAddTarget           = dmTaskAddTargetFct
+	DmTaskCreate              = dmTaskCreateFct
+	DmTaskDestroy             = dmTaskDestroyFct
+	DmTaskGetDeps             = dmTaskGetDepsFct
+	DmTaskGetInfo             = dmTaskGetInfoFct
+	DmTaskGetDriverVersion    = dmTaskGetDriverVersionFct
+	DmTaskRun                 = dmTaskRunFct
+	DmTaskSetAddNode          = dmTaskSetAddNodeFct
+	DmTaskSetCookie           = dmTaskSetCookieFct
+	DmTaskSetMessage          = dmTaskSetMessageFct
+	DmTaskSetName             = dmTaskSetNameFct
+	DmTaskSetRo               = dmTaskSetRoFct
+	DmTaskSetSector           = dmTaskSetSectorFct
+	DmUdevWait                = dmUdevWaitFct
+	DmUdevSetSyncSupport      = dmUdevSetSyncSupportFct
+	DmUdevGetSyncSupport      = dmUdevGetSyncSupportFct
+	DmCookieSupported         = dmCookieSupportedFct
+	LogWithErrnoInit          = logWithErrnoInitFct
+	DmTaskDeferredRemove      = dmTaskDeferredRemoveFct
+	DmTaskGetInfoWithDeferred = dmTaskGetInfoWithDeferredFct
 )
 
 func free(p *C.char) {

pkg/devicemapper/devmapper_wrapper_deferred_remove.go

@@ -0,0 +1,33 @@
+// +build linux,!libdm_no_deferred_remove
+
+package devicemapper
+
+/*
+#cgo LDFLAGS: -L. -ldevmapper
+#include <libdevmapper.h>
+*/
+import "C"
+
+const LibraryDeferredRemovalSupport = true
+
+func dmTaskDeferredRemoveFct(task *CDmTask) int {
+	return int(C.dm_task_deferred_remove((*C.struct_dm_task)(task)))
+}
+
+func dmTaskGetInfoWithDeferredFct(task *CDmTask, info *Info) int {
+	Cinfo := C.struct_dm_info{}
+	defer func() {
+		info.Exists = int(Cinfo.exists)
+		info.Suspended = int(Cinfo.suspended)
+		info.LiveTable = int(Cinfo.live_table)
+		info.InactiveTable = int(Cinfo.inactive_table)
+		info.OpenCount = int32(Cinfo.open_count)
+		info.EventNr = uint32(Cinfo.event_nr)
+		info.Major = uint32(Cinfo.major)
+		info.Minor = uint32(Cinfo.minor)
+		info.ReadOnly = int(Cinfo.read_only)
+		info.TargetCount = int32(Cinfo.target_count)
+		info.DeferredRemove = int(Cinfo.deferred_remove)
+	}()
+	return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
+}

pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go

@@ -0,0 +1,14 @@
+// +build linux,libdm_no_deferred_remove
+
+package devicemapper
+
+const LibraryDeferredRemovalSupport = false
+
+func dmTaskDeferredRemoveFct(task *CDmTask) int {
+	// Error. Nobody should be calling it.
+	return -1
+}
+
+func dmTaskGetInfoWithDeferredFct(task *CDmTask, info *Info) int {
+	return -1
+}