diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index a090b731fa..bd5b67c49c 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -252,3 +252,23 @@ Here is the list of supported options: > Otherwise, set this flag for migrating existing Docker daemons to a > daemon with a supported environment. + * `dm.use_deferred_removal` + + Enables use of deferred device removal if libdm and kernel driver + support the mechanism. + + Deferred device removal means that if device is busy when devices is + being removed/deactivated, then a deferred removal is scheduled on + device. And devices automatically goes away when last user of device + exits. + + For example, when contianer exits, its associated thin device is + removed. If that devices has leaked into some other mount namespace + can can't be removed now, container exit will still be successful + and this option will just schedule device for deferred removal and + will not wait in a loop trying to remove a busy device. + + Example use: + + ``docker -d --storage-opt dm.use_deferred_device_removal=true`` + diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index ca302a951f..24805179f8 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -37,7 +37,9 @@ var ( // We retry device removal so many a times that even error messages // will fill up console during normal operation. So only log Fatal // messages by default. - DMLogLevel int = devicemapper.LogLevelFatal + DMLogLevel int = devicemapper.LogLevelFatal + DriverDeferredRemovalSupport bool = false + EnableDeferredRemoval bool = false ) const deviceSetMetaFile string = "deviceset-metadata" @@ -103,6 +105,7 @@ type DeviceSet struct { thinPoolDevice string Transaction `json:"-"` overrideUdevSyncCheck bool + deferredRemove bool // use deferred removal } type DiskUsage struct { @@ -112,15 +115,16 @@ type DiskUsage struct { } type Status struct { - PoolName string - DataFile string // actual block device for data - DataLoopback string // loopback file, if used - MetadataFile string // actual block device for metadata - MetadataLoopback string // loopback file, if used - Data DiskUsage - Metadata DiskUsage - SectorSize uint64 - UdevSyncSupported bool + PoolName string + DataFile string // actual block device for data + DataLoopback string // loopback file, if used + MetadataFile string // actual block device for metadata + MetadataLoopback string // loopback file, if used + Data DiskUsage + Metadata DiskUsage + SectorSize uint64 + UdevSyncSupported bool + DeferredRemoveEnabled bool } type DevStatus struct { @@ -434,6 +438,12 @@ func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, trans func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error { logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash) + // Make sure deferred removal on device is canceled, if one was + // scheduled. + if err := devices.cancelDeferredRemoval(info); err != nil { + return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err) + } + if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 { return nil } @@ -960,16 +970,67 @@ func (devices *DeviceSet) closeTransaction() error { return nil } +func determineDriverCapabilities(version string) error { + /* + * Driver version 4.27.0 and greater support deferred activation + * feature. + */ + + logrus.Debugf("devicemapper: driver version is %s", version) + + versionSplit := strings.Split(version, ".") + major, err := strconv.Atoi(versionSplit[0]) + if err != nil { + return graphdriver.ErrNotSupported + } + + if major > 4 { + DriverDeferredRemovalSupport = true + return nil + } + + if major < 4 { + return nil + } + + minor, err := strconv.Atoi(versionSplit[1]) + if err != nil { + return graphdriver.ErrNotSupported + } + + /* + * If major is 4 and minor is 27, then there is no need to + * check for patch level as it can not be less than 0. + */ + if minor >= 27 { + DriverDeferredRemovalSupport = true + return nil + } + + return nil +} + func (devices *DeviceSet) initDevmapper(doInit bool) error { // give ourselves to libdm as a log handler devicemapper.LogInit(devices) - _, err := devicemapper.GetDriverVersion() + version, err := devicemapper.GetDriverVersion() if err != nil { // Can't even get driver version, assume not supported return graphdriver.ErrNotSupported } + if err := determineDriverCapabilities(version); err != nil { + return graphdriver.ErrNotSupported + } + + // If user asked for deferred removal and both library and driver + // supports deferred removal use it. + if EnableDeferredRemoval && DriverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport == true { + logrus.Debugf("devmapper: Deferred removal support enabled.") + devices.deferredRemove = true + } + // https://github.com/docker/docker/issues/4036 if supported := devicemapper.UdevSetSyncSupport(true); !supported { logrus.Errorf("Udev sync is not supported. This will lead to unexpected behavior, data loss and errors. For more information, see https://docs.docker.com/reference/commandline/cli/#daemon-storage-driver-option") @@ -1233,12 +1294,20 @@ func (devices *DeviceSet) deactivateDevice(info *DevInfo) error { if err != nil { return err } - if devinfo.Exists != 0 { + + if devinfo.Exists == 0 { + return nil + } + + if devices.deferredRemove { + if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil { + return err + } + } else { if err := devices.removeDevice(info.Name()); err != nil { return err } } - return nil } @@ -1268,6 +1337,45 @@ func (devices *DeviceSet) removeDevice(devname string) error { return err } +func (devices *DeviceSet) cancelDeferredRemoval(info *DevInfo) error { + if !devices.deferredRemove { + return nil + } + + logrus.Debugf("[devmapper] cancelDeferredRemoval START(%s)", info.Name()) + defer logrus.Debugf("[devmapper] cancelDeferredRemoval END(%s)", info.Name) + + devinfo, err := devicemapper.GetInfoWithDeferred(info.Name()) + + if devinfo != nil && devinfo.DeferredRemove == 0 { + return nil + } + + // Cancel deferred remove + for i := 0; i < 100; i++ { + err = devicemapper.CancelDeferredRemove(info.Name()) + if err == nil { + break + } + + if err == devicemapper.ErrEnxio { + // Device is probably already gone. Return success. + return nil + } + + if err != devicemapper.ErrBusy { + return err + } + + // If we see EBUSY it may be a transient error, + // sleep a bit a retry a few times. + devices.Unlock() + time.Sleep(100 * time.Millisecond) + devices.Lock() + } + return err +} + func (devices *DeviceSet) Shutdown() error { logrus.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix) logrus.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) @@ -1556,6 +1664,7 @@ func (devices *DeviceSet) Status() *Status { status.MetadataFile = devices.MetadataDevicePath() status.MetadataLoopback = devices.metadataLoopFile status.UdevSyncSupported = devicemapper.UdevSyncSupported() + status.DeferredRemoveEnabled = devices.deferredRemove totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus() if err == nil { @@ -1666,6 +1775,13 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error if err != nil { return nil, err } + + case "dm.use_deferred_removal": + EnableDeferredRemoval, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } + default: return nil, fmt.Errorf("Unknown option %s\n", key) } diff --git a/daemon/graphdriver/devmapper/driver.go b/daemon/graphdriver/devmapper/driver.go index fad0a0c55d..bdf7f874f9 100644 --- a/daemon/graphdriver/devmapper/driver.go +++ b/daemon/graphdriver/devmapper/driver.go @@ -77,6 +77,7 @@ func (d *Driver) Status() [][2]string { {"Metadata Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Total)))}, {"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))}, {"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)}, + {"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)}, } if len(s.DataLoopback) > 0 { status = append(status, [2]string{"Data loop file", s.DataLoopback}) diff --git a/hack/make.sh b/hack/make.sh index 4226af5bb2..699bedb379 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -109,6 +109,15 @@ if \ DOCKER_BUILDTAGS+=' btrfs_noversion' fi +# test whether "libdevmapper.h" is new enough to support deferred remove +# functionality. +if \ + command -v gcc &> /dev/null \ + && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -ldevmapper -xc - &> /dev/null ) \ +; then + DOCKER_BUILDTAGS+=' libdm_no_deferred_remove' +fi + # Use these flags when compiling the tests and final binary IAMSTATIC='true' diff --git a/pkg/devicemapper/devmapper.go b/pkg/devicemapper/devmapper.go index bb89f7fac2..e7f17b88c4 100644 --- a/pkg/devicemapper/devmapper.go +++ b/pkg/devicemapper/devmapper.go @@ -55,6 +55,7 @@ var ( ErrTaskGetDeps = errors.New("dm_task_get_deps failed") ErrTaskGetInfo = errors.New("dm_task_get_info failed") ErrTaskGetDriverVersion = errors.New("dm_task_get_driver_version failed") + ErrTaskDeferredRemove = errors.New("dm_task_deferred_remove failed") ErrTaskSetCookie = errors.New("dm_task_set_cookie failed") ErrNilCookie = errors.New("cookie ptr can't be nil") ErrAttachLoopbackDevice = errors.New("loopback mounting failed") @@ -69,9 +70,11 @@ var ( ErrLoopbackSetCapacity = errors.New("Unable set loopback capacity") ErrBusy = errors.New("Device is Busy") ErrDeviceIdExists = errors.New("Device Id Exists") + ErrEnxio = errors.New("No such device or address") dmSawBusy bool dmSawExist bool + dmSawEnxio bool // No Such Device or Address ) type ( @@ -84,16 +87,17 @@ type ( Device []uint64 } Info struct { - Exists int - Suspended int - LiveTable int - InactiveTable int - OpenCount int32 - EventNr uint32 - Major uint32 - Minor uint32 - ReadOnly int - TargetCount int32 + Exists int + Suspended int + LiveTable int + InactiveTable int + OpenCount int32 + EventNr uint32 + Major uint32 + Minor uint32 + ReadOnly int + TargetCount int32 + DeferredRemove int } TaskType int AddNodeType int @@ -219,6 +223,14 @@ func (t *Task) GetInfo() (*Info, error) { return info, nil } +func (t *Task) GetInfoWithDeferred() (*Info, error) { + info := &Info{} + if res := DmTaskGetInfoWithDeferred(t.unmanaged, info); res != 1 { + return nil, ErrTaskGetInfo + } + return info, nil +} + func (t *Task) GetDriverVersion() (string, error) { res := DmTaskGetDriverVersion(t.unmanaged) if res == "" { @@ -371,6 +383,55 @@ func RemoveDevice(name string) error { return nil } +func RemoveDeviceDeferred(name string) error { + logrus.Debugf("[devmapper] RemoveDeviceDeferred START(%s)", name) + defer logrus.Debugf("[devmapper] RemoveDeviceDeferred END(%s)", name) + task, err := TaskCreateNamed(DeviceRemove, name) + if task == nil { + return err + } + + if err := DmTaskDeferredRemove(task.unmanaged); err != 1 { + return ErrTaskDeferredRemove + } + + if err = task.Run(); err != nil { + return fmt.Errorf("Error running RemoveDeviceDeferred %s", err) + } + + return nil +} + +// Useful helper for cleanup +func CancelDeferredRemove(deviceName string) error { + task, err := TaskCreateNamed(DeviceTargetMsg, deviceName) + if task == nil { + return err + } + + if err := task.SetSector(0); err != nil { + return fmt.Errorf("Can't set sector %s", err) + } + + if err := task.SetMessage(fmt.Sprintf("@cancel_deferred_remove")); err != nil { + return fmt.Errorf("Can't set message %s", err) + } + + dmSawBusy = false + dmSawEnxio = false + if err := task.Run(); err != nil { + // A device might be being deleted already + if dmSawBusy { + return ErrBusy + } else if dmSawEnxio { + return ErrEnxio + } + return fmt.Errorf("Error running CancelDeferredRemove %s", err) + + } + return nil +} + func GetBlockDeviceSize(file *os.File) (uint64, error) { size, err := ioctlBlkGetSize64(file.Fd()) if err != nil { @@ -479,6 +540,17 @@ func GetInfo(name string) (*Info, error) { return task.GetInfo() } +func GetInfoWithDeferred(name string) (*Info, error) { + task, err := TaskCreateNamed(DeviceInfo, name) + if task == nil { + return nil, err + } + if err := task.Run(); err != nil { + return nil, err + } + return task.GetInfoWithDeferred() +} + func GetDriverVersion() (string, error) { task := TaskCreate(DeviceVersion) if task == nil { diff --git a/pkg/devicemapper/devmapper_log.go b/pkg/devicemapper/devmapper_log.go index d6550bd626..f66a20884b 100644 --- a/pkg/devicemapper/devmapper_log.go +++ b/pkg/devicemapper/devmapper_log.go @@ -22,6 +22,10 @@ func DevmapperLogCallback(level C.int, file *C.char, line C.int, dm_errno_or_cla if strings.Contains(msg, "File exists") { dmSawExist = true } + + if strings.Contains(msg, "No such device or address") { + dmSawEnxio = true + } } if dmLogger != nil { diff --git a/pkg/devicemapper/devmapper_wrapper.go b/pkg/devicemapper/devmapper_wrapper.go index e436cca32d..87c200376f 100644 --- a/pkg/devicemapper/devmapper_wrapper.go +++ b/pkg/devicemapper/devmapper_wrapper.go @@ -90,28 +90,30 @@ const ( ) var ( - DmGetLibraryVersion = dmGetLibraryVersionFct - DmGetNextTarget = dmGetNextTargetFct - DmLogInitVerbose = dmLogInitVerboseFct - DmSetDevDir = dmSetDevDirFct - DmTaskAddTarget = dmTaskAddTargetFct - DmTaskCreate = dmTaskCreateFct - DmTaskDestroy = dmTaskDestroyFct - DmTaskGetDeps = dmTaskGetDepsFct - DmTaskGetInfo = dmTaskGetInfoFct - DmTaskGetDriverVersion = dmTaskGetDriverVersionFct - DmTaskRun = dmTaskRunFct - DmTaskSetAddNode = dmTaskSetAddNodeFct - DmTaskSetCookie = dmTaskSetCookieFct - DmTaskSetMessage = dmTaskSetMessageFct - DmTaskSetName = dmTaskSetNameFct - DmTaskSetRo = dmTaskSetRoFct - DmTaskSetSector = dmTaskSetSectorFct - DmUdevWait = dmUdevWaitFct - DmUdevSetSyncSupport = dmUdevSetSyncSupportFct - DmUdevGetSyncSupport = dmUdevGetSyncSupportFct - DmCookieSupported = dmCookieSupportedFct - LogWithErrnoInit = logWithErrnoInitFct + DmGetLibraryVersion = dmGetLibraryVersionFct + DmGetNextTarget = dmGetNextTargetFct + DmLogInitVerbose = dmLogInitVerboseFct + DmSetDevDir = dmSetDevDirFct + DmTaskAddTarget = dmTaskAddTargetFct + DmTaskCreate = dmTaskCreateFct + DmTaskDestroy = dmTaskDestroyFct + DmTaskGetDeps = dmTaskGetDepsFct + DmTaskGetInfo = dmTaskGetInfoFct + DmTaskGetDriverVersion = dmTaskGetDriverVersionFct + DmTaskRun = dmTaskRunFct + DmTaskSetAddNode = dmTaskSetAddNodeFct + DmTaskSetCookie = dmTaskSetCookieFct + DmTaskSetMessage = dmTaskSetMessageFct + DmTaskSetName = dmTaskSetNameFct + DmTaskSetRo = dmTaskSetRoFct + DmTaskSetSector = dmTaskSetSectorFct + DmUdevWait = dmUdevWaitFct + DmUdevSetSyncSupport = dmUdevSetSyncSupportFct + DmUdevGetSyncSupport = dmUdevGetSyncSupportFct + DmCookieSupported = dmCookieSupportedFct + LogWithErrnoInit = logWithErrnoInitFct + DmTaskDeferredRemove = dmTaskDeferredRemoveFct + DmTaskGetInfoWithDeferred = dmTaskGetInfoWithDeferredFct ) func free(p *C.char) { diff --git a/pkg/devicemapper/devmapper_wrapper_deferred_remove.go b/pkg/devicemapper/devmapper_wrapper_deferred_remove.go new file mode 100644 index 0000000000..ced482c965 --- /dev/null +++ b/pkg/devicemapper/devmapper_wrapper_deferred_remove.go @@ -0,0 +1,33 @@ +// +build linux,!libdm_no_deferred_remove + +package devicemapper + +/* +#cgo LDFLAGS: -L. -ldevmapper +#include +*/ +import "C" + +const LibraryDeferredRemovalSupport = true + +func dmTaskDeferredRemoveFct(task *CDmTask) int { + return int(C.dm_task_deferred_remove((*C.struct_dm_task)(task))) +} + +func dmTaskGetInfoWithDeferredFct(task *CDmTask, info *Info) int { + Cinfo := C.struct_dm_info{} + defer func() { + info.Exists = int(Cinfo.exists) + info.Suspended = int(Cinfo.suspended) + info.LiveTable = int(Cinfo.live_table) + info.InactiveTable = int(Cinfo.inactive_table) + info.OpenCount = int32(Cinfo.open_count) + info.EventNr = uint32(Cinfo.event_nr) + info.Major = uint32(Cinfo.major) + info.Minor = uint32(Cinfo.minor) + info.ReadOnly = int(Cinfo.read_only) + info.TargetCount = int32(Cinfo.target_count) + info.DeferredRemove = int(Cinfo.deferred_remove) + }() + return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo)) +} diff --git a/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go b/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go new file mode 100644 index 0000000000..16631bf19c --- /dev/null +++ b/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -0,0 +1,14 @@ +// +build linux,libdm_no_deferred_remove + +package devicemapper + +const LibraryDeferredRemovalSupport = false + +func dmTaskDeferredRemoveFct(task *CDmTask) int { + // Error. Nobody should be calling it. + return -1 +} + +func dmTaskGetInfoWithDeferredFct(task *CDmTask, info *Info) int { + return -1 +}