mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
098a44c07f
Finish the refactor which was partially completed with commit
34536c498d, passing around IdentityMapping structs instead of pairs of
[]IDMap slices.
Existing code which uses []IDMap relies on zero-valued fields to be
valid, empty mappings. So in order to successfully finish the
refactoring without introducing bugs, their replacement therefore also
needs to have a useful zero value which represents an empty mapping.
Change IdentityMapping to be a pass-by-value type so that there are no
nil pointers to worry about.
The functionality provided by the deprecated NewIDMappingsFromMaps
function is required by unit tests to to construct arbitrary
IdentityMapping values. And the daemon will always need to access the
mappings to pass them to the Linux kernel. Accommodate these use cases
by exporting the struct fields instead. BuildKit currently depends on
the UIDs and GIDs methods so we cannot get rid of them yet.
Signed-off-by: Cory Snider <csnider@mirantis.com>
2822 lines
84 KiB
Go
2822 lines
84 KiB
Go
//go:build linux
|
|
// +build linux
|
|
|
|
package devmapper // import "github.com/docker/docker/daemon/graphdriver/devmapper"
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/docker/docker/daemon/graphdriver"
|
|
"github.com/docker/docker/dockerversion"
|
|
"github.com/docker/docker/pkg/devicemapper"
|
|
"github.com/docker/docker/pkg/dmesg"
|
|
"github.com/docker/docker/pkg/idtools"
|
|
"github.com/docker/docker/pkg/loopback"
|
|
"github.com/docker/docker/pkg/parsers"
|
|
"github.com/docker/docker/pkg/parsers/kernel"
|
|
units "github.com/docker/go-units"
|
|
"github.com/moby/sys/mount"
|
|
"github.com/opencontainers/selinux/go-selinux/label"
|
|
"github.com/pkg/errors"
|
|
"github.com/sirupsen/logrus"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
var (
|
|
defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024
|
|
defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024
|
|
defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024
|
|
defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors
|
|
defaultUdevSyncOverride = false
|
|
maxDeviceID = 0xffffff // 24 bit, pool limit
|
|
deviceIDMapSz = (maxDeviceID + 1) / 8
|
|
driverDeferredRemovalSupport = false
|
|
enableDeferredRemoval = false
|
|
enableDeferredDeletion = false
|
|
userBaseSize = false
|
|
defaultMinFreeSpacePercent uint32 = 10
|
|
lvmSetupConfigForce bool
|
|
)
|
|
|
|
const deviceSetMetaFile = "deviceset-metadata"
|
|
const transactionMetaFile = "transaction-metadata"
|
|
|
|
type transaction struct {
|
|
OpenTransactionID uint64 `json:"open_transaction_id"`
|
|
DeviceIDHash string `json:"device_hash"`
|
|
DeviceID int `json:"device_id"`
|
|
}
|
|
|
|
type devInfo struct {
|
|
Hash string `json:"-"`
|
|
DeviceID int `json:"device_id"`
|
|
Size uint64 `json:"size"`
|
|
TransactionID uint64 `json:"transaction_id"`
|
|
Initialized bool `json:"initialized"`
|
|
Deleted bool `json:"deleted"`
|
|
devices *DeviceSet
|
|
|
|
// The global DeviceSet lock guarantees that we serialize all
|
|
// the calls to libdevmapper (which is not threadsafe), but we
|
|
// sometimes release that lock while sleeping. In that case
|
|
// this per-device lock is still held, protecting against
|
|
// other accesses to the device that we're doing the wait on.
|
|
//
|
|
// WARNING: In order to avoid AB-BA deadlocks when releasing
|
|
// the global lock while holding the per-device locks all
|
|
// device locks must be acquired *before* the device lock, and
|
|
// multiple device locks should be acquired parent before child.
|
|
lock sync.Mutex
|
|
}
|
|
|
|
type metaData struct {
|
|
Devices map[string]*devInfo `json:"Devices"`
|
|
}
|
|
|
|
// DeviceSet holds information about list of devices
|
|
type DeviceSet struct {
|
|
metaData `json:"-"`
|
|
sync.Mutex `json:"-"` // Protects all fields of DeviceSet and serializes calls into libdevmapper
|
|
root string
|
|
devicePrefix string
|
|
TransactionID uint64 `json:"-"`
|
|
NextDeviceID int `json:"next_device_id"`
|
|
deviceIDMap []byte
|
|
|
|
// Options
|
|
dataLoopbackSize int64
|
|
metaDataLoopbackSize int64
|
|
baseFsSize uint64
|
|
filesystem string
|
|
mountOptions string
|
|
mkfsArgs []string
|
|
dataDevice string // block or loop dev
|
|
dataLoopFile string // loopback file, if used
|
|
metadataDevice string // block or loop dev
|
|
metadataLoopFile string // loopback file, if used
|
|
doBlkDiscard bool
|
|
thinpBlockSize uint32
|
|
thinPoolDevice string
|
|
transaction `json:"-"`
|
|
overrideUdevSyncCheck bool
|
|
deferredRemove bool // use deferred removal
|
|
deferredDelete bool // use deferred deletion
|
|
BaseDeviceUUID string // save UUID of base device
|
|
BaseDeviceFilesystem string // save filesystem of base device
|
|
nrDeletedDevices uint // number of deleted devices
|
|
deletionWorkerTicker *time.Ticker
|
|
idMap idtools.IdentityMapping
|
|
minFreeSpacePercent uint32 // min free space percentage in thinpool
|
|
xfsNospaceRetries string // max retries when xfs receives ENOSPC
|
|
lvmSetupConfig directLVMConfig
|
|
}
|
|
|
|
// DiskUsage contains information about disk usage and is used when reporting Status of a device.
|
|
type DiskUsage struct {
|
|
// Used bytes on the disk.
|
|
Used uint64
|
|
// Total bytes on the disk.
|
|
Total uint64
|
|
// Available bytes on the disk.
|
|
Available uint64
|
|
}
|
|
|
|
// Status returns the information about the device.
|
|
type Status struct {
|
|
// PoolName is the name of the data pool.
|
|
PoolName string
|
|
// DataFile is the actual block device for data.
|
|
DataFile string
|
|
// DataLoopback loopback file, if used.
|
|
DataLoopback string
|
|
// MetadataFile is the actual block device for metadata.
|
|
MetadataFile string
|
|
// MetadataLoopback is the loopback file, if used.
|
|
MetadataLoopback string
|
|
// Data is the disk used for data.
|
|
Data DiskUsage
|
|
// Metadata is the disk used for meta data.
|
|
Metadata DiskUsage
|
|
// BaseDeviceSize is base size of container and image
|
|
BaseDeviceSize uint64
|
|
// BaseDeviceFS is backing filesystem.
|
|
BaseDeviceFS string
|
|
// SectorSize size of the vector.
|
|
SectorSize uint64
|
|
// UdevSyncSupported is true if sync is supported.
|
|
UdevSyncSupported bool
|
|
// DeferredRemoveEnabled is true then the device is not unmounted.
|
|
DeferredRemoveEnabled bool
|
|
// True if deferred deletion is enabled. This is different from
|
|
// deferred removal. "removal" means that device mapper device is
|
|
// deactivated. Thin device is still in thin pool and can be activated
|
|
// again. But "deletion" means that thin device will be deleted from
|
|
// thin pool and it can't be activated again.
|
|
DeferredDeleteEnabled bool
|
|
DeferredDeletedDeviceCount uint
|
|
MinFreeSpace uint64
|
|
}
|
|
|
|
// Structure used to export image/container metadata in docker inspect.
|
|
type deviceMetadata struct {
|
|
deviceID int
|
|
deviceSize uint64 // size in bytes
|
|
deviceName string // Device name as used during activation
|
|
}
|
|
|
|
// DevStatus returns information about device mounted containing its id, size and sector information.
|
|
type DevStatus struct {
|
|
// DeviceID is the id of the device.
|
|
DeviceID int
|
|
// Size is the size of the filesystem.
|
|
Size uint64
|
|
// TransactionID is a unique integer per device set used to identify an operation on the file system, this number is incremental.
|
|
TransactionID uint64
|
|
// SizeInSectors indicates the size of the sectors allocated.
|
|
SizeInSectors uint64
|
|
// MappedSectors indicates number of mapped sectors.
|
|
MappedSectors uint64
|
|
// HighestMappedSector is the pointer to the highest mapped sector.
|
|
HighestMappedSector uint64
|
|
}
|
|
|
|
func getDevName(name string) string {
|
|
return "/dev/mapper/" + name
|
|
}
|
|
|
|
func (info *devInfo) Name() string {
|
|
hash := info.Hash
|
|
if hash == "" {
|
|
hash = "base"
|
|
}
|
|
return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
|
|
}
|
|
|
|
func (info *devInfo) DevName() string {
|
|
return getDevName(info.Name())
|
|
}
|
|
|
|
func (devices *DeviceSet) loopbackDir() string {
|
|
return path.Join(devices.root, "devicemapper")
|
|
}
|
|
|
|
func (devices *DeviceSet) metadataDir() string {
|
|
return path.Join(devices.root, "metadata")
|
|
}
|
|
|
|
func (devices *DeviceSet) metadataFile(info *devInfo) string {
|
|
file := info.Hash
|
|
if file == "" {
|
|
file = "base"
|
|
}
|
|
return path.Join(devices.metadataDir(), file)
|
|
}
|
|
|
|
func (devices *DeviceSet) transactionMetaFile() string {
|
|
return path.Join(devices.metadataDir(), transactionMetaFile)
|
|
}
|
|
|
|
func (devices *DeviceSet) deviceSetMetaFile() string {
|
|
return path.Join(devices.metadataDir(), deviceSetMetaFile)
|
|
}
|
|
|
|
func (devices *DeviceSet) oldMetadataFile() string {
|
|
return path.Join(devices.loopbackDir(), "json")
|
|
}
|
|
|
|
func (devices *DeviceSet) getPoolName() string {
|
|
if devices.thinPoolDevice == "" {
|
|
return devices.devicePrefix + "-pool"
|
|
}
|
|
return devices.thinPoolDevice
|
|
}
|
|
|
|
func (devices *DeviceSet) getPoolDevName() string {
|
|
return getDevName(devices.getPoolName())
|
|
}
|
|
|
|
func (devices *DeviceSet) hasImage(name string) bool {
|
|
dirname := devices.loopbackDir()
|
|
filename := path.Join(dirname, name)
|
|
|
|
_, err := os.Stat(filename)
|
|
return err == nil
|
|
}
|
|
|
|
// ensureImage creates a sparse file of <size> bytes at the path
|
|
// <root>/devicemapper/<name>.
|
|
// If the file already exists and new size is larger than its current size, it grows to the new size.
|
|
// Either way it returns the full path.
|
|
func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
|
|
dirname := devices.loopbackDir()
|
|
filename := path.Join(dirname, name)
|
|
|
|
if err := idtools.MkdirAllAndChown(dirname, 0700, devices.idMap.RootPair()); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if fi, err := os.Stat(filename); err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return "", err
|
|
}
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Creating loopback file %s for device-manage use", filename)
|
|
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
|
|
if err := file.Truncate(size); err != nil {
|
|
return "", err
|
|
}
|
|
} else {
|
|
if fi.Size() < size {
|
|
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
if err := file.Truncate(size); err != nil {
|
|
return "", fmt.Errorf("devmapper: Unable to grow loopback file %s: %v", filename, err)
|
|
}
|
|
} else if fi.Size() > size {
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("Can't shrink loopback file %s", filename)
|
|
}
|
|
}
|
|
return filename, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) allocateTransactionID() uint64 {
|
|
devices.OpenTransactionID = devices.TransactionID + 1
|
|
return devices.OpenTransactionID
|
|
}
|
|
|
|
func (devices *DeviceSet) updatePoolTransactionID() error {
|
|
if err := devicemapper.SetTransactionID(devices.getPoolDevName(), devices.TransactionID, devices.OpenTransactionID); err != nil {
|
|
return fmt.Errorf("devmapper: Error setting devmapper transaction ID: %s", err)
|
|
}
|
|
devices.TransactionID = devices.OpenTransactionID
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) removeMetadata(info *devInfo) error {
|
|
if err := os.RemoveAll(devices.metadataFile(info)); err != nil {
|
|
return fmt.Errorf("devmapper: Error removing metadata file %s: %s", devices.metadataFile(info), err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Given json data and file path, write it to disk
|
|
func (devices *DeviceSet) writeMetaFile(jsonData []byte, filePath string) error {
|
|
tmpFile, err := os.CreateTemp(devices.metadataDir(), ".tmp")
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: Error creating metadata file: %s", err)
|
|
}
|
|
|
|
n, err := tmpFile.Write(jsonData)
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: Error writing metadata to %s: %s", tmpFile.Name(), err)
|
|
}
|
|
if n < len(jsonData) {
|
|
return io.ErrShortWrite
|
|
}
|
|
if err := tmpFile.Sync(); err != nil {
|
|
return fmt.Errorf("devmapper: Error syncing metadata file %s: %s", tmpFile.Name(), err)
|
|
}
|
|
if err := tmpFile.Close(); err != nil {
|
|
return fmt.Errorf("devmapper: Error closing metadata file %s: %s", tmpFile.Name(), err)
|
|
}
|
|
if err := os.Rename(tmpFile.Name(), filePath); err != nil {
|
|
return fmt.Errorf("devmapper: Error committing metadata file %s: %s", tmpFile.Name(), err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) saveMetadata(info *devInfo) error {
|
|
jsonData, err := json.Marshal(info)
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
|
|
}
|
|
return devices.writeMetaFile(jsonData, devices.metadataFile(info))
|
|
}
|
|
|
|
func (devices *DeviceSet) markDeviceIDUsed(deviceID int) {
|
|
var mask byte
|
|
i := deviceID % 8
|
|
mask = 1 << uint(i)
|
|
devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] | mask
|
|
}
|
|
|
|
func (devices *DeviceSet) markDeviceIDFree(deviceID int) {
|
|
var mask byte
|
|
i := deviceID % 8
|
|
mask = ^(1 << uint(i))
|
|
devices.deviceIDMap[deviceID/8] = devices.deviceIDMap[deviceID/8] & mask
|
|
}
|
|
|
|
func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool {
|
|
var mask byte
|
|
i := deviceID % 8
|
|
mask = (1 << uint(i))
|
|
return (devices.deviceIDMap[deviceID/8] & mask) == 0
|
|
}
|
|
|
|
// Should be called with devices.Lock() held.
|
|
func (devices *DeviceSet) lookupDevice(hash string) (*devInfo, error) {
|
|
info := devices.Devices[hash]
|
|
if info == nil {
|
|
info = devices.loadMetadata(hash)
|
|
if info == nil {
|
|
return nil, fmt.Errorf("devmapper: Unknown device %s", hash)
|
|
}
|
|
|
|
devices.Devices[hash] = info
|
|
}
|
|
return info, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
info, err := devices.lookupDevice(hash)
|
|
return info, err
|
|
}
|
|
|
|
// This function relies on that device hash map has been loaded in advance.
|
|
// Should be called with devices.Lock() held.
|
|
func (devices *DeviceSet) constructDeviceIDMap() {
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("constructDeviceIDMap()")
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debug("constructDeviceIDMap() END")
|
|
|
|
for _, info := range devices.Devices {
|
|
devices.markDeviceIDUsed(info.DeviceID)
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Added deviceId=%d to DeviceIdMap", info.DeviceID)
|
|
}
|
|
}
|
|
|
|
func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) error {
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
// Skip some of the meta files which are not device files.
|
|
if strings.HasSuffix(finfo.Name(), ".migrated") {
|
|
logger.Debugf("Skipping file %s", path)
|
|
return nil
|
|
}
|
|
|
|
if strings.HasPrefix(finfo.Name(), ".") {
|
|
logger.Debugf("Skipping file %s", path)
|
|
return nil
|
|
}
|
|
|
|
if finfo.Name() == deviceSetMetaFile {
|
|
logger.Debugf("Skipping file %s", path)
|
|
return nil
|
|
}
|
|
|
|
if finfo.Name() == transactionMetaFile {
|
|
logger.Debugf("Skipping file %s", path)
|
|
return nil
|
|
}
|
|
|
|
logger.Debugf("Loading data for file %s", path)
|
|
|
|
hash := finfo.Name()
|
|
if hash == "base" {
|
|
hash = ""
|
|
}
|
|
|
|
// Include deleted devices also as cleanup delete device logic
|
|
// will go through it and see if there are any deleted devices.
|
|
if _, err := devices.lookupDevice(hash); err != nil {
|
|
return fmt.Errorf("devmapper: Error looking up device %s:%v", hash, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) loadDeviceFilesOnStart() error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("loadDeviceFilesOnStart()")
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debug("loadDeviceFilesOnStart() END")
|
|
|
|
var scan = func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Can't walk the file %s", path)
|
|
return nil
|
|
}
|
|
|
|
// Skip any directories
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
return devices.deviceFileWalkFunction(path, info)
|
|
}
|
|
|
|
return filepath.Walk(devices.metadataDir(), scan)
|
|
}
|
|
|
|
// Should be called with devices.Lock() held.
|
|
func (devices *DeviceSet) unregisterDevice(hash string) error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("unregisterDevice(%v)", hash)
|
|
info := &devInfo{
|
|
Hash: hash,
|
|
}
|
|
|
|
delete(devices.Devices, hash)
|
|
|
|
if err := devices.removeMetadata(info); err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Error removing metadata: %s", err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Should be called with devices.Lock() held.
|
|
func (devices *DeviceSet) registerDevice(id int, hash string, size uint64, transactionID uint64) (*devInfo, error) {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("registerDevice(%v, %v)", id, hash)
|
|
info := &devInfo{
|
|
Hash: hash,
|
|
DeviceID: id,
|
|
Size: size,
|
|
TransactionID: transactionID,
|
|
Initialized: false,
|
|
devices: devices,
|
|
}
|
|
|
|
devices.Devices[hash] = info
|
|
|
|
if err := devices.saveMetadata(info); err != nil {
|
|
// Try to remove unused device
|
|
delete(devices.Devices, hash)
|
|
return nil, err
|
|
}
|
|
|
|
return info, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bool) error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("activateDeviceIfNeeded(%v)", info.Hash)
|
|
|
|
if info.Deleted && !ignoreDeleted {
|
|
return fmt.Errorf("devmapper: Can't activate device %v as it is marked for deletion", info.Hash)
|
|
}
|
|
|
|
// Make sure deferred removal on device is canceled, if one was
|
|
// scheduled.
|
|
if err := devices.cancelDeferredRemovalIfNeeded(info); err != nil {
|
|
return fmt.Errorf("devmapper: Device Deferred Removal Cancellation Failed: %s", err)
|
|
}
|
|
|
|
if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
|
|
return nil
|
|
}
|
|
|
|
return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size)
|
|
}
|
|
|
|
// xfsSupported checks if xfs is supported, returns nil if it is, otherwise an error
|
|
func xfsSupported() error {
|
|
// Make sure mkfs.xfs is available
|
|
if _, err := exec.LookPath("mkfs.xfs"); err != nil {
|
|
return err // error text is descriptive enough
|
|
}
|
|
|
|
mountTarget, err := os.MkdirTemp("", "supportsXFS")
|
|
if err != nil {
|
|
return errors.Wrapf(err, "error checking for xfs support")
|
|
}
|
|
|
|
/* The mounting will fail--after the module has been loaded.*/
|
|
defer os.RemoveAll(mountTarget)
|
|
unix.Mount("none", mountTarget, "xfs", 0, "")
|
|
|
|
f, err := os.Open("/proc/filesystems")
|
|
if err != nil {
|
|
return errors.Wrapf(err, "error checking for xfs support")
|
|
}
|
|
defer f.Close()
|
|
|
|
s := bufio.NewScanner(f)
|
|
for s.Scan() {
|
|
if strings.HasSuffix(s.Text(), "\txfs") {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
if err := s.Err(); err != nil {
|
|
return errors.Wrapf(err, "error checking for xfs support")
|
|
}
|
|
|
|
return errors.New(`kernel does not support xfs, or "modprobe xfs" failed`)
|
|
}
|
|
|
|
func determineDefaultFS() string {
|
|
err := xfsSupported()
|
|
if err == nil {
|
|
return "xfs"
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("XFS is not supported in your system (%v). Defaulting to ext4 filesystem", err)
|
|
return "ext4"
|
|
}
|
|
|
|
// mkfsOptions tries to figure out whether some additional mkfs options are required
|
|
func mkfsOptions(fs string) []string {
|
|
if fs == "xfs" && !kernel.CheckKernelVersion(3, 16, 0) {
|
|
// For kernels earlier than 3.16 (and newer xfsutils),
|
|
// some xfs features need to be explicitly disabled.
|
|
return []string{"-m", "crc=0,finobt=0"}
|
|
}
|
|
|
|
return []string{}
|
|
}
|
|
|
|
func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
|
|
devname := info.DevName()
|
|
|
|
if devices.filesystem == "" {
|
|
devices.filesystem = determineDefaultFS()
|
|
}
|
|
if err := devices.saveBaseDeviceFilesystem(devices.filesystem); err != nil {
|
|
return err
|
|
}
|
|
|
|
args := mkfsOptions(devices.filesystem)
|
|
args = append(args, devices.mkfsArgs...)
|
|
args = append(args, devname)
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Infof("Creating filesystem %s on device %s, mkfs args: %v", devices.filesystem, info.Name(), args)
|
|
defer func() {
|
|
if err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Infof("Error while creating filesystem %s on device %s: %v", devices.filesystem, info.Name(), err)
|
|
} else {
|
|
logrus.WithField("storage-driver", "devicemapper").Infof("Successfully created filesystem %s on device %s", devices.filesystem, info.Name())
|
|
}
|
|
}()
|
|
|
|
switch devices.filesystem {
|
|
case "xfs":
|
|
err = exec.Command("mkfs.xfs", args...).Run()
|
|
case "ext4":
|
|
err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0"}, args...)...).Run()
|
|
if err != nil {
|
|
err = exec.Command("mkfs.ext4", append([]string{"-E", "nodiscard,lazy_itable_init=0"}, args...)...).Run()
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = exec.Command("tune2fs", append([]string{"-c", "-1", "-i", "0"}, devname)...).Run()
|
|
default:
|
|
err = fmt.Errorf("devmapper: Unsupported filesystem type %s", devices.filesystem)
|
|
}
|
|
return
|
|
}
|
|
|
|
func (devices *DeviceSet) migrateOldMetaData() error {
|
|
// Migrate old metadata file
|
|
jsonData, err := os.ReadFile(devices.oldMetadataFile())
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
|
|
if jsonData != nil {
|
|
m := metaData{Devices: make(map[string]*devInfo)}
|
|
|
|
if err := json.Unmarshal(jsonData, &m); err != nil {
|
|
return err
|
|
}
|
|
|
|
for hash, info := range m.Devices {
|
|
info.Hash = hash
|
|
devices.saveMetadata(info)
|
|
}
|
|
if err := os.Rename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
|
|
return err
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Cleanup deleted devices. It assumes that all the devices have been
|
|
// loaded in the hash table.
|
|
func (devices *DeviceSet) cleanupDeletedDevices() error {
|
|
devices.Lock()
|
|
|
|
// If there are no deleted devices, there is nothing to do.
|
|
if devices.nrDeletedDevices == 0 {
|
|
devices.Unlock()
|
|
return nil
|
|
}
|
|
|
|
var deletedDevices []*devInfo
|
|
|
|
for _, info := range devices.Devices {
|
|
if !info.Deleted {
|
|
continue
|
|
}
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Found deleted device %s.", info.Hash)
|
|
deletedDevices = append(deletedDevices, info)
|
|
}
|
|
|
|
// Delete the deleted devices. DeleteDevice() first takes the info lock
|
|
// and then devices.Lock(). So drop it to avoid deadlock.
|
|
devices.Unlock()
|
|
|
|
for _, info := range deletedDevices {
|
|
// This will again try deferred deletion.
|
|
if err := devices.DeleteDevice(info.Hash, false); err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("Deletion of device %s, device_id=%v failed:%v", info.Hash, info.DeviceID, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) countDeletedDevices() {
|
|
for _, info := range devices.Devices {
|
|
if !info.Deleted {
|
|
continue
|
|
}
|
|
devices.nrDeletedDevices++
|
|
}
|
|
}
|
|
|
|
func (devices *DeviceSet) startDeviceDeletionWorker() {
|
|
// Deferred deletion is not enabled. Don't do anything.
|
|
if !devices.deferredDelete {
|
|
return
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Worker to cleanup deleted devices started")
|
|
for range devices.deletionWorkerTicker.C {
|
|
devices.cleanupDeletedDevices()
|
|
}
|
|
}
|
|
|
|
func (devices *DeviceSet) initMetaData() error {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
if err := devices.migrateOldMetaData(); err != nil {
|
|
return err
|
|
}
|
|
|
|
_, transactionID, _, _, _, _, err := devices.poolStatus()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
devices.TransactionID = transactionID
|
|
|
|
if err := devices.loadDeviceFilesOnStart(); err != nil {
|
|
return fmt.Errorf("devmapper: Failed to load device files:%v", err)
|
|
}
|
|
|
|
devices.constructDeviceIDMap()
|
|
devices.countDeletedDevices()
|
|
|
|
if err := devices.processPendingTransaction(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Start a goroutine to cleanup Deleted Devices
|
|
go devices.startDeviceDeletionWorker()
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) incNextDeviceID() {
|
|
// IDs are 24bit, so wrap around
|
|
devices.NextDeviceID = (devices.NextDeviceID + 1) & maxDeviceID
|
|
}
|
|
|
|
func (devices *DeviceSet) getNextFreeDeviceID() (int, error) {
|
|
devices.incNextDeviceID()
|
|
for i := 0; i <= maxDeviceID; i++ {
|
|
if devices.isDeviceIDFree(devices.NextDeviceID) {
|
|
devices.markDeviceIDUsed(devices.NextDeviceID)
|
|
return devices.NextDeviceID, nil
|
|
}
|
|
devices.incNextDeviceID()
|
|
}
|
|
|
|
return 0, fmt.Errorf("devmapper: Unable to find a free device ID")
|
|
}
|
|
|
|
func (devices *DeviceSet) poolHasFreeSpace() error {
|
|
if devices.minFreeSpacePercent == 0 {
|
|
return nil
|
|
}
|
|
|
|
_, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
minFreeData := (dataTotal * uint64(devices.minFreeSpacePercent)) / 100
|
|
if minFreeData < 1 {
|
|
minFreeData = 1
|
|
}
|
|
dataFree := dataTotal - dataUsed
|
|
if dataFree < minFreeData {
|
|
return fmt.Errorf("devmapper: Thin Pool has %v free data blocks which is less than minimum required %v free data blocks. Create more free space in thin pool or use dm.min_free_space option to change behavior", (dataTotal - dataUsed), minFreeData)
|
|
}
|
|
|
|
minFreeMetadata := (metadataTotal * uint64(devices.minFreeSpacePercent)) / 100
|
|
if minFreeMetadata < 1 {
|
|
minFreeMetadata = 1
|
|
}
|
|
|
|
metadataFree := metadataTotal - metadataUsed
|
|
if metadataFree < minFreeMetadata {
|
|
return fmt.Errorf("devmapper: Thin Pool has %v free metadata blocks which is less than minimum required %v free metadata blocks. Create more free metadata space in thin pool or use dm.min_free_space option to change behavior", (metadataTotal - metadataUsed), minFreeMetadata)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
deviceID, err := devices.getNextFreeDeviceID()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
if err := devices.openTransaction(hash, deviceID); err != nil {
|
|
logger.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return nil, err
|
|
}
|
|
|
|
for {
|
|
if err := devicemapper.CreateDevice(devices.getPoolDevName(), deviceID); err != nil {
|
|
if devicemapper.DeviceIDExists(err) {
|
|
// Device ID already exists. This should not
|
|
// happen. Now we have a mechanism to find
|
|
// a free device ID. So something is not right.
|
|
// Give a warning and continue.
|
|
logger.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID)
|
|
deviceID, err = devices.getNextFreeDeviceID()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Save new device id into transaction
|
|
devices.refreshTransaction(deviceID)
|
|
continue
|
|
}
|
|
logger.Debugf("Error creating device: %s", err)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return nil, err
|
|
}
|
|
break
|
|
}
|
|
|
|
logger.Debugf("Registering device (id %v) with FS size %v", deviceID, devices.baseFsSize)
|
|
info, err := devices.registerDevice(deviceID, hash, devices.baseFsSize, devices.OpenTransactionID)
|
|
if err != nil {
|
|
_ = devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return nil, err
|
|
}
|
|
|
|
if err := devices.closeTransaction(); err != nil {
|
|
devices.unregisterDevice(hash)
|
|
devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return nil, err
|
|
}
|
|
return info, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) takeSnapshot(hash string, baseInfo *devInfo, size uint64) error {
|
|
var (
|
|
devinfo *devicemapper.Info
|
|
err error
|
|
)
|
|
|
|
if err = devices.poolHasFreeSpace(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if devices.deferredRemove {
|
|
devinfo, err = devicemapper.GetInfoWithDeferred(baseInfo.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if devinfo != nil && devinfo.DeferredRemove != 0 {
|
|
err = devices.cancelDeferredRemoval(baseInfo)
|
|
if err != nil {
|
|
// If Error is ErrEnxio. Device is probably already gone. Continue.
|
|
if err != devicemapper.ErrEnxio {
|
|
return err
|
|
}
|
|
devinfo = nil
|
|
} else {
|
|
defer devices.deactivateDevice(baseInfo)
|
|
}
|
|
}
|
|
} else {
|
|
devinfo, err = devicemapper.GetInfo(baseInfo.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
doSuspend := devinfo != nil && devinfo.Exists != 0
|
|
|
|
if doSuspend {
|
|
if err = devicemapper.SuspendDevice(baseInfo.Name()); err != nil {
|
|
return err
|
|
}
|
|
defer devicemapper.ResumeDevice(baseInfo.Name())
|
|
}
|
|
|
|
return devices.createRegisterSnapDevice(hash, baseInfo, size)
|
|
}
|
|
|
|
func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error {
|
|
deviceID, err := devices.getNextFreeDeviceID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
if err := devices.openTransaction(hash, deviceID); err != nil {
|
|
logger.Debugf("Error opening transaction hash = %s deviceID = %d", hash, deviceID)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return err
|
|
}
|
|
|
|
for {
|
|
if err := devicemapper.CreateSnapDeviceRaw(devices.getPoolDevName(), deviceID, baseInfo.DeviceID); err != nil {
|
|
if devicemapper.DeviceIDExists(err) {
|
|
// Device ID already exists. This should not
|
|
// happen. Now we have a mechanism to find
|
|
// a free device ID. So something is not right.
|
|
// Give a warning and continue.
|
|
logger.Errorf("Device ID %d exists in pool but it is supposed to be unused", deviceID)
|
|
deviceID, err = devices.getNextFreeDeviceID()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Save new device id into transaction
|
|
devices.refreshTransaction(deviceID)
|
|
continue
|
|
}
|
|
logger.Debugf("Error creating snap device: %s", err)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return err
|
|
}
|
|
break
|
|
}
|
|
|
|
if _, err := devices.registerDevice(deviceID, hash, size, devices.OpenTransactionID); err != nil {
|
|
devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
|
|
devices.markDeviceIDFree(deviceID)
|
|
logger.Debugf("Error registering device: %s", err)
|
|
return err
|
|
}
|
|
|
|
if err := devices.closeTransaction(); err != nil {
|
|
devices.unregisterDevice(hash)
|
|
devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
|
|
devices.markDeviceIDFree(deviceID)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) loadMetadata(hash string) *devInfo {
|
|
info := &devInfo{Hash: hash, devices: devices}
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
jsonData, err := os.ReadFile(devices.metadataFile(info))
|
|
if err != nil {
|
|
logger.Debugf("Failed to read %s with err: %v", devices.metadataFile(info), err)
|
|
return nil
|
|
}
|
|
|
|
if err := json.Unmarshal(jsonData, &info); err != nil {
|
|
logger.Debugf("Failed to unmarshal devInfo from %s with err: %v", devices.metadataFile(info), err)
|
|
return nil
|
|
}
|
|
|
|
if info.DeviceID > maxDeviceID {
|
|
logger.Errorf("Ignoring Invalid DeviceId=%d", info.DeviceID)
|
|
return nil
|
|
}
|
|
|
|
return info
|
|
}
|
|
|
|
func getDeviceUUID(device string) (string, error) {
|
|
out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", device).Output()
|
|
if err != nil {
|
|
return "", fmt.Errorf("devmapper: Failed to find uuid for device %s:%v", device, err)
|
|
}
|
|
|
|
uuid := strings.TrimSuffix(string(out), "\n")
|
|
uuid = strings.TrimSpace(uuid)
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("UUID for device: %s is:%s", device, uuid)
|
|
return uuid, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) getBaseDeviceSize() uint64 {
|
|
info, _ := devices.lookupDevice("")
|
|
if info == nil {
|
|
return 0
|
|
}
|
|
return info.Size
|
|
}
|
|
|
|
func (devices *DeviceSet) getBaseDeviceFS() string {
|
|
return devices.BaseDeviceFilesystem
|
|
}
|
|
|
|
func (devices *DeviceSet) verifyBaseDeviceUUIDFS(baseInfo *devInfo) error {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
|
|
return err
|
|
}
|
|
defer devices.deactivateDevice(baseInfo)
|
|
|
|
uuid, err := getDeviceUUID(baseInfo.DevName())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if devices.BaseDeviceUUID != uuid {
|
|
return fmt.Errorf("devmapper: Current Base Device UUID:%s does not match with stored UUID:%s. Possibly using a different thin pool than last invocation", uuid, devices.BaseDeviceUUID)
|
|
}
|
|
|
|
if devices.BaseDeviceFilesystem == "" {
|
|
fsType, err := ProbeFsType(baseInfo.DevName())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := devices.saveBaseDeviceFilesystem(fsType); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// If user specified a filesystem using dm.fs option and current
|
|
// file system of base image is not same, warn user that dm.fs
|
|
// will be ignored.
|
|
if devices.BaseDeviceFilesystem != devices.filesystem {
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("Base device already exists and has filesystem %s on it. User specified filesystem %s will be ignored.", devices.BaseDeviceFilesystem, devices.filesystem)
|
|
devices.filesystem = devices.BaseDeviceFilesystem
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) saveBaseDeviceFilesystem(fs string) error {
|
|
devices.BaseDeviceFilesystem = fs
|
|
return devices.saveDeviceSetMetaData()
|
|
}
|
|
|
|
func (devices *DeviceSet) saveBaseDeviceUUID(baseInfo *devInfo) error {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
if err := devices.activateDeviceIfNeeded(baseInfo, false); err != nil {
|
|
return err
|
|
}
|
|
defer devices.deactivateDevice(baseInfo)
|
|
|
|
uuid, err := getDeviceUUID(baseInfo.DevName())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
devices.BaseDeviceUUID = uuid
|
|
return devices.saveDeviceSetMetaData()
|
|
}
|
|
|
|
func (devices *DeviceSet) createBaseImage() error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Initializing base device-mapper thin volume")
|
|
|
|
// Create initial device
|
|
info, err := devices.createRegisterDevice("")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Creating filesystem on base device-mapper thin volume")
|
|
|
|
if err := devices.activateDeviceIfNeeded(info, false); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := devices.createFilesystem(info); err != nil {
|
|
return err
|
|
}
|
|
|
|
info.Initialized = true
|
|
if err := devices.saveMetadata(info); err != nil {
|
|
info.Initialized = false
|
|
return err
|
|
}
|
|
|
|
if err := devices.saveBaseDeviceUUID(info); err != nil {
|
|
return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Returns if thin pool device exists or not. If device exists, also makes
|
|
// sure it is a thin pool device and not some other type of device.
|
|
func (devices *DeviceSet) thinPoolExists(thinPoolDevice string) (bool, error) {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Checking for existence of the pool %s", thinPoolDevice)
|
|
|
|
info, err := devicemapper.GetInfo(thinPoolDevice)
|
|
if err != nil {
|
|
return false, fmt.Errorf("devmapper: GetInfo() on device %s failed: %v", thinPoolDevice, err)
|
|
}
|
|
|
|
// Device does not exist.
|
|
if info.Exists == 0 {
|
|
return false, nil
|
|
}
|
|
|
|
_, _, deviceType, _, err := devicemapper.GetStatus(thinPoolDevice)
|
|
if err != nil {
|
|
return false, fmt.Errorf("devmapper: GetStatus() on device %s failed: %v", thinPoolDevice, err)
|
|
}
|
|
|
|
if deviceType != "thin-pool" {
|
|
return false, fmt.Errorf("devmapper: Device %s is not a thin pool", thinPoolDevice)
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) checkThinPool() error {
|
|
_, transactionID, dataUsed, _, _, _, err := devices.poolStatus()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dataUsed != 0 {
|
|
return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) that already has used data blocks",
|
|
devices.thinPoolDevice)
|
|
}
|
|
if transactionID != 0 {
|
|
return fmt.Errorf("devmapper: Unable to take ownership of thin-pool (%s) with non-zero transaction ID",
|
|
devices.thinPoolDevice)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Base image is initialized properly. Either save UUID for first time (for
|
|
// upgrade case or verify UUID.
|
|
func (devices *DeviceSet) setupVerifyBaseImageUUIDFS(baseInfo *devInfo) error {
|
|
// If BaseDeviceUUID is nil (upgrade case), save it and return success.
|
|
if devices.BaseDeviceUUID == "" {
|
|
if err := devices.saveBaseDeviceUUID(baseInfo); err != nil {
|
|
return fmt.Errorf("devmapper: Could not query and save base device UUID:%v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if err := devices.verifyBaseDeviceUUIDFS(baseInfo); err != nil {
|
|
return fmt.Errorf("devmapper: Base Device UUID and Filesystem verification failed: %v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) checkGrowBaseDeviceFS(info *devInfo) error {
|
|
|
|
if !userBaseSize {
|
|
return nil
|
|
}
|
|
|
|
if devices.baseFsSize < devices.getBaseDeviceSize() {
|
|
return fmt.Errorf("devmapper: Base device size cannot be smaller than %s", units.HumanSize(float64(devices.getBaseDeviceSize())))
|
|
}
|
|
|
|
if devices.baseFsSize == devices.getBaseDeviceSize() {
|
|
return nil
|
|
}
|
|
|
|
info.lock.Lock()
|
|
defer info.lock.Unlock()
|
|
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
info.Size = devices.baseFsSize
|
|
|
|
if err := devices.saveMetadata(info); err != nil {
|
|
// Try to remove unused device
|
|
delete(devices.Devices, info.Hash)
|
|
return err
|
|
}
|
|
|
|
return devices.growFS(info)
|
|
}
|
|
|
|
func (devices *DeviceSet) growFS(info *devInfo) error {
|
|
if err := devices.activateDeviceIfNeeded(info, false); err != nil {
|
|
return fmt.Errorf("Error activating devmapper device: %s", err)
|
|
}
|
|
|
|
defer devices.deactivateDevice(info)
|
|
|
|
fsMountPoint := "/run/docker/mnt"
|
|
if _, err := os.Stat(fsMountPoint); os.IsNotExist(err) {
|
|
if err := os.MkdirAll(fsMountPoint, 0700); err != nil {
|
|
return err
|
|
}
|
|
defer os.RemoveAll(fsMountPoint)
|
|
}
|
|
|
|
options := ""
|
|
if devices.BaseDeviceFilesystem == "xfs" {
|
|
// XFS needs nouuid or it can't mount filesystems with the same fs
|
|
options = joinMountOptions(options, "nouuid")
|
|
}
|
|
options = joinMountOptions(options, devices.mountOptions)
|
|
|
|
if err := mount.Mount(info.DevName(), fsMountPoint, devices.BaseDeviceFilesystem, options); err != nil {
|
|
return errors.Wrapf(err, "Failed to mount; dmesg: %s", string(dmesg.Dmesg(256)))
|
|
}
|
|
|
|
defer unix.Unmount(fsMountPoint, unix.MNT_DETACH)
|
|
|
|
switch devices.BaseDeviceFilesystem {
|
|
case "ext4":
|
|
if out, err := exec.Command("resize2fs", info.DevName()).CombinedOutput(); err != nil {
|
|
return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
|
|
}
|
|
case "xfs":
|
|
if out, err := exec.Command("xfs_growfs", info.DevName()).CombinedOutput(); err != nil {
|
|
return fmt.Errorf("Failed to grow rootfs:%v:%s", err, string(out))
|
|
}
|
|
default:
|
|
return fmt.Errorf("Unsupported filesystem type %s", devices.BaseDeviceFilesystem)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) setupBaseImage() error {
|
|
oldInfo, _ := devices.lookupDeviceWithLock("")
|
|
|
|
// base image already exists. If it is initialized properly, do UUID
|
|
// verification and return. Otherwise remove image and set it up
|
|
// fresh.
|
|
|
|
if oldInfo != nil {
|
|
if oldInfo.Initialized && !oldInfo.Deleted {
|
|
if err := devices.setupVerifyBaseImageUUIDFS(oldInfo); err != nil {
|
|
return err
|
|
}
|
|
return devices.checkGrowBaseDeviceFS(oldInfo)
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Removing uninitialized base image")
|
|
// If previous base device is in deferred delete state,
|
|
// that needs to be cleaned up first. So don't try
|
|
// deferred deletion.
|
|
if err := devices.DeleteDevice("", true); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// If we are setting up base image for the first time, make sure
|
|
// thin pool is empty.
|
|
if devices.thinPoolDevice != "" && oldInfo == nil {
|
|
if err := devices.checkThinPool(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Create new base image device
|
|
return devices.createBaseImage()
|
|
}
|
|
|
|
func setCloseOnExec(name string) {
|
|
fileInfos, _ := os.ReadDir("/proc/self/fd")
|
|
for _, i := range fileInfos {
|
|
link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
|
|
if link == name {
|
|
fd, err := strconv.Atoi(i.Name())
|
|
if err == nil {
|
|
unix.CloseOnExec(fd)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func major(device uint64) uint64 {
|
|
return (device >> 8) & 0xfff
|
|
}
|
|
|
|
func minor(device uint64) uint64 {
|
|
return (device & 0xff) | ((device >> 12) & 0xfff00)
|
|
}
|
|
|
|
// ResizePool increases the size of the pool.
|
|
func (devices *DeviceSet) ResizePool(size int64) error {
|
|
dirname := devices.loopbackDir()
|
|
datafilename := path.Join(dirname, "data")
|
|
if len(devices.dataDevice) > 0 {
|
|
datafilename = devices.dataDevice
|
|
}
|
|
metadatafilename := path.Join(dirname, "metadata")
|
|
if len(devices.metadataDevice) > 0 {
|
|
metadatafilename = devices.metadataDevice
|
|
}
|
|
|
|
datafile, err := os.OpenFile(datafilename, os.O_RDWR, 0)
|
|
if datafile == nil {
|
|
return err
|
|
}
|
|
defer datafile.Close()
|
|
|
|
fi, err := datafile.Stat()
|
|
if fi == nil {
|
|
return err
|
|
}
|
|
|
|
if fi.Size() > size {
|
|
return fmt.Errorf("devmapper: Can't shrink file")
|
|
}
|
|
|
|
dataloopback := loopback.FindLoopDeviceFor(datafile)
|
|
if dataloopback == nil {
|
|
return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", datafilename)
|
|
}
|
|
defer dataloopback.Close()
|
|
|
|
metadatafile, err := os.OpenFile(metadatafilename, os.O_RDWR, 0)
|
|
if metadatafile == nil {
|
|
return err
|
|
}
|
|
defer metadatafile.Close()
|
|
|
|
metadataloopback := loopback.FindLoopDeviceFor(metadatafile)
|
|
if metadataloopback == nil {
|
|
return fmt.Errorf("devmapper: Unable to find loopback mount for: %s", metadatafilename)
|
|
}
|
|
defer metadataloopback.Close()
|
|
|
|
// Grow loopback file
|
|
if err := datafile.Truncate(size); err != nil {
|
|
return fmt.Errorf("devmapper: Unable to grow loopback file: %s", err)
|
|
}
|
|
|
|
// Reload size for loopback device
|
|
if err := loopback.SetCapacity(dataloopback); err != nil {
|
|
return fmt.Errorf("Unable to update loopback capacity: %s", err)
|
|
}
|
|
|
|
// Suspend the pool
|
|
if err := devicemapper.SuspendDevice(devices.getPoolName()); err != nil {
|
|
return fmt.Errorf("devmapper: Unable to suspend pool: %s", err)
|
|
}
|
|
|
|
// Reload with the new block sizes
|
|
if err := devicemapper.ReloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil {
|
|
return fmt.Errorf("devmapper: Unable to reload pool: %s", err)
|
|
}
|
|
|
|
// Resume the pool
|
|
if err := devicemapper.ResumeDevice(devices.getPoolName()); err != nil {
|
|
return fmt.Errorf("devmapper: Unable to resume pool: %s", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) loadTransactionMetaData() error {
|
|
jsonData, err := os.ReadFile(devices.transactionMetaFile())
|
|
if err != nil {
|
|
// There is no active transaction. This will be the case
|
|
// during upgrade.
|
|
if os.IsNotExist(err) {
|
|
devices.OpenTransactionID = devices.TransactionID
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
json.Unmarshal(jsonData, &devices.transaction)
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) saveTransactionMetaData() error {
|
|
jsonData, err := json.Marshal(&devices.transaction)
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
|
|
}
|
|
|
|
return devices.writeMetaFile(jsonData, devices.transactionMetaFile())
|
|
}
|
|
|
|
func (devices *DeviceSet) removeTransactionMetaData() error {
|
|
return os.RemoveAll(devices.transactionMetaFile())
|
|
}
|
|
|
|
func (devices *DeviceSet) rollbackTransaction() error {
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
logger.Debugf("Rolling back open transaction: TransactionID=%d hash=%s device_id=%d", devices.OpenTransactionID, devices.DeviceIDHash, devices.DeviceID)
|
|
|
|
// A device id might have already been deleted before transaction
|
|
// closed. In that case this call will fail. Just leave a message
|
|
// in case of failure.
|
|
if err := devicemapper.DeleteDevice(devices.getPoolDevName(), devices.DeviceID); err != nil {
|
|
logger.Errorf("Unable to delete device: %s", err)
|
|
}
|
|
|
|
dinfo := &devInfo{Hash: devices.DeviceIDHash}
|
|
if err := devices.removeMetadata(dinfo); err != nil {
|
|
logger.Errorf("Unable to remove metadata: %s", err)
|
|
} else {
|
|
devices.markDeviceIDFree(devices.DeviceID)
|
|
}
|
|
|
|
if err := devices.removeTransactionMetaData(); err != nil {
|
|
logger.Errorf("Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) processPendingTransaction() error {
|
|
if err := devices.loadTransactionMetaData(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// If there was open transaction but pool transaction ID is same
|
|
// as open transaction ID, nothing to roll back.
|
|
if devices.TransactionID == devices.OpenTransactionID {
|
|
return nil
|
|
}
|
|
|
|
// If open transaction ID is less than pool transaction ID, something
|
|
// is wrong. Bail out.
|
|
if devices.OpenTransactionID < devices.TransactionID {
|
|
logrus.WithField("storage-driver", "devicemapper").Errorf("Open Transaction id %d is less than pool transaction id %d", devices.OpenTransactionID, devices.TransactionID)
|
|
return nil
|
|
}
|
|
|
|
// Pool transaction ID is not same as open transaction. There is
|
|
// a transaction which was not completed.
|
|
if err := devices.rollbackTransaction(); err != nil {
|
|
return fmt.Errorf("devmapper: Rolling back open transaction failed: %s", err)
|
|
}
|
|
|
|
devices.OpenTransactionID = devices.TransactionID
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) loadDeviceSetMetaData() error {
|
|
jsonData, err := os.ReadFile(devices.deviceSetMetaFile())
|
|
if err != nil {
|
|
// For backward compatibility return success if file does
|
|
// not exist.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
return json.Unmarshal(jsonData, devices)
|
|
}
|
|
|
|
func (devices *DeviceSet) saveDeviceSetMetaData() error {
|
|
jsonData, err := json.Marshal(devices)
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: Error encoding metadata to json: %s", err)
|
|
}
|
|
|
|
return devices.writeMetaFile(jsonData, devices.deviceSetMetaFile())
|
|
}
|
|
|
|
func (devices *DeviceSet) openTransaction(hash string, DeviceID int) error {
|
|
devices.allocateTransactionID()
|
|
devices.DeviceIDHash = hash
|
|
devices.DeviceID = DeviceID
|
|
if err := devices.saveTransactionMetaData(); err != nil {
|
|
return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) refreshTransaction(DeviceID int) error {
|
|
devices.DeviceID = DeviceID
|
|
if err := devices.saveTransactionMetaData(); err != nil {
|
|
return fmt.Errorf("devmapper: Error saving transaction metadata: %s", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) closeTransaction() error {
|
|
if err := devices.updatePoolTransactionID(); err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Failed to close Transaction")
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func determineDriverCapabilities(version string) error {
|
|
// Kernel driver version >= 4.27.0 support deferred removal
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("kernel dm driver version is %s", version)
|
|
|
|
versionSplit := strings.Split(version, ".")
|
|
major, err := strconv.Atoi(versionSplit[0])
|
|
if err != nil {
|
|
return graphdriver.ErrNotSupported
|
|
}
|
|
|
|
if major > 4 {
|
|
driverDeferredRemovalSupport = true
|
|
return nil
|
|
}
|
|
|
|
if major < 4 {
|
|
return nil
|
|
}
|
|
|
|
minor, err := strconv.Atoi(versionSplit[1])
|
|
if err != nil {
|
|
return graphdriver.ErrNotSupported
|
|
}
|
|
|
|
/*
|
|
* If major is 4 and minor is 27, then there is no need to
|
|
* check for patch level as it can not be less than 0.
|
|
*/
|
|
if minor >= 27 {
|
|
driverDeferredRemovalSupport = true
|
|
return nil
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Determine the major and minor number of loopback device
|
|
func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
|
|
var stat unix.Stat_t
|
|
err := unix.Stat(file.Name(), &stat)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
|
|
// the type is 32bit on mips
|
|
dev := uint64(stat.Rdev) //nolint: unconvert
|
|
majorNum := major(dev)
|
|
minorNum := minor(dev)
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum)
|
|
return majorNum, minorNum, nil
|
|
}
|
|
|
|
// Given a file which is backing file of a loop back device, find the
|
|
// loopback device name and its major/minor number.
|
|
func getLoopFileDeviceMajMin(filename string) (string, uint64, uint64, error) {
|
|
file, err := os.Open(filename)
|
|
if err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Failed to open file %s", filename)
|
|
return "", 0, 0, err
|
|
}
|
|
|
|
defer file.Close()
|
|
loopbackDevice := loopback.FindLoopDeviceFor(file)
|
|
if loopbackDevice == nil {
|
|
return "", 0, 0, fmt.Errorf("devmapper: Unable to find loopback mount for: %s", filename)
|
|
}
|
|
defer loopbackDevice.Close()
|
|
|
|
Major, Minor, err := getDeviceMajorMinor(loopbackDevice)
|
|
if err != nil {
|
|
return "", 0, 0, err
|
|
}
|
|
return loopbackDevice.Name(), Major, Minor, nil
|
|
}
|
|
|
|
// Get the major/minor numbers of thin pool data and metadata devices
|
|
func (devices *DeviceSet) getThinPoolDataMetaMajMin() (uint64, uint64, uint64, uint64, error) {
|
|
var params, poolDataMajMin, poolMetadataMajMin string
|
|
|
|
_, _, _, params, err := devicemapper.GetTable(devices.getPoolName())
|
|
if err != nil {
|
|
return 0, 0, 0, 0, err
|
|
}
|
|
|
|
if _, err = fmt.Sscanf(params, "%s %s", &poolMetadataMajMin, &poolDataMajMin); err != nil {
|
|
return 0, 0, 0, 0, err
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("poolDataMajMin=%s poolMetaMajMin=%s\n", poolDataMajMin, poolMetadataMajMin)
|
|
|
|
poolDataMajMinorSplit := strings.Split(poolDataMajMin, ":")
|
|
poolDataMajor, err := strconv.ParseUint(poolDataMajMinorSplit[0], 10, 32)
|
|
if err != nil {
|
|
return 0, 0, 0, 0, err
|
|
}
|
|
|
|
poolDataMinor, err := strconv.ParseUint(poolDataMajMinorSplit[1], 10, 32)
|
|
if err != nil {
|
|
return 0, 0, 0, 0, err
|
|
}
|
|
|
|
poolMetadataMajMinorSplit := strings.Split(poolMetadataMajMin, ":")
|
|
poolMetadataMajor, err := strconv.ParseUint(poolMetadataMajMinorSplit[0], 10, 32)
|
|
if err != nil {
|
|
return 0, 0, 0, 0, err
|
|
}
|
|
|
|
poolMetadataMinor, err := strconv.ParseUint(poolMetadataMajMinorSplit[1], 10, 32)
|
|
if err != nil {
|
|
return 0, 0, 0, 0, err
|
|
}
|
|
|
|
return poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) loadThinPoolLoopBackInfo() error {
|
|
poolDataMajor, poolDataMinor, poolMetadataMajor, poolMetadataMinor, err := devices.getThinPoolDataMetaMajMin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
dirname := devices.loopbackDir()
|
|
|
|
// data device has not been passed in. So there should be a data file
|
|
// which is being mounted as loop device.
|
|
if devices.dataDevice == "" {
|
|
datafilename := path.Join(dirname, "data")
|
|
dataLoopDevice, dataMajor, dataMinor, err := getLoopFileDeviceMajMin(datafilename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Compare the two
|
|
if poolDataMajor == dataMajor && poolDataMinor == dataMinor {
|
|
devices.dataDevice = dataLoopDevice
|
|
devices.dataLoopFile = datafilename
|
|
}
|
|
|
|
}
|
|
|
|
// metadata device has not been passed in. So there should be a
|
|
// metadata file which is being mounted as loop device.
|
|
if devices.metadataDevice == "" {
|
|
metadatafilename := path.Join(dirname, "metadata")
|
|
metadataLoopDevice, metadataMajor, metadataMinor, err := getLoopFileDeviceMajMin(metadatafilename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if poolMetadataMajor == metadataMajor && poolMetadataMinor == metadataMinor {
|
|
devices.metadataDevice = metadataLoopDevice
|
|
devices.metadataLoopFile = metadatafilename
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) enableDeferredRemovalDeletion() error {
|
|
|
|
// If user asked for deferred removal then check both libdm library
|
|
// and kernel driver support deferred removal otherwise error out.
|
|
if enableDeferredRemoval {
|
|
if !driverDeferredRemovalSupport {
|
|
return fmt.Errorf("devmapper: Deferred removal can not be enabled as kernel does not support it")
|
|
}
|
|
if !devicemapper.LibraryDeferredRemovalSupport {
|
|
return fmt.Errorf("devmapper: Deferred removal can not be enabled as libdm does not support it")
|
|
}
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Deferred removal support enabled.")
|
|
devices.deferredRemove = true
|
|
}
|
|
|
|
if enableDeferredDeletion {
|
|
if !devices.deferredRemove {
|
|
return fmt.Errorf("devmapper: Deferred deletion can not be enabled as deferred removal is not enabled. Enable deferred removal using --storage-opt dm.use_deferred_removal=true parameter")
|
|
}
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("Deferred deletion support enabled.")
|
|
devices.deferredDelete = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) {
|
|
if err := devices.enableDeferredRemovalDeletion(); err != nil {
|
|
return err
|
|
}
|
|
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
// https://github.com/docker/docker/issues/4036
|
|
if supported := devicemapper.UdevSetSyncSupport(true); !supported {
|
|
if dockerversion.IAmStatic == "true" {
|
|
logger.Error("Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
|
|
} else {
|
|
logger.Error("Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
|
|
}
|
|
|
|
if !devices.overrideUdevSyncCheck {
|
|
return graphdriver.ErrNotSupported
|
|
}
|
|
}
|
|
|
|
// create the root dir of the devmapper driver ownership to match this
|
|
// daemon's remapped root uid/gid so containers can start properly
|
|
if err := idtools.MkdirAndChown(devices.root, 0700, devices.idMap.RootPair()); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil {
|
|
return err
|
|
}
|
|
|
|
prevSetupConfig, err := readLVMConfig(devices.root)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
|
|
if devices.thinPoolDevice != "" {
|
|
return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
|
|
}
|
|
|
|
if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
|
|
if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
|
|
return errors.New("changing direct-lvm config is not supported")
|
|
}
|
|
logger.WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
|
|
if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
|
|
return err
|
|
}
|
|
if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
|
|
return err
|
|
}
|
|
if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
devices.thinPoolDevice = "docker-thinpool"
|
|
logger.Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
|
|
}
|
|
|
|
// Set the device prefix from the device id and inode of the docker root dir
|
|
var st unix.Stat_t
|
|
if err := unix.Stat(devices.root, &st); err != nil {
|
|
return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
|
|
}
|
|
// "reg-" stands for "regular file".
|
|
// In the future we might use "dev-" for "device file", etc.
|
|
// docker-maj,min[-inode] stands for:
|
|
// - Managed by docker
|
|
// - The target of this device is at major <maj> and minor <min>
|
|
// - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
|
|
// The type Dev in Stat_t is 32bit on mips.
|
|
devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(uint64(st.Dev)), minor(uint64(st.Dev)), st.Ino) //nolint: unconvert
|
|
logger.Debugf("Generated prefix: %s", devices.devicePrefix)
|
|
|
|
// Check for the existence of the thin-pool device
|
|
poolExists, err := devices.thinPoolExists(devices.getPoolName())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
|
|
// that are not Close-on-exec,
|
|
// so we add this badhack to make sure it closes itself
|
|
setCloseOnExec("/dev/mapper/control")
|
|
|
|
// Make sure the sparse images exist in <root>/devicemapper/data and
|
|
// <root>/devicemapper/metadata
|
|
|
|
createdLoopback := false
|
|
|
|
// If the pool doesn't exist, create it
|
|
if !poolExists && devices.thinPoolDevice == "" {
|
|
logger.Debug("Pool doesn't exist. Creating it.")
|
|
|
|
var (
|
|
dataFile *os.File
|
|
metadataFile *os.File
|
|
)
|
|
|
|
if devices.dataDevice == "" {
|
|
// Make sure the sparse images exist in <root>/devicemapper/data
|
|
|
|
hasData := devices.hasImage("data")
|
|
|
|
if !doInit && !hasData {
|
|
return errors.New("loopback data file not found")
|
|
}
|
|
|
|
if !hasData {
|
|
createdLoopback = true
|
|
}
|
|
|
|
data, err := devices.ensureImage("data", devices.dataLoopbackSize)
|
|
if err != nil {
|
|
logger.Debugf("Error device ensureImage (data): %s", err)
|
|
return err
|
|
}
|
|
|
|
dataFile, err = loopback.AttachLoopDevice(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
devices.dataLoopFile = data
|
|
devices.dataDevice = dataFile.Name()
|
|
} else {
|
|
dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
defer dataFile.Close()
|
|
|
|
if devices.metadataDevice == "" {
|
|
// Make sure the sparse images exist in <root>/devicemapper/metadata
|
|
|
|
hasMetadata := devices.hasImage("metadata")
|
|
|
|
if !doInit && !hasMetadata {
|
|
return errors.New("loopback metadata file not found")
|
|
}
|
|
|
|
if !hasMetadata {
|
|
createdLoopback = true
|
|
}
|
|
|
|
metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
|
|
if err != nil {
|
|
logger.Debugf("Error device ensureImage (metadata): %s", err)
|
|
return err
|
|
}
|
|
|
|
metadataFile, err = loopback.AttachLoopDevice(metadata)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
devices.metadataLoopFile = metadata
|
|
devices.metadataDevice = metadataFile.Name()
|
|
} else {
|
|
metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
defer metadataFile.Close()
|
|
|
|
if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
if retErr != nil {
|
|
err = devices.deactivatePool()
|
|
if err != nil {
|
|
logger.Warnf("Failed to deactivatePool: %v", err)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Pool already exists and caller did not pass us a pool. That means
|
|
// we probably created pool earlier and could not remove it as some
|
|
// containers were still using it. Detect some of the properties of
|
|
// pool, like is it using loop devices.
|
|
if poolExists && devices.thinPoolDevice == "" {
|
|
if err := devices.loadThinPoolLoopBackInfo(); err != nil {
|
|
logger.Debugf("Failed to load thin pool loopback device information:%v", err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
// If we didn't just create the data or metadata image, we need to
|
|
// load the transaction id and migrate old metadata
|
|
if !createdLoopback {
|
|
if err := devices.initMetaData(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if devices.thinPoolDevice == "" {
|
|
if devices.metadataLoopFile != "" || devices.dataLoopFile != "" {
|
|
logger.Warn("Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man dockerd` to refer to dm.thinpooldev section.")
|
|
}
|
|
}
|
|
|
|
// Right now this loads only NextDeviceID. If there is more metadata
|
|
// down the line, we might have to move it earlier.
|
|
if err := devices.loadDeviceSetMetaData(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Setup the base image
|
|
if doInit {
|
|
if err := devices.setupBaseImage(); err != nil {
|
|
logger.Debugf("Error device setupBaseImage: %s", err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// AddDevice adds a device and registers in the hash.
|
|
func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string]string) error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("AddDevice START(hash=%s basehash=%s)", hash, baseHash)
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debugf("AddDevice END(hash=%s basehash=%s)", hash, baseHash)
|
|
|
|
// If a deleted device exists, return error.
|
|
baseInfo, err := devices.lookupDeviceWithLock(baseHash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if baseInfo.Deleted {
|
|
return fmt.Errorf("devmapper: Base device %v has been marked for deferred deletion", baseInfo.Hash)
|
|
}
|
|
|
|
baseInfo.lock.Lock()
|
|
defer baseInfo.lock.Unlock()
|
|
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
// Also include deleted devices in case hash of new device is
|
|
// same as one of the deleted devices.
|
|
if info, _ := devices.lookupDevice(hash); info != nil {
|
|
return fmt.Errorf("devmapper: device %s already exists. Deleted=%v", hash, info.Deleted)
|
|
}
|
|
|
|
size, err := devices.parseStorageOpt(storageOpt)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if size == 0 {
|
|
size = baseInfo.Size
|
|
}
|
|
|
|
if size < baseInfo.Size {
|
|
return fmt.Errorf("devmapper: Container size cannot be smaller than %s", units.HumanSize(float64(baseInfo.Size)))
|
|
}
|
|
|
|
if err := devices.takeSnapshot(hash, baseInfo, size); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Grow the container rootfs.
|
|
if size > baseInfo.Size {
|
|
info, err := devices.lookupDevice(hash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := devices.growFS(info); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) parseStorageOpt(storageOpt map[string]string) (uint64, error) {
|
|
|
|
// Read size to change the block device size per container.
|
|
for key, val := range storageOpt {
|
|
key := strings.ToLower(key)
|
|
switch key {
|
|
case "size":
|
|
size, err := units.RAMInBytes(val)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return uint64(size), nil
|
|
default:
|
|
return 0, fmt.Errorf("Unknown option %s", key)
|
|
}
|
|
}
|
|
|
|
return 0, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) markForDeferredDeletion(info *devInfo) error {
|
|
// If device is already in deleted state, there is nothing to be done.
|
|
if info.Deleted {
|
|
return nil
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Marking device %s for deferred deletion.", info.Hash)
|
|
|
|
info.Deleted = true
|
|
|
|
// save device metadata to reflect deleted state.
|
|
if err := devices.saveMetadata(info); err != nil {
|
|
info.Deleted = false
|
|
return err
|
|
}
|
|
|
|
devices.nrDeletedDevices++
|
|
return nil
|
|
}
|
|
|
|
// Should be called with devices.Lock() held.
|
|
func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) error {
|
|
if err := devices.openTransaction(info.Hash, info.DeviceID); err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceID)
|
|
return err
|
|
}
|
|
|
|
defer devices.closeTransaction()
|
|
|
|
err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceID)
|
|
if err != nil {
|
|
// If syncDelete is true, we want to return error. If deferred
|
|
// deletion is not enabled, we return an error. If error is
|
|
// something other then EBUSY, return an error.
|
|
if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Error deleting device: %s", err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err == nil {
|
|
if err := devices.unregisterDevice(info.Hash); err != nil {
|
|
return err
|
|
}
|
|
// If device was already in deferred delete state that means
|
|
// deletion was being tried again later. Reduce the deleted
|
|
// device count.
|
|
if info.Deleted {
|
|
devices.nrDeletedDevices--
|
|
}
|
|
devices.markDeviceIDFree(info.DeviceID)
|
|
} else {
|
|
if err := devices.markForDeferredDeletion(info); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Issue discard only if device open count is zero.
|
|
func (devices *DeviceSet) issueDiscard(info *devInfo) error {
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
logger.Debugf("issueDiscard START(device: %s).", info.Hash)
|
|
defer logger.Debugf("issueDiscard END(device: %s).", info.Hash)
|
|
// This is a workaround for the kernel not discarding block so
|
|
// on the thin pool when we remove a thinp device, so we do it
|
|
// manually.
|
|
// Even if device is deferred deleted, activate it and issue
|
|
// discards.
|
|
if err := devices.activateDeviceIfNeeded(info, true); err != nil {
|
|
return err
|
|
}
|
|
|
|
devinfo, err := devicemapper.GetInfo(info.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if devinfo.OpenCount != 0 {
|
|
logger.Debugf("Device: %s is in use. OpenCount=%d. Not issuing discards.", info.Hash, devinfo.OpenCount)
|
|
return nil
|
|
}
|
|
|
|
if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
|
|
logger.Debugf("Error discarding block on device: %s (ignoring)", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Should be called with devices.Lock() held.
|
|
func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error {
|
|
if devices.doBlkDiscard {
|
|
devices.issueDiscard(info)
|
|
}
|
|
|
|
// Try to deactivate device in case it is active.
|
|
// If deferred removal is enabled and deferred deletion is disabled
|
|
// then make sure device is removed synchronously. There have been
|
|
// some cases of device being busy for short duration and we would
|
|
// rather busy wait for device removal to take care of these cases.
|
|
deferredRemove := devices.deferredRemove
|
|
if !devices.deferredDelete {
|
|
deferredRemove = false
|
|
}
|
|
|
|
if err := devices.deactivateDeviceMode(info, deferredRemove); err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("Error deactivating device: %s", err)
|
|
return err
|
|
}
|
|
|
|
return devices.deleteTransaction(info, syncDelete)
|
|
}
|
|
|
|
// DeleteDevice will return success if device has been marked for deferred
|
|
// removal. If one wants to override that and want DeleteDevice() to fail if
|
|
// device was busy and could not be deleted, set syncDelete=true.
|
|
func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("DeleteDevice START(hash=%v syncDelete=%v)", hash, syncDelete)
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debugf("DeleteDevice END(hash=%v syncDelete=%v)", hash, syncDelete)
|
|
info, err := devices.lookupDeviceWithLock(hash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
info.lock.Lock()
|
|
defer info.lock.Unlock()
|
|
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
return devices.deleteDevice(info, syncDelete)
|
|
}
|
|
|
|
func (devices *DeviceSet) deactivatePool() error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debug("deactivatePool() START")
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debug("deactivatePool() END")
|
|
devname := devices.getPoolDevName()
|
|
|
|
devinfo, err := devicemapper.GetInfo(devname)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if devinfo.Exists == 0 {
|
|
return nil
|
|
}
|
|
if err := devicemapper.RemoveDevice(devname); err != nil {
|
|
return err
|
|
}
|
|
|
|
if d, err := devicemapper.GetDeps(devname); err == nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("device %s still has %d active dependents", devname, d.Count)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
|
|
return devices.deactivateDeviceMode(info, devices.deferredRemove)
|
|
}
|
|
|
|
func (devices *DeviceSet) deactivateDeviceMode(info *devInfo, deferredRemove bool) error {
|
|
var err error
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("deactivateDevice START(%s)", info.Hash)
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debugf("deactivateDevice END(%s)", info.Hash)
|
|
|
|
devinfo, err := devicemapper.GetInfo(info.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if devinfo.Exists == 0 {
|
|
return nil
|
|
}
|
|
|
|
if deferredRemove {
|
|
err = devicemapper.RemoveDeviceDeferred(info.Name())
|
|
} else {
|
|
err = devices.removeDevice(info.Name())
|
|
}
|
|
|
|
// This function's semantics is such that it does not return an
|
|
// error if device does not exist. So if device went away by
|
|
// the time we actually tried to remove it, do not return error.
|
|
if err != devicemapper.ErrEnxio {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Issues the underlying dm remove operation.
|
|
func (devices *DeviceSet) removeDevice(devname string) error {
|
|
var err error
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("removeDevice START(%s)", devname)
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debugf("removeDevice END(%s)", devname)
|
|
|
|
for i := 0; i < 200; i++ {
|
|
err = devicemapper.RemoveDevice(devname)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if err != devicemapper.ErrBusy {
|
|
return err
|
|
}
|
|
|
|
// If we see EBUSY it may be a transient error,
|
|
// sleep a bit a retry a few times.
|
|
devices.Unlock()
|
|
time.Sleep(100 * time.Millisecond)
|
|
devices.Lock()
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func (devices *DeviceSet) cancelDeferredRemovalIfNeeded(info *devInfo) error {
|
|
if !devices.deferredRemove {
|
|
return nil
|
|
}
|
|
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemovalIfNeeded START(%s)", info.Name())
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemovalIfNeeded END(%s)", info.Name())
|
|
|
|
devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if devinfo != nil && devinfo.DeferredRemove == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Cancel deferred remove
|
|
if err := devices.cancelDeferredRemoval(info); err != nil {
|
|
// If Error is ErrEnxio. Device is probably already gone. Continue.
|
|
if err != devicemapper.ErrEnxio {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
|
|
logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemoval START(%s)", info.Name())
|
|
defer logrus.WithField("storage-driver", "devicemapper").Debugf("cancelDeferredRemoval END(%s)", info.Name())
|
|
|
|
var err error
|
|
|
|
// Cancel deferred remove
|
|
for i := 0; i < 100; i++ {
|
|
err = devicemapper.CancelDeferredRemove(info.Name())
|
|
if err != nil {
|
|
if err == devicemapper.ErrBusy {
|
|
// If we see EBUSY it may be a transient error,
|
|
// sleep a bit a retry a few times.
|
|
devices.Unlock()
|
|
time.Sleep(100 * time.Millisecond)
|
|
devices.Lock()
|
|
continue
|
|
}
|
|
}
|
|
break
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (devices *DeviceSet) unmountAndDeactivateAll(dir string) {
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
files, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
logger.Warnf("unmountAndDeactivate: %s", err)
|
|
return
|
|
}
|
|
|
|
for _, d := range files {
|
|
if !d.IsDir() {
|
|
continue
|
|
}
|
|
|
|
name := d.Name()
|
|
fullname := path.Join(dir, name)
|
|
|
|
// We use MNT_DETACH here in case it is still busy in some running
|
|
// container. This means it'll go away from the global scope directly,
|
|
// and the device will be released when that container dies.
|
|
if err := unix.Unmount(fullname, unix.MNT_DETACH); err != nil && err != unix.EINVAL {
|
|
logger.Warnf("Shutdown unmounting %s, error: %s", fullname, err)
|
|
}
|
|
|
|
if devInfo, err := devices.lookupDevice(name); err != nil {
|
|
logger.Debugf("Shutdown lookup device %s, error: %s", name, err)
|
|
} else {
|
|
if err := devices.deactivateDevice(devInfo); err != nil {
|
|
logger.Debugf("Shutdown deactivate %s, error: %s", devInfo.Hash, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Shutdown shuts down the device by unmounting the root.
|
|
func (devices *DeviceSet) Shutdown(home string) error {
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
logger.Debugf("[deviceset %s] Shutdown()", devices.devicePrefix)
|
|
logger.Debugf("Shutting down DeviceSet: %s", devices.root)
|
|
defer logger.Debugf("[deviceset %s] Shutdown() END", devices.devicePrefix)
|
|
|
|
// Stop deletion worker. This should start delivering new events to
|
|
// ticker channel. That means no new instance of cleanupDeletedDevice()
|
|
// will run after this call. If one instance is already running at
|
|
// the time of the call, it must be holding devices.Lock() and
|
|
// we will block on this lock till cleanup function exits.
|
|
devices.deletionWorkerTicker.Stop()
|
|
|
|
devices.Lock()
|
|
// Save DeviceSet Metadata first. Docker kills all threads if they
|
|
// don't finish in certain time. It is possible that Shutdown()
|
|
// routine does not finish in time as we loop trying to deactivate
|
|
// some devices while these are busy. In that case shutdown() routine
|
|
// will be killed and we will not get a chance to save deviceset
|
|
// metadata. Hence save this early before trying to deactivate devices.
|
|
devices.saveDeviceSetMetaData()
|
|
devices.unmountAndDeactivateAll(path.Join(home, "mnt"))
|
|
devices.Unlock()
|
|
|
|
info, _ := devices.lookupDeviceWithLock("")
|
|
if info != nil {
|
|
info.lock.Lock()
|
|
devices.Lock()
|
|
if err := devices.deactivateDevice(info); err != nil {
|
|
logger.Debugf("Shutdown deactivate base , error: %s", err)
|
|
}
|
|
devices.Unlock()
|
|
info.lock.Unlock()
|
|
}
|
|
|
|
devices.Lock()
|
|
if devices.thinPoolDevice == "" {
|
|
if err := devices.deactivatePool(); err != nil {
|
|
logger.Debugf("Shutdown deactivate pool , error: %s", err)
|
|
}
|
|
}
|
|
devices.Unlock()
|
|
|
|
return nil
|
|
}
|
|
|
|
// Recent XFS changes allow changing behavior of filesystem in case of errors.
|
|
// When thin pool gets full and XFS gets ENOSPC error, currently it tries
|
|
// IO infinitely and sometimes it can block the container process
|
|
// and process can't be killWith 0 value, XFS will not retry upon error
|
|
// and instead will shutdown filesystem.
|
|
|
|
func (devices *DeviceSet) xfsSetNospaceRetries(info *devInfo) error {
|
|
dmDevicePath, err := os.Readlink(info.DevName())
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: readlink failed for device %v:%v", info.DevName(), err)
|
|
}
|
|
|
|
dmDeviceName := path.Base(dmDevicePath)
|
|
filePath := "/sys/fs/xfs/" + dmDeviceName + "/error/metadata/ENOSPC/max_retries"
|
|
maxRetriesFile, err := os.OpenFile(filePath, os.O_WRONLY, 0)
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: user specified daemon option dm.xfs_nospace_max_retries but it does not seem to be supported on this system :%v", err)
|
|
}
|
|
defer maxRetriesFile.Close()
|
|
|
|
// Set max retries to 0
|
|
_, err = maxRetriesFile.WriteString(devices.xfsNospaceRetries)
|
|
if err != nil {
|
|
return fmt.Errorf("devmapper: Failed to write string %v to file %v:%v", devices.xfsNospaceRetries, filePath, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// MountDevice mounts the device if not already mounted.
|
|
func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
|
|
info, err := devices.lookupDeviceWithLock(hash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if info.Deleted {
|
|
return fmt.Errorf("devmapper: Can't mount device %v as it has been marked for deferred deletion", info.Hash)
|
|
}
|
|
|
|
info.lock.Lock()
|
|
defer info.lock.Unlock()
|
|
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
if err := devices.activateDeviceIfNeeded(info, false); err != nil {
|
|
return fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
|
|
}
|
|
|
|
fstype, err := ProbeFsType(info.DevName())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
options := ""
|
|
|
|
if fstype == "xfs" {
|
|
// XFS needs nouuid or it can't mount filesystems with the same fs
|
|
options = joinMountOptions(options, "nouuid")
|
|
}
|
|
|
|
options = joinMountOptions(options, devices.mountOptions)
|
|
options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
|
|
|
|
if err := mount.Mount(info.DevName(), path, fstype, options); err != nil {
|
|
return errors.Wrapf(err, "Failed to mount; dmesg: %s", string(dmesg.Dmesg(256)))
|
|
}
|
|
|
|
if fstype == "xfs" && devices.xfsNospaceRetries != "" {
|
|
if err := devices.xfsSetNospaceRetries(info); err != nil {
|
|
unix.Unmount(path, unix.MNT_DETACH)
|
|
devices.deactivateDevice(info)
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UnmountDevice unmounts the device and removes it from hash.
|
|
func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error {
|
|
logger := logrus.WithField("storage-driver", "devicemapper")
|
|
|
|
logger.Debugf("UnmountDevice START(hash=%s)", hash)
|
|
defer logger.Debugf("UnmountDevice END(hash=%s)", hash)
|
|
|
|
info, err := devices.lookupDeviceWithLock(hash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
info.lock.Lock()
|
|
defer info.lock.Unlock()
|
|
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
logger.Debugf("Unmount(%s)", mountPath)
|
|
if err := unix.Unmount(mountPath, unix.MNT_DETACH); err != nil {
|
|
return err
|
|
}
|
|
logger.Debug("Unmount done")
|
|
|
|
// Remove the mountpoint here. Removing the mountpoint (in newer kernels)
|
|
// will cause all other instances of this mount in other mount namespaces
|
|
// to be killed (this is an anti-DoS measure that is necessary for things
|
|
// like devicemapper). This is necessary to avoid cases where a libdm mount
|
|
// that is present in another namespace will cause subsequent RemoveDevice
|
|
// operations to fail. We ignore any errors here because this may fail on
|
|
// older kernels which don't have
|
|
// torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe applied.
|
|
if err := os.Remove(mountPath); err != nil {
|
|
logger.Debugf("error doing a remove on unmounted device %s: %v", mountPath, err)
|
|
}
|
|
|
|
return devices.deactivateDevice(info)
|
|
}
|
|
|
|
// HasDevice returns true if the device metadata exists.
|
|
func (devices *DeviceSet) HasDevice(hash string) bool {
|
|
info, _ := devices.lookupDeviceWithLock(hash)
|
|
return info != nil
|
|
}
|
|
|
|
// List returns a list of device ids.
|
|
func (devices *DeviceSet) List() []string {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
ids := make([]string, len(devices.Devices))
|
|
i := 0
|
|
for k := range devices.Devices {
|
|
ids[i] = k
|
|
i++
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func (devices *DeviceSet) deviceStatus(devName string) (sizeInSectors, mappedSectors, highestMappedSector uint64, err error) {
|
|
var params string
|
|
_, sizeInSectors, _, params, err = devicemapper.GetStatus(devName)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if _, err = fmt.Sscanf(params, "%d %d", &mappedSectors, &highestMappedSector); err == nil {
|
|
return
|
|
}
|
|
return
|
|
}
|
|
|
|
// GetDeviceStatus provides size, mapped sectors
|
|
func (devices *DeviceSet) GetDeviceStatus(hash string) (*DevStatus, error) {
|
|
info, err := devices.lookupDeviceWithLock(hash)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
info.lock.Lock()
|
|
defer info.lock.Unlock()
|
|
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
status := &DevStatus{
|
|
DeviceID: info.DeviceID,
|
|
Size: info.Size,
|
|
TransactionID: info.TransactionID,
|
|
}
|
|
|
|
if err := devices.activateDeviceIfNeeded(info, false); err != nil {
|
|
return nil, fmt.Errorf("devmapper: Error activating devmapper device for '%s': %s", hash, err)
|
|
}
|
|
|
|
sizeInSectors, mappedSectors, highestMappedSector, err := devices.deviceStatus(info.DevName())
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
status.SizeInSectors = sizeInSectors
|
|
status.MappedSectors = mappedSectors
|
|
status.HighestMappedSector = highestMappedSector
|
|
|
|
return status, nil
|
|
}
|
|
|
|
func (devices *DeviceSet) poolStatus() (totalSizeInSectors, transactionID, dataUsed, dataTotal, metadataUsed, metadataTotal uint64, err error) {
|
|
var params string
|
|
if _, totalSizeInSectors, _, params, err = devicemapper.GetStatus(devices.getPoolName()); err == nil {
|
|
_, err = fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionID, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal)
|
|
}
|
|
return
|
|
}
|
|
|
|
// DataDevicePath returns the path to the data storage for this deviceset,
|
|
// regardless of loopback or block device
|
|
func (devices *DeviceSet) DataDevicePath() string {
|
|
return devices.dataDevice
|
|
}
|
|
|
|
// MetadataDevicePath returns the path to the metadata storage for this deviceset,
|
|
// regardless of loopback or block device
|
|
func (devices *DeviceSet) MetadataDevicePath() string {
|
|
return devices.metadataDevice
|
|
}
|
|
|
|
func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) {
|
|
buf := new(unix.Statfs_t)
|
|
if err := unix.Statfs(loopFile, buf); err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("Couldn't stat loopfile filesystem %v: %v", loopFile, err)
|
|
return 0, err
|
|
}
|
|
return buf.Bfree * uint64(buf.Bsize), nil
|
|
}
|
|
|
|
func (devices *DeviceSet) isRealFile(loopFile string) (bool, error) {
|
|
if loopFile != "" {
|
|
fi, err := os.Stat(loopFile)
|
|
if err != nil {
|
|
logrus.WithField("storage-driver", "devicemapper").Warnf("Couldn't stat loopfile %v: %v", loopFile, err)
|
|
return false, err
|
|
}
|
|
return fi.Mode().IsRegular(), nil
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// Status returns the current status of this deviceset
|
|
func (devices *DeviceSet) Status() *Status {
|
|
devices.Lock()
|
|
defer devices.Unlock()
|
|
|
|
status := &Status{}
|
|
|
|
status.PoolName = devices.getPoolName()
|
|
status.DataFile = devices.DataDevicePath()
|
|
status.DataLoopback = devices.dataLoopFile
|
|
status.MetadataFile = devices.MetadataDevicePath()
|
|
status.MetadataLoopback = devices.metadataLoopFile
|
|
status.UdevSyncSupported = devicemapper.UdevSyncSupported()
|
|
status.DeferredRemoveEnabled = devices.deferredRemove
|
|
status.DeferredDeleteEnabled = devices.deferredDelete
|
|
status.DeferredDeletedDeviceCount = devices.nrDeletedDevices
|
|
status.BaseDeviceSize = devices.getBaseDeviceSize()
|
|
status.BaseDeviceFS = devices.getBaseDeviceFS()
|
|
|
|
totalSizeInSectors, _, dataUsed, dataTotal, metadataUsed, metadataTotal, err := devices.poolStatus()
|
|
if err == nil {
|
|
// Convert from blocks to bytes
|
|
blockSizeInSectors := totalSizeInSectors / dataTotal
|
|
|
|
status.Data.Used = dataUsed * blockSizeInSectors * 512
|
|
status.Data.Total = dataTotal * blockSizeInSectors * 512
|
|
status.Data.Available = status.Data.Total - status.Data.Used
|
|
|
|
// metadata blocks are always 4k
|
|
status.Metadata.Used = metadataUsed * 4096
|
|
status.Metadata.Total = metadataTotal * 4096
|
|
status.Metadata.Available = status.Metadata.Total - status.Metadata.Used
|
|
|
|
status.SectorSize = blockSizeInSectors * 512
|
|
|
|
if check, _ := devices.isRealFile(devices.dataLoopFile); check {
|
|
actualSpace, err := devices.getUnderlyingAvailableSpace(devices.dataLoopFile)
|
|
if err == nil && actualSpace < status.Data.Available {
|
|
status.Data.Available = actualSpace
|
|
}
|
|
}
|
|
|
|
if check, _ := devices.isRealFile(devices.metadataLoopFile); check {
|
|
actualSpace, err := devices.getUnderlyingAvailableSpace(devices.metadataLoopFile)
|
|
if err == nil && actualSpace < status.Metadata.Available {
|
|
status.Metadata.Available = actualSpace
|
|
}
|
|
}
|
|
|
|
minFreeData := (dataTotal * uint64(devices.minFreeSpacePercent)) / 100
|
|
status.MinFreeSpace = minFreeData * blockSizeInSectors * 512
|
|
}
|
|
|
|
return status
|
|
}
|
|
|
|
// Status returns the current status of this deviceset
|
|
func (devices *DeviceSet) exportDeviceMetadata(hash string) (*deviceMetadata, error) {
|
|
info, err := devices.lookupDeviceWithLock(hash)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
info.lock.Lock()
|
|
defer info.lock.Unlock()
|
|
|
|
metadata := &deviceMetadata{info.DeviceID, info.Size, info.Name()}
|
|
return metadata, nil
|
|
}
|
|
|
|
// NewDeviceSet creates the device set based on the options provided.
|
|
func NewDeviceSet(root string, doInit bool, options []string, idMap idtools.IdentityMapping) (*DeviceSet, error) {
|
|
devicemapper.SetDevDir("/dev")
|
|
|
|
devices := &DeviceSet{
|
|
root: root,
|
|
metaData: metaData{Devices: make(map[string]*devInfo)},
|
|
dataLoopbackSize: defaultDataLoopbackSize,
|
|
metaDataLoopbackSize: defaultMetaDataLoopbackSize,
|
|
baseFsSize: defaultBaseFsSize,
|
|
overrideUdevSyncCheck: defaultUdevSyncOverride,
|
|
doBlkDiscard: true,
|
|
thinpBlockSize: defaultThinpBlockSize,
|
|
deviceIDMap: make([]byte, deviceIDMapSz),
|
|
deletionWorkerTicker: time.NewTicker(time.Second * 30),
|
|
idMap: idMap,
|
|
minFreeSpacePercent: defaultMinFreeSpacePercent,
|
|
}
|
|
|
|
version, err := devicemapper.GetDriverVersion()
|
|
if err != nil {
|
|
// Can't even get driver version, assume not supported
|
|
return nil, graphdriver.ErrNotSupported
|
|
}
|
|
|
|
if err := determineDriverCapabilities(version); err != nil {
|
|
return nil, graphdriver.ErrNotSupported
|
|
}
|
|
|
|
if driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport {
|
|
// enable deferred stuff by default
|
|
enableDeferredDeletion = true
|
|
enableDeferredRemoval = true
|
|
}
|
|
|
|
foundBlkDiscard := false
|
|
var lvmSetupConfig directLVMConfig
|
|
for _, option := range options {
|
|
key, val, err := parsers.ParseKeyValueOpt(option)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
key = strings.ToLower(key)
|
|
switch key {
|
|
case "dm.basesize":
|
|
size, err := units.RAMInBytes(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
userBaseSize = true
|
|
devices.baseFsSize = uint64(size)
|
|
case "dm.loopdatasize":
|
|
size, err := units.RAMInBytes(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
devices.dataLoopbackSize = size
|
|
case "dm.loopmetadatasize":
|
|
size, err := units.RAMInBytes(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
devices.metaDataLoopbackSize = size
|
|
case "dm.fs":
|
|
if val != "ext4" && val != "xfs" {
|
|
return nil, fmt.Errorf("devmapper: Unsupported filesystem %s", val)
|
|
}
|
|
devices.filesystem = val
|
|
case "dm.mkfsarg":
|
|
devices.mkfsArgs = append(devices.mkfsArgs, val)
|
|
case "dm.mountopt":
|
|
devices.mountOptions = joinMountOptions(devices.mountOptions, val)
|
|
case "dm.metadatadev":
|
|
devices.metadataDevice = val
|
|
case "dm.datadev":
|
|
devices.dataDevice = val
|
|
case "dm.thinpooldev":
|
|
devices.thinPoolDevice = strings.TrimPrefix(val, "/dev/mapper/")
|
|
case "dm.blkdiscard":
|
|
foundBlkDiscard = true
|
|
devices.doBlkDiscard, err = strconv.ParseBool(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
case "dm.blocksize":
|
|
size, err := units.RAMInBytes(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// convert to 512b sectors
|
|
devices.thinpBlockSize = uint32(size) >> 9
|
|
case "dm.override_udev_sync_check":
|
|
devices.overrideUdevSyncCheck, err = strconv.ParseBool(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "dm.use_deferred_removal":
|
|
enableDeferredRemoval, err = strconv.ParseBool(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "dm.use_deferred_deletion":
|
|
enableDeferredDeletion, err = strconv.ParseBool(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
case "dm.min_free_space":
|
|
if !strings.HasSuffix(val, "%") {
|
|
return nil, fmt.Errorf("devmapper: Option dm.min_free_space requires %% suffix")
|
|
}
|
|
|
|
valstring := strings.TrimSuffix(val, "%")
|
|
minFreeSpacePercent, err := strconv.ParseUint(valstring, 10, 32)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if minFreeSpacePercent >= 100 {
|
|
return nil, fmt.Errorf("devmapper: Invalid value %v for option dm.min_free_space", val)
|
|
}
|
|
|
|
devices.minFreeSpacePercent = uint32(minFreeSpacePercent)
|
|
case "dm.xfs_nospace_max_retries":
|
|
_, err := strconv.ParseUint(val, 10, 64)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
devices.xfsNospaceRetries = val
|
|
case "dm.directlvm_device":
|
|
lvmSetupConfig.Device = val
|
|
case "dm.directlvm_device_force":
|
|
lvmSetupConfigForce, err = strconv.ParseBool(val)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
case "dm.thinp_percent":
|
|
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
|
|
}
|
|
if per >= 100 {
|
|
return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
|
|
}
|
|
lvmSetupConfig.ThinpPercent = per
|
|
case "dm.thinp_metapercent":
|
|
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
|
|
}
|
|
if per >= 100 {
|
|
return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
|
|
}
|
|
lvmSetupConfig.ThinpMetaPercent = per
|
|
case "dm.thinp_autoextend_percent":
|
|
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
|
|
}
|
|
if per > 100 {
|
|
return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
|
|
}
|
|
lvmSetupConfig.AutoExtendPercent = per
|
|
case "dm.thinp_autoextend_threshold":
|
|
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
|
|
}
|
|
if per > 100 {
|
|
return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
|
|
}
|
|
lvmSetupConfig.AutoExtendThreshold = per
|
|
case "dm.libdm_log_level":
|
|
level, err := strconv.ParseInt(val, 10, 32)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "could not parse `dm.libdm_log_level=%s`", val)
|
|
}
|
|
if level < devicemapper.LogLevelFatal || level > devicemapper.LogLevelDebug {
|
|
return nil, errors.Errorf("dm.libdm_log_level must be in range [%d,%d]", devicemapper.LogLevelFatal, devicemapper.LogLevelDebug)
|
|
}
|
|
// Register a new logging callback with the specified level.
|
|
devicemapper.LogInit(devicemapper.DefaultLogger{
|
|
Level: int(level),
|
|
})
|
|
default:
|
|
return nil, fmt.Errorf("devmapper: Unknown option %s", key)
|
|
}
|
|
}
|
|
|
|
if err := validateLVMConfig(lvmSetupConfig); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
devices.lvmSetupConfig = lvmSetupConfig
|
|
|
|
// By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
|
|
if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
|
|
devices.doBlkDiscard = false
|
|
}
|
|
|
|
if err := devices.initDevmapper(doInit); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return devices, nil
|
|
}
|