Add option to auto-configure blkdev for devmapper
Instead of forcing users to manually configure a block device to use with devmapper, this gives the user the option to let the devmapper driver configure a device for them. Adds several new options to the devmapper storage-opts: - dm.directlvm_device="" - path to the block device to configure for direct-lvm - dm.thinp_percent=95 - sets the percentage of space to use for storage from the passed in block device - dm.thinp_metapercent=1 - sets the percentage of space to for metadata storage from the passed in block device - dm.thinp_autoextend_threshold=80 - sets the threshold for when `lvm` should automatically extend the thin pool as a percentage of the total storage space - dm.thinp_autoextend_percent=20 - sets the percentage to increase the thin pool by when an autoextend is triggered. Defaults are taken from [here](https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#/configure-direct-lvm-mode-for-production) The only option that is required is `dm.directlvm_device` for docker to set everything up. Changes to these settings are not currently supported and will error out. Future work could support allowing changes to these values. Signed-off-by: Brian Goff <cpuguy83@gmail.com>
This commit is contained in:
Brian Goff
parent
dab6c7d4f2
commit
5ef07d79c4
|
|
@ -5,7 +5,9 @@
|
|||
The device mapper graphdriver uses the device mapper thin provisioning
|
||||
module (dm-thinp) to implement CoW snapshots. The preferred model is
|
||||
to have a thin pool reserved outside of Docker and passed to the
|
||||
daemon via the `--storage-opt dm.thinpooldev` option.
|
||||
daemon via the `--storage-opt dm.thinpooldev` option. Alternatively,
|
||||
the device mapper graphdriver can setup a block device to handle this
|
||||
for you via the `--storage-opt dm.directlvm_device` option.
|
||||
|
||||
As a fallback if no thin pool is provided, loopback files will be
|
||||
created. Loopback is very slow, but can be used without any
|
||||
|
|
|
|||
|
|
@ -0,0 +1,247 @@
|
|||
package devmapper
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type directLVMConfig struct {
|
||||
Device string
|
||||
ThinpPercent uint64
|
||||
ThinpMetaPercent uint64
|
||||
AutoExtendPercent uint64
|
||||
AutoExtendThreshold uint64
|
||||
}
|
||||
|
||||
var (
|
||||
errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
|
||||
errThinpPercentTooBig = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
|
||||
errMissingSetupDevice = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm")
|
||||
)
|
||||
|
||||
func validateLVMConfig(cfg directLVMConfig) error {
|
||||
if reflect.DeepEqual(cfg, directLVMConfig{}) {
|
||||
return nil
|
||||
}
|
||||
if cfg.Device == "" {
|
||||
return errMissingSetupDevice
|
||||
}
|
||||
if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
|
||||
return errThinpPercentMissing
|
||||
}
|
||||
|
||||
if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
|
||||
return errThinpPercentTooBig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkDevAvailable(dev string) error {
|
||||
lvmScan, err := exec.LookPath("lvmdiskscan")
|
||||
if err != nil {
|
||||
logrus.Debug("could not find lvmdiskscan")
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := exec.Command(lvmScan).CombinedOutput()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
if !bytes.Contains(out, []byte(dev)) {
|
||||
return errors.Errorf("%s is not available for use with devicemapper", dev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkDevInVG(dev string) error {
|
||||
pvDisplay, err := exec.LookPath("pvdisplay")
|
||||
if err != nil {
|
||||
logrus.Debug("could not find pvdisplay")
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := exec.Command(pvDisplay, dev).CombinedOutput()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
|
||||
for scanner.Scan() {
|
||||
fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
|
||||
if len(fields) > 1 {
|
||||
// got "VG Name" line"
|
||||
vg := strings.TrimSpace(fields[1])
|
||||
if len(vg) > 0 {
|
||||
return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
|
||||
}
|
||||
logrus.Error(fields)
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkDevHasFS(dev string) error {
|
||||
blkid, err := exec.LookPath("blkid")
|
||||
if err != nil {
|
||||
logrus.Debug("could not find blkid")
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := exec.Command(blkid, dev).CombinedOutput()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
fields := bytes.Fields(out)
|
||||
for _, f := range fields {
|
||||
kv := bytes.Split(f, []byte{'='})
|
||||
if bytes.Equal(kv[0], []byte("TYPE")) {
|
||||
v := bytes.Trim(kv[1], "\"")
|
||||
if len(v) > 0 {
|
||||
return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyBlockDevice(dev string, force bool) error {
|
||||
if err := checkDevAvailable(dev); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkDevInVG(dev); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if force {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := checkDevHasFS(dev); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readLVMConfig(root string) (directLVMConfig, error) {
|
||||
var cfg directLVMConfig
|
||||
|
||||
p := filepath.Join(root, "setup-config.json")
|
||||
b, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return cfg, nil
|
||||
}
|
||||
return cfg, errors.Wrap(err, "error reading existing setup config")
|
||||
}
|
||||
|
||||
// check if this is just an empty file, no need to produce a json error later if so
|
||||
if len(b) == 0 {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
err = json.Unmarshal(b, &cfg)
|
||||
return cfg, errors.Wrap(err, "error unmarshaling previous device setup config")
|
||||
}
|
||||
|
||||
func writeLVMConfig(root string, cfg directLVMConfig) error {
|
||||
p := filepath.Join(root, "setup-config.json")
|
||||
b, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error marshalling direct lvm config")
|
||||
}
|
||||
err = ioutil.WriteFile(p, b, 0600)
|
||||
return errors.Wrap(err, "error writing direct lvm config to file")
|
||||
}
|
||||
|
||||
func setupDirectLVM(cfg directLVMConfig) error {
|
||||
pvCreate, err := exec.LookPath("pvcreate")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error lookuping up command `pvcreate` while setting up direct lvm")
|
||||
}
|
||||
|
||||
vgCreate, err := exec.LookPath("vgcreate")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error lookuping up command `vgcreate` while setting up direct lvm")
|
||||
}
|
||||
|
||||
lvCreate, err := exec.LookPath("lvcreate")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error lookuping up command `lvcreate` while setting up direct lvm")
|
||||
}
|
||||
|
||||
lvConvert, err := exec.LookPath("lvconvert")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error lookuping up command `lvconvert` while setting up direct lvm")
|
||||
}
|
||||
|
||||
lvChange, err := exec.LookPath("lvchange")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error lookuping up command `lvchange` while setting up direct lvm")
|
||||
}
|
||||
|
||||
if cfg.AutoExtendPercent == 0 {
|
||||
cfg.AutoExtendPercent = 20
|
||||
}
|
||||
|
||||
if cfg.AutoExtendThreshold == 0 {
|
||||
cfg.AutoExtendThreshold = 80
|
||||
}
|
||||
|
||||
if cfg.ThinpPercent == 0 {
|
||||
cfg.ThinpPercent = 95
|
||||
}
|
||||
if cfg.ThinpMetaPercent == 0 {
|
||||
cfg.ThinpMetaPercent = 1
|
||||
}
|
||||
|
||||
out, err := exec.Command(pvCreate, "-f", cfg.Device).CombinedOutput()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, string(out))
|
||||
}
|
||||
|
||||
out, err = exec.Command(vgCreate, "docker", cfg.Device).CombinedOutput()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, string(out))
|
||||
}
|
||||
|
||||
out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpool", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, string(out))
|
||||
}
|
||||
out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpoolmeta", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, string(out))
|
||||
}
|
||||
|
||||
out, err = exec.Command(lvConvert, "-y", "--zero", "n", "-c", "512K", "--thinpool", "docker/thinpool", "--poolmetadata", "docker/thinpoolmeta").CombinedOutput()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, string(out))
|
||||
}
|
||||
|
||||
profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
|
||||
err = ioutil.WriteFile("/etc/lvm/profile/docker-thinpool.profile", []byte(profile), 0600)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error writing docker thinp autoextend profile")
|
||||
}
|
||||
|
||||
out, err = exec.Command(lvChange, "--metadataprofile", "docker-thinpool", "docker/thinpool").CombinedOutput()
|
||||
return errors.Wrap(err, string(out))
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ package devmapper
|
|||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
|
@ -13,6 +12,7 @@ import (
|
|||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"github.com/docker/docker/pkg/mount"
|
||||
"github.com/docker/docker/pkg/parsers"
|
||||
units "github.com/docker/go-units"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/opencontainers/runc/libcontainer/label"
|
||||
)
|
||||
|
|
@ -50,6 +51,7 @@ var (
|
|||
enableDeferredDeletion = false
|
||||
userBaseSize = false
|
||||
defaultMinFreeSpacePercent uint32 = 10
|
||||
lvmSetupConfigForce bool
|
||||
)
|
||||
|
||||
const deviceSetMetaFile string = "deviceset-metadata"
|
||||
|
|
@ -123,6 +125,7 @@ type DeviceSet struct {
|
|||
gidMaps []idtools.IDMap
|
||||
minFreeSpacePercent uint32 //min free space percentage in thinpool
|
||||
xfsNospaceRetries string // max retries when xfs receives ENOSPC
|
||||
lvmSetupConfig directLVMConfig
|
||||
}
|
||||
|
||||
// DiskUsage contains information about disk usage and is used when reporting Status of a device.
|
||||
|
|
@ -1730,8 +1733,36 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Set the device prefix from the device id and inode of the docker root dir
|
||||
prevSetupConfig, err := readLVMConfig(devices.root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
|
||||
if devices.thinPoolDevice != "" {
|
||||
return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
|
||||
if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
|
||||
return errors.New("changing direct-lvm config is not supported")
|
||||
}
|
||||
logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
|
||||
if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
devices.thinPoolDevice = "docker-thinpool"
|
||||
logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
|
||||
}
|
||||
|
||||
// Set the device prefix from the device id and inode of the docker root dir
|
||||
st, err := os.Stat(devices.root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
|
||||
|
|
@ -2605,6 +2636,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
|
|||
}
|
||||
|
||||
foundBlkDiscard := false
|
||||
var lvmSetupConfig directLVMConfig
|
||||
for _, option := range options {
|
||||
key, val, err := parsers.ParseKeyValueOpt(option)
|
||||
if err != nil {
|
||||
|
|
@ -2699,11 +2731,60 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
|
|||
return nil, err
|
||||
}
|
||||
devices.xfsNospaceRetries = val
|
||||
case "dm.directlvm_device":
|
||||
lvmSetupConfig.Device = val
|
||||
case "dm.directlvm_device_force":
|
||||
lvmSetupConfigForce, err = strconv.ParseBool(val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "dm.thinp_percent":
|
||||
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
|
||||
}
|
||||
if per >= 100 {
|
||||
return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
|
||||
}
|
||||
lvmSetupConfig.ThinpPercent = per
|
||||
case "dm.thinp_metapercent":
|
||||
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
|
||||
}
|
||||
if per >= 100 {
|
||||
return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
|
||||
}
|
||||
lvmSetupConfig.ThinpMetaPercent = per
|
||||
case "dm.thinp_autoextend_percent":
|
||||
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
|
||||
}
|
||||
if per > 100 {
|
||||
return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
|
||||
}
|
||||
lvmSetupConfig.AutoExtendPercent = per
|
||||
case "dm.thinp_autoextend_threshold":
|
||||
per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
|
||||
}
|
||||
if per > 100 {
|
||||
return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
|
||||
}
|
||||
lvmSetupConfig.AutoExtendThreshold = per
|
||||
default:
|
||||
return nil, fmt.Errorf("devmapper: Unknown option %s\n", key)
|
||||
}
|
||||
}
|
||||
|
||||
if err := validateLVMConfig(lvmSetupConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
devices.lvmSetupConfig = lvmSetupConfig
|
||||
|
||||
// By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
|
||||
if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
|
||||
devices.doBlkDiscard = false
|
||||
|
|
|
|||
|
|
@ -322,6 +322,60 @@ not use loopback in production. Ensure your Engine daemon has a
|
|||
$ sudo dockerd --storage-opt dm.thinpooldev=/dev/mapper/thin-pool
|
||||
```
|
||||
|
||||
##### `dm.directlvm_device`
|
||||
|
||||
As an alternative to providing a thin pool as above, Docker can setup a block
|
||||
device for you.
|
||||
|
||||
###### Example:
|
||||
|
||||
```bash
|
||||
$ sudo dockerd --storage-opt dm.directlvm_device=/dev/xvdf
|
||||
```
|
||||
|
||||
##### `dm.thinp_percent`
|
||||
|
||||
Sets the percentage of passed in block device to use for storage.
|
||||
|
||||
###### Example:
|
||||
|
||||
```bash
|
||||
$ sudo dockerd --storage-opt dm.thinp_percent=95
|
||||
```
|
||||
|
||||
##### `dm.thinp_metapercent`
|
||||
|
||||
Sets the percentage of the passed in block device to use for metadata storage.
|
||||
|
||||
###### Example:
|
||||
|
||||
```bash
|
||||
$ sudo dockerd --storage-opt dm.thinp_metapercent=1
|
||||
```
|
||||
|
||||
##### `dm.thinp_autoextend_threshold`
|
||||
|
||||
Sets the value of the percentage of space used before `lvm` attempts to
|
||||
autoextend the available space [100 = disabled]
|
||||
|
||||
###### Example:
|
||||
|
||||
```bash
|
||||
$ sudo dockerd --storage-opt dm.thinp_autoextend_threshold=80
|
||||
```
|
||||
|
||||
##### `dm.thinp_autoextend_percent`
|
||||
|
||||
Sets the value percentage value to increase the thin pool by when when `lvm`
|
||||
attempts to autoextend the available space [100 = disabled]
|
||||
|
||||
###### Example:
|
||||
|
||||
```bash
|
||||
$ sudo dockerd --storage-opt dm.thinp_autoextend_percent=20
|
||||
```
|
||||
|
||||
|
||||
##### `dm.basesize`
|
||||
|
||||
Specifies the size to use when creating the base device, which limits the
|
||||
|
|
|
|||
|
|
@ -418,6 +418,54 @@ Example use:
|
|||
$ dockerd \
|
||||
--storage-opt dm.thinpooldev=/dev/mapper/thin-pool
|
||||
|
||||
#### dm.directlvm_device
|
||||
|
||||
As an alternative to manually creating a thin pool as above, Docker can
|
||||
automatically configure a block device for you.
|
||||
|
||||
Example use:
|
||||
|
||||
$ dockerd \
|
||||
--storage-opt dm.directlvm_device=/dev/xvdf
|
||||
|
||||
##### dm.thinp_percent
|
||||
|
||||
Sets the percentage of passed in block device to use for storage.
|
||||
|
||||
###### Example:
|
||||
|
||||
$ sudo dockerd \
|
||||
--storage-opt dm.thinp_percent=95
|
||||
|
||||
##### `dm.thinp_metapercent`
|
||||
|
||||
Sets the percentage of the passed in block device to use for metadata storage.
|
||||
|
||||
###### Example:
|
||||
|
||||
$ sudo dockerd \
|
||||
--storage-opt dm.thinp_metapercent=1
|
||||
|
||||
##### dm.thinp_autoextend_threshold
|
||||
|
||||
Sets the value of the percentage of space used before `lvm` attempts to
|
||||
autoextend the available space [100 = disabled]
|
||||
|
||||
###### Example:
|
||||
|
||||
$ sudo dockerd \
|
||||
--storage-opt dm.thinp_autoextend_threshold=80
|
||||
|
||||
##### dm.thinp_autoextend_percent
|
||||
|
||||
Sets the value percentage value to increase the thin pool by when when `lvm`
|
||||
attempts to autoextend the available space [100 = disabled]
|
||||
|
||||
###### Example:
|
||||
|
||||
$ sudo dockerd \
|
||||
--storage-opt dm.thinp_autoextend_percent=20
|
||||
|
||||
#### dm.basesize
|
||||
|
||||
Specifies the size to use when creating the base device, which limits
|
||||
|
|
|
|||
Loading…
Reference in New Issue