mirror of
https://github.com/moby/moby.git
synced 2022-11-09 12:21:53 -05:00
57f06409b1
For some reason, retrying to unmount in case of getting EBUSY error
was only performed in Remove(), but not Put().
I have done some testing on Ubuntu 16.04 and 18.04 with aufs,
performing massively parallel container creation using this script:
```
NUMCTS=5000
PARALLEL=100
IMAGE=busybox
docker pull $IMAGE >/dev/null
seq $NUMCTS | parallel -j$PARALLEL docker create $IMAGE true > /dev/null
docker ps -qa | shuf | tail -n $NUMCTS | parallel -j$PARALLEL docker rm -f '{}' > /dev/null
```
Sometimes (1 to 5 times per 10000 `docker create`), aufs.Put() fails on Unmount syscall
with EBUSY during container creation:
> Error response from daemon: device or resource busy
and in docker log, with debug turned on:
> level=debug msg="Failed to unmount ID-init aufs: device or resource busy"
> level=error msg="Handler for POST /v1.30/containers/create returned error: device or resource busy"
I did some debugging by running fuser -v -M -m $MOUNT_POINT but
that reveals nothing.
This commit:
* implements retry on EBUSY in Unmount()
* calls Unmount() from Remove()
* increases the number of retries from 3 to 5
Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>
59 lines
1.4 KiB
Go
59 lines
1.4 KiB
Go
// +build linux
|
|
|
|
package aufs // import "github.com/docker/docker/daemon/graphdriver/aufs"
|
|
|
|
import (
|
|
"os/exec"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/docker/docker/pkg/mount"
|
|
"github.com/pkg/errors"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// Unmount the target specified.
|
|
func Unmount(target string) error {
|
|
const retries = 5
|
|
|
|
// auplink flush
|
|
for i := 0; ; i++ {
|
|
out, err := exec.Command("auplink", target, "flush").CombinedOutput()
|
|
if err == nil {
|
|
break
|
|
}
|
|
rc := 0
|
|
if exiterr, ok := err.(*exec.ExitError); ok {
|
|
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
|
rc = status.ExitStatus()
|
|
}
|
|
}
|
|
if i >= retries || rc != int(unix.EINVAL) {
|
|
logger.WithError(err).WithField("method", "Unmount").Warnf("auplink flush failed: %s", out)
|
|
break
|
|
}
|
|
// auplink failed to find target in /proc/self/mounts because
|
|
// kernel can't guarantee continuity while reading from it
|
|
// while mounts table is being changed
|
|
logger.Debugf("auplink flush error (retrying %d/%d): %s", i+1, retries, out)
|
|
}
|
|
|
|
// unmount
|
|
var err error
|
|
for i := 0; i < retries; i++ {
|
|
err = mount.Unmount(target)
|
|
switch errors.Cause(err) {
|
|
case nil:
|
|
return nil
|
|
case unix.EBUSY:
|
|
logger.Debugf("aufs unmount %s failed with EBUSY (retrying %d/%d)", target, i+1, retries)
|
|
time.Sleep(100 * time.Millisecond)
|
|
continue // try again
|
|
default:
|
|
// any other error is fatal
|
|
break
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|