diff --git a/daemon/graphdriver/driver.go b/daemon/graphdriver/driver.go
index 7a3a0d1892..d08c6dc5b7 100644
--- a/daemon/graphdriver/driver.go
+++ b/daemon/graphdriver/driver.go
@@ -283,8 +283,27 @@ func scanPriorDrivers(root string) map[string]bool {
 	for driver := range drivers {
 		p := filepath.Join(root, driver)
 		if _, err := os.Stat(p); err == nil && driver != "vfs" {
-			driversMap[driver] = true
+			if !isEmptyDir(p) {
+				driversMap[driver] = true
+			}
 		}
 	}
 	return driversMap
 }
+
+// isEmptyDir checks if a directory is empty. It is used to check if prior
+// storage-driver directories exist. If an error occurs, it also assumes the
+// directory is not empty (which preserves the behavior _before_ this check
+// was added)
+func isEmptyDir(name string) bool {
+	f, err := os.Open(name)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	if _, err = f.Readdirnames(1); err == io.EOF {
+		return true
+	}
+	return false
+}
diff --git a/daemon/graphdriver/driver_test.go b/daemon/graphdriver/driver_test.go
new file mode 100644
index 0000000000..40084be88d
--- /dev/null
+++ b/daemon/graphdriver/driver_test.go
@@ -0,0 +1,37 @@
+package graphdriver
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsEmptyDir(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "test-is-empty-dir")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmp)
+
+	d := filepath.Join(tmp, "empty-dir")
+	err = os.Mkdir(d, 0755)
+	require.NoError(t, err)
+	empty := isEmptyDir(d)
+	assert.True(t, empty)
+
+	d = filepath.Join(tmp, "dir-with-subdir")
+	err = os.MkdirAll(filepath.Join(d, "subdir"), 0755)
+	require.NoError(t, err)
+	empty = isEmptyDir(d)
+	assert.False(t, empty)
+
+	d = filepath.Join(tmp, "dir-with-empty-file")
+	err = os.Mkdir(d, 0755)
+	require.NoError(t, err)
+	_, err = ioutil.TempFile(d, "file")
+	require.NoError(t, err)
+	empty = isEmptyDir(d)
+	assert.False(t, empty)
+}