diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go
index 739f7b83cd..1bad3713f4 100644
--- a/plugin/backend_linux.go
+++ b/plugin/backend_linux.go
@@ -5,7 +5,6 @@ package plugin
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -23,6 +22,7 @@ import (
 	"github.com/docker/docker/plugin/distribution"
 	"github.com/docker/docker/plugin/v2"
 	"github.com/docker/docker/reference"
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -268,7 +268,7 @@ func (pm *Manager) Push(name string, metaHeader http.Header, authConfig *types.A
 }
 
 // Remove deletes plugin's root directory.
-func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error {
+func (pm *Manager) Remove(name string, config *types.PluginRmConfig) (err error) {
 	p, err := pm.pluginStore.GetByName(name)
 	pm.mu.RLock()
 	c := pm.cMap[p]
@@ -294,12 +294,18 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error {
 	}
 
 	id := p.GetID()
-	pm.pluginStore.Remove(p)
 	pluginDir := filepath.Join(pm.libRoot, id)
-	if err := os.RemoveAll(pluginDir); err != nil {
-		logrus.Warnf("unable to remove %q from plugin remove: %v", pluginDir, err)
+
+	defer func() {
+		if err == nil || config.ForceRemove {
+			pm.pluginStore.Remove(p)
+			pm.pluginEventLogger(id, name, "remove")
+		}
+	}()
+
+	if err = os.RemoveAll(pluginDir); err != nil {
+		return errors.Wrap(err, "failed to remove plugin directory")
 	}
-	pm.pluginEventLogger(id, name, "remove")
 	return nil
 }