diff --git a/client/plugin_install.go b/client/plugin_install.go
index d0a3d517fc..407f1cddf2 100644
--- a/client/plugin_install.go
+++ b/client/plugin_install.go
@@ -10,7 +10,7 @@ import (
 )
 
 // PluginInstall installs a plugin
-func (cli *Client) PluginInstall(ctx context.Context, name string, options types.PluginInstallOptions) error {
+func (cli *Client) PluginInstall(ctx context.Context, name string, options types.PluginInstallOptions) (err error) {
 	// FIXME(vdemeester) name is a ref, we might want to parse/validate it here.
 	query := url.Values{}
 	query.Set("name", name)
@@ -27,6 +27,14 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types
 		ensureReaderClosed(resp)
 		return err
 	}
+
+	defer func() {
+		if err != nil {
+			delResp, _ := cli.delete(ctx, "/plugins/"+name, nil, nil)
+			ensureReaderClosed(delResp)
+		}
+	}()
+
 	var privileges types.PluginPrivileges
 	if err := json.NewDecoder(resp.body).Decode(&privileges); err != nil {
 		ensureReaderClosed(resp)
@@ -40,8 +48,6 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types
 			return err
 		}
 		if !accept {
-			resp, _ := cli.delete(ctx, "/plugins/"+name, nil, nil)
-			ensureReaderClosed(resp)
 			return pluginPermissionDenied{name}
 		}
 	}
diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go
index 7a8b05035d..26f0c39f2e 100644
--- a/plugin/manager_linux.go
+++ b/plugin/manager_linux.go
@@ -36,6 +36,7 @@ func (pm *Manager) enable(p *v2.Plugin, force bool) error {
 		p.Lock()
 		p.Restart = false
 		p.Unlock()
+		shutdownPlugin(p, pm.containerdClient)
 		return err
 	}