diff --git a/plugin/manager.go b/plugin/manager.go
index cc7efa1223..cd17e11c8b 100644
--- a/plugin/manager.go
+++ b/plugin/manager.go
@@ -32,14 +32,12 @@ type eventLogger func(id, name, action string)
 
 // Manager controls the plugin subsystem.
 type Manager struct {
-	sync.RWMutex
 	libRoot           string
 	runRoot           string
 	pluginStore       *store.Store
 	containerdClient  libcontainerd.Client
 	registryService   registry.Service
 	liveRestore       bool
-	shutdown          bool
 	pluginEventLogger eventLogger
 }
 
@@ -83,17 +81,20 @@ func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error {
 
 	switch e.State {
 	case libcontainerd.StateExit:
-		var shutdown bool
-		pm.RLock()
-		shutdown = pm.shutdown
-		pm.RUnlock()
-		if shutdown {
-			p, err := pm.pluginStore.GetByID(id)
-			if err != nil {
-				return err
-			}
+		p, err := pm.pluginStore.GetByID(id)
+		if err != nil {
+			return err
+		}
+		p.RLock()
+		if p.ExitChan != nil {
 			close(p.ExitChan)
 		}
+		restart := p.Restart
+		p.RUnlock()
+		p.RemoveFromDisk()
+		if restart {
+			pm.enable(p, true)
+		}
 	}
 
 	return nil
diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go
index 257aaf4bd7..14a577ddc5 100644
--- a/plugin/manager_linux.go
+++ b/plugin/manager_linux.go
@@ -9,12 +9,9 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/oci"
 	"github.com/docker/docker/pkg/plugins"
 	"github.com/docker/docker/plugin/v2"
-	"github.com/docker/docker/restartmanager"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -26,20 +23,18 @@ func (pm *Manager) enable(p *v2.Plugin, force bool) error {
 	if err != nil {
 		return err
 	}
-
-	p.RestartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0)
-	if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), libcontainerd.WithRestartManager(p.RestartManager)); err != nil {
-		if err := p.RestartManager.Cancel(); err != nil {
-			logrus.Errorf("enable: restartManager.Cancel failed due to %v", err)
-		}
+	p.Lock()
+	p.Restart = true
+	p.Unlock()
+	if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec)); err != nil {
 		return err
 	}
 
 	p.PClient, err = plugins.NewClient("unix://"+filepath.Join(p.RuntimeSourcePath, p.GetSocket()), nil)
 	if err != nil {
-		if err := p.RestartManager.Cancel(); err != nil {
-			logrus.Errorf("enable: restartManager.Cancel failed due to %v", err)
-		}
+		p.Lock()
+		p.Restart = false
+		p.Unlock()
 		return err
 	}
 
@@ -50,49 +45,37 @@ func (pm *Manager) enable(p *v2.Plugin, force bool) error {
 }
 
 func (pm *Manager) restore(p *v2.Plugin) error {
-	p.RestartManager = restartmanager.New(container.RestartPolicy{Name: "always"}, 0)
-	return pm.containerdClient.Restore(p.GetID(), libcontainerd.WithRestartManager(p.RestartManager))
+	return pm.containerdClient.Restore(p.GetID())
 }
 
 func (pm *Manager) disable(p *v2.Plugin) error {
 	if !p.IsEnabled() {
 		return fmt.Errorf("plugin %s is already disabled", p.Name())
 	}
-	if err := p.RestartManager.Cancel(); err != nil {
-		logrus.Error(err)
-	}
+	p.Lock()
+	p.Restart = false
+	p.Unlock()
 	if err := pm.containerdClient.Signal(p.GetID(), int(syscall.SIGKILL)); err != nil {
 		logrus.Error(err)
 	}
-	if err := p.RemoveFromDisk(); err != nil {
-		logrus.Error(err)
-	}
 	pm.pluginStore.SetState(p, false)
 	return nil
 }
 
 // Shutdown stops all plugins and called during daemon shutdown.
 func (pm *Manager) Shutdown() {
-	pm.Lock()
-	pm.shutdown = true
-	pm.Unlock()
-
-	pm.RLock()
-	defer pm.RUnlock()
 	plugins := pm.pluginStore.GetAll()
 	for _, p := range plugins {
 		if pm.liveRestore && p.IsEnabled() {
 			logrus.Debug("Plugin active when liveRestore is set, skipping shutdown")
 			continue
 		}
-		if p.RestartManager != nil {
-			if err := p.RestartManager.Cancel(); err != nil {
-				logrus.Error(err)
-			}
-		}
 		if pm.containerdClient != nil && p.IsEnabled() {
 			pluginID := p.GetID()
+			p.Lock()
 			p.ExitChan = make(chan bool)
+			p.Restart = false
+			p.Unlock()
 			err := pm.containerdClient.Signal(p.PluginObj.ID, int(syscall.SIGTERM))
 			if err != nil {
 				logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err)
@@ -108,8 +91,5 @@ func (pm *Manager) Shutdown() {
 				}
 			}
 		}
-		if err := p.RemoveFromDisk(); err != nil {
-			logrus.Errorf("Remove plugin runtime failed with error: %v", err)
-		}
 	}
 }
diff --git a/plugin/store/store_experimental.go b/plugin/store/store_experimental.go
index 6e9e0b7d91..3b33a45aa9 100644
--- a/plugin/store/store_experimental.go
+++ b/plugin/store/store_experimental.go
@@ -111,13 +111,12 @@ func (ps *Store) Add(p *v2.Plugin) {
 	ps.Unlock()
 }
 
-// Remove removes a plugin from memory, plugindb and disk.
+// Remove removes a plugin from memory and plugindb.
 func (ps *Store) Remove(p *v2.Plugin) {
 	ps.Lock()
 	delete(ps.plugins, p.GetID())
 	delete(ps.nameToID, p.Name())
 	ps.updatePluginDB()
-	p.RemoveFromDisk()
 	ps.Unlock()
 }
 
diff --git a/plugin/v2/plugin.go b/plugin/v2/plugin.go
index 9cd37d3f06..8cc06bd824 100644
--- a/plugin/v2/plugin.go
+++ b/plugin/v2/plugin.go
@@ -5,16 +5,15 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/pkg/plugins"
-	"github.com/docker/docker/restartmanager"
 )
 
 // Plugin represents an individual plugin.
 type Plugin struct {
 	sync.RWMutex
-	PluginObj         types.Plugin                  `json:"plugin"`
-	PClient           *plugins.Client               `json:"-"`
-	RestartManager    restartmanager.RestartManager `json:"-"`
-	RuntimeSourcePath string                        `json:"-"`
-	ExitChan          chan bool                     `json:"-"`
-	RefCount          int                           `json:"-"`
+	PluginObj         types.Plugin    `json:"plugin"`
+	PClient           *plugins.Client `json:"-"`
+	RuntimeSourcePath string          `json:"-"`
+	RefCount          int             `json:"-"`
+	Restart           bool            `json:"-"`
+	ExitChan          chan bool       `json:"-"`
 }