diff --git a/libcontainerd/remote/client.go b/libcontainerd/remote/client.go
index 4af29300ab..f159a54c16 100644
--- a/libcontainerd/remote/client.go
+++ b/libcontainerd/remote/client.go
@@ -233,7 +233,9 @@ func (c *container) Start(ctx context.Context, checkpointDir string, withStdin b
 	stdinCloseSync <- t
 
 	if err := t.Start(ctx); err != nil {
-		if _, err := t.Delete(ctx); err != nil {
+		// Only Stopped tasks can be deleted. Created tasks have to be
+		// killed first, to transition them to Stopped.
+		if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil {
 			c.client.logger.WithError(err).WithField("container", c.c8dCtr.ID()).
 				Error("failed to delete task after fail start")
 		}
diff --git a/plugin/executor/containerd/containerd.go b/plugin/executor/containerd/containerd.go
index 0327e65dc4..474e7774f6 100644
--- a/plugin/executor/containerd/containerd.go
+++ b/plugin/executor/containerd/containerd.go
@@ -60,7 +60,7 @@ type c8dPlugin struct {
 // deleteTaskAndContainer deletes plugin task and then plugin container from containerd
 func (p c8dPlugin) deleteTaskAndContainer(ctx context.Context) {
 	if p.tsk != nil {
-		if _, err := p.tsk.Delete(ctx); err != nil && !errdefs.IsNotFound(err) {
+		if err := p.tsk.ForceDelete(ctx); err != nil && !errdefs.IsNotFound(err) {
 			p.log.WithError(err).Error("failed to delete plugin task from containerd")
 		}
 	}