Merge pull request #31683 from AkihiroSuda/it-on-swarm-easy-analysis

integration-cli-on-swarm: new CLI flag: -keep-executor

hack/integration-cli-on-swarm/README.md

@@ -64,3 +64,4 @@ Flags for debugging IT on Swarm itself:
  - `-rand-seed N`: the random seed. This flag is useful for deterministic replaying. By default(0), the timestamp is used.
  - `-filters-file FILE`: the file contains `-check.f` strings. By default, the file is automatically generated.
  - `-dry-run`: skip the actual workload
+ - `keep-executor`: do not auto-remove executor containers, which is used for running privileged programs on Swarm

hack/integration-cli-on-swarm/agent/worker/executor.go

@@ -19,69 +19,76 @@ import (
 // image needs to be the worker image itself. testFlags are OR-set of regexp for filtering tests.
 type testChunkExecutor func(image string, tests []string) (int64, string, error)
 
-func dryTestChunkExecutor(image string, tests []string) (int64, string, error) {
-	return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil
+func dryTestChunkExecutor() testChunkExecutor {
+	return func(image string, tests []string) (int64, string, error) {
+		return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil
+	}
 }
 
 // privilegedTestChunkExecutor invokes a privileged container from the worker
 // service via bind-mounted API socket so as to execute the test chunk
-func privilegedTestChunkExecutor(image string, tests []string) (int64, string, error) {
-	cli, err := client.NewEnvClient()
-	if err != nil {
-		return 0, "", err
-	}
-	// propagate variables from the host (needs to be defined in the compose file)
-	experimental := os.Getenv("DOCKER_EXPERIMENTAL")
-	graphdriver := os.Getenv("DOCKER_GRAPHDRIVER")
-	if graphdriver == "" {
-		info, err := cli.Info(context.Background())
+func privilegedTestChunkExecutor(autoRemove bool) testChunkExecutor {
+	return func(image string, tests []string) (int64, string, error) {
+		cli, err := client.NewEnvClient()
 		if err != nil {
 			return 0, "", err
 		}
-		graphdriver = info.Driver
-	}
-	// `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration-cli`)
-	// but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work.
-	//
-	// Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs`
-	//
-	// see integration-cli/daemon/daemon.go
-	daemonDest := "/daemon_dest"
-	config := container.Config{
-		Image: image,
-		Env: []string{
-			"TESTFLAGS=-check.f " + strings.Join(tests, "|"),
-			"KEEPBUNDLE=1",
-			"DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli
-			"DOCKER_EXPERIMENTAL=" + experimental,
-			"DOCKER_GRAPHDRIVER=" + graphdriver,
-			"DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest,
-		},
-		// TODO: set label?
-		Entrypoint: []string{"hack/dind"},
-		Cmd:        []string{"hack/make.sh", "test-integration-cli"},
-	}
-	hostConfig := container.HostConfig{
-		AutoRemove: true,
-		Privileged: true,
-		Mounts: []mount.Mount{
-			{
-				Type:   mount.TypeVolume,
-				Target: daemonDest,
+		// propagate variables from the host (needs to be defined in the compose file)
+		experimental := os.Getenv("DOCKER_EXPERIMENTAL")
+		graphdriver := os.Getenv("DOCKER_GRAPHDRIVER")
+		if graphdriver == "" {
+			info, err := cli.Info(context.Background())
+			if err != nil {
+				return 0, "", err
+			}
+			graphdriver = info.Driver
+		}
+		// `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration-cli`)
+		// but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work.
+		//
+		// Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs`
+		//
+		// see integration-cli/daemon/daemon.go
+		daemonDest := "/daemon_dest"
+		config := container.Config{
+			Image: image,
+			Env: []string{
+				"TESTFLAGS=-check.f " + strings.Join(tests, "|"),
+				"KEEPBUNDLE=1",
+				"DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli
+				"DOCKER_EXPERIMENTAL=" + experimental,
+				"DOCKER_GRAPHDRIVER=" + graphdriver,
+				"DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest,
 			},
-		},
+			Labels: map[string]string{
+				"org.dockerproject.integration-cli-on-swarm":         "",
+				"org.dockerproject.integration-cli-on-swarm.comment": "this non-service container is created for running privileged programs on Swarm. you can remove this container manually if the corresponding service is already stopped.",
+			},
+			Entrypoint: []string{"hack/dind"},
+			Cmd:        []string{"hack/make.sh", "test-integration-cli"},
+		}
+		hostConfig := container.HostConfig{
+			AutoRemove: autoRemove,
+			Privileged: true,
+			Mounts: []mount.Mount{
+				{
+					Type:   mount.TypeVolume,
+					Target: daemonDest,
+				},
+			},
+		}
+		id, stream, err := runContainer(context.Background(), cli, config, hostConfig)
+		if err != nil {
+			return 0, "", err
+		}
+		var b bytes.Buffer
+		teeContainerStream(&b, os.Stdout, os.Stderr, stream)
+		rc, err := cli.ContainerWait(context.Background(), id)
+		if err != nil {
+			return 0, "", err
+		}
+		return rc, b.String(), nil
 	}
-	id, stream, err := runContainer(context.Background(), cli, config, hostConfig)
-	if err != nil {
-		return 0, "", err
-	}
-	var b bytes.Buffer
-	teeContainerStream(&b, os.Stdout, os.Stderr, stream)
-	rc, err := cli.ContainerWait(context.Background(), id)
-	if err != nil {
-		return 0, "", err
-	}
-	return rc, b.String(), nil
 }
 
 func runContainer(ctx context.Context, cli *client.Client, config container.Config, hostConfig container.HostConfig) (string, io.ReadCloser, error) {

hack/integration-cli-on-swarm/agent/worker/worker.go

@@ -24,6 +24,7 @@ func validImageDigest(s string) bool {
 func xmain() error {
 	workerImageDigest := flag.String("worker-image-digest", "", "Needs to be the digest of this worker image itself")
 	dryRun := flag.Bool("dry-run", false, "Dry run")
+	keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm")
 	flag.Parse()
 	if !validImageDigest(*workerImageDigest) {
 		// Because of issue #29582.
@@ -31,9 +32,9 @@ func xmain() error {
 		// So, `docker run localregistry.example.com/blahblah:latest` fails: `Unable to find image 'localregistry.example.com/blahblah:latest' locally`
 		return fmt.Errorf("worker-image-digest must be a digest, got %q", *workerImageDigest)
 	}
-	executor := privilegedTestChunkExecutor
+	executor := privilegedTestChunkExecutor(!*keepExecutor)
 	if *dryRun {
-		executor = dryTestChunkExecutor
+		executor = dryTestChunkExecutor()
 	}
 	return handle(*workerImageDigest, executor)
 }

hack/integration-cli-on-swarm/host/compose.go

@@ -16,7 +16,7 @@ version: "3"
 services:
   worker:
     image: "{{.WorkerImage}}"
-    command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}"]
+    command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}", "-keep-executor={{.KeepExecutor}}"]
     networks:
       - net
     volumes:
@@ -57,14 +57,15 @@ volumes:
 `
 
 type composeOptions struct {
-	Replicas    int
-	Chunks      int
-	MasterImage string
-	WorkerImage string
-	Volume      string
-	Shuffle     bool
-	RandSeed    int64
-	DryRun      bool
+	Replicas     int
+	Chunks       int
+	MasterImage  string
+	WorkerImage  string
+	Volume       string
+	Shuffle      bool
+	RandSeed     int64
+	DryRun       bool
+	KeepExecutor bool
 }
 
 type composeTemplateOptions struct {

hack/integration-cli-on-swarm/host/host.go

@@ -42,6 +42,7 @@ func xmain() (int, error) {
 	randSeed := flag.Int64("rand-seed", int64(0), "Random seed used for shuffling (0 == curent time)")
 	filtersFile := flag.String("filters-file", "", "Path to optional file composed of `-check.f` filter strings")
 	dryRun := flag.Bool("dry-run", false, "Dry run")
+	keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm")
 	flag.Parse()
 	if *chunks == 0 {
 		*chunks = *replicas
@@ -73,14 +74,15 @@ func xmain() (int, error) {
 		workerImageForStack = *pushWorkerImage
 	}
 	compose, err := createCompose("", cli, composeOptions{
-		Replicas:    *replicas,
-		Chunks:      *chunks,
-		MasterImage: defaultMasterImageName,
-		WorkerImage: workerImageForStack,
-		Volume:      defaultVolumeName,
-		Shuffle:     *shuffle,
-		RandSeed:    *randSeed,
-		DryRun:      *dryRun,
+		Replicas:     *replicas,
+		Chunks:       *chunks,
+		MasterImage:  defaultMasterImageName,
+		WorkerImage:  workerImageForStack,
+		Volume:       defaultVolumeName,
+		Shuffle:      *shuffle,
+		RandSeed:     *randSeed,
+		DryRun:       *dryRun,
+		KeepExecutor: *keepExecutor,
 	})
 	if err != nil {
 		return 1, err