From b9aa04cf9afceb0822bba01c59dc3fba09639d8b Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 9 Mar 2017 08:13:13 +0000 Subject: [PATCH] integration-cli-on-swarm: new CLI flag: -keep-executor Signed-off-by: Akihiro Suda --- hack/integration-cli-on-swarm/README.md | 1 + .../agent/worker/executor.go | 115 ++++++++++-------- .../agent/worker/worker.go | 5 +- hack/integration-cli-on-swarm/host/compose.go | 19 +-- hack/integration-cli-on-swarm/host/host.go | 18 +-- 5 files changed, 85 insertions(+), 73 deletions(-) diff --git a/hack/integration-cli-on-swarm/README.md b/hack/integration-cli-on-swarm/README.md index 0c4b47700c..7366f72336 100644 --- a/hack/integration-cli-on-swarm/README.md +++ b/hack/integration-cli-on-swarm/README.md @@ -64,3 +64,4 @@ Flags for debugging IT on Swarm itself: - `-rand-seed N`: the random seed. This flag is useful for deterministic replaying. By default(0), the timestamp is used. - `-filters-file FILE`: the file contains `-check.f` strings. By default, the file is automatically generated. - `-dry-run`: skip the actual workload + - `keep-executor`: do not auto-remove executor containers, which is used for running privileged programs on Swarm diff --git a/hack/integration-cli-on-swarm/agent/worker/executor.go b/hack/integration-cli-on-swarm/agent/worker/executor.go index fc9960e91d..442428ac81 100644 --- a/hack/integration-cli-on-swarm/agent/worker/executor.go +++ b/hack/integration-cli-on-swarm/agent/worker/executor.go @@ -19,69 +19,76 @@ import ( // image needs to be the worker image itself. testFlags are OR-set of regexp for filtering tests. type testChunkExecutor func(image string, tests []string) (int64, string, error) -func dryTestChunkExecutor(image string, tests []string) (int64, string, error) { - return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil +func dryTestChunkExecutor() testChunkExecutor { + return func(image string, tests []string) (int64, string, error) { + return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil + } } // privilegedTestChunkExecutor invokes a privileged container from the worker // service via bind-mounted API socket so as to execute the test chunk -func privilegedTestChunkExecutor(image string, tests []string) (int64, string, error) { - cli, err := client.NewEnvClient() - if err != nil { - return 0, "", err - } - // propagate variables from the host (needs to be defined in the compose file) - experimental := os.Getenv("DOCKER_EXPERIMENTAL") - graphdriver := os.Getenv("DOCKER_GRAPHDRIVER") - if graphdriver == "" { - info, err := cli.Info(context.Background()) +func privilegedTestChunkExecutor(autoRemove bool) testChunkExecutor { + return func(image string, tests []string) (int64, string, error) { + cli, err := client.NewEnvClient() if err != nil { return 0, "", err } - graphdriver = info.Driver - } - // `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration-cli`) - // but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work. - // - // Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs` - // - // see integration-cli/daemon/daemon.go - daemonDest := "/daemon_dest" - config := container.Config{ - Image: image, - Env: []string{ - "TESTFLAGS=-check.f " + strings.Join(tests, "|"), - "KEEPBUNDLE=1", - "DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli - "DOCKER_EXPERIMENTAL=" + experimental, - "DOCKER_GRAPHDRIVER=" + graphdriver, - "DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest, - }, - // TODO: set label? - Entrypoint: []string{"hack/dind"}, - Cmd: []string{"hack/make.sh", "test-integration-cli"}, - } - hostConfig := container.HostConfig{ - AutoRemove: true, - Privileged: true, - Mounts: []mount.Mount{ - { - Type: mount.TypeVolume, - Target: daemonDest, + // propagate variables from the host (needs to be defined in the compose file) + experimental := os.Getenv("DOCKER_EXPERIMENTAL") + graphdriver := os.Getenv("DOCKER_GRAPHDRIVER") + if graphdriver == "" { + info, err := cli.Info(context.Background()) + if err != nil { + return 0, "", err + } + graphdriver = info.Driver + } + // `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration-cli`) + // but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work. + // + // Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs` + // + // see integration-cli/daemon/daemon.go + daemonDest := "/daemon_dest" + config := container.Config{ + Image: image, + Env: []string{ + "TESTFLAGS=-check.f " + strings.Join(tests, "|"), + "KEEPBUNDLE=1", + "DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli + "DOCKER_EXPERIMENTAL=" + experimental, + "DOCKER_GRAPHDRIVER=" + graphdriver, + "DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest, }, - }, + Labels: map[string]string{ + "org.dockerproject.integration-cli-on-swarm": "", + "org.dockerproject.integration-cli-on-swarm.comment": "this non-service container is created for running privileged programs on Swarm. you can remove this container manually if the corresponding service is already stopped.", + }, + Entrypoint: []string{"hack/dind"}, + Cmd: []string{"hack/make.sh", "test-integration-cli"}, + } + hostConfig := container.HostConfig{ + AutoRemove: autoRemove, + Privileged: true, + Mounts: []mount.Mount{ + { + Type: mount.TypeVolume, + Target: daemonDest, + }, + }, + } + id, stream, err := runContainer(context.Background(), cli, config, hostConfig) + if err != nil { + return 0, "", err + } + var b bytes.Buffer + teeContainerStream(&b, os.Stdout, os.Stderr, stream) + rc, err := cli.ContainerWait(context.Background(), id) + if err != nil { + return 0, "", err + } + return rc, b.String(), nil } - id, stream, err := runContainer(context.Background(), cli, config, hostConfig) - if err != nil { - return 0, "", err - } - var b bytes.Buffer - teeContainerStream(&b, os.Stdout, os.Stderr, stream) - rc, err := cli.ContainerWait(context.Background(), id) - if err != nil { - return 0, "", err - } - return rc, b.String(), nil } func runContainer(ctx context.Context, cli *client.Client, config container.Config, hostConfig container.HostConfig) (string, io.ReadCloser, error) { diff --git a/hack/integration-cli-on-swarm/agent/worker/worker.go b/hack/integration-cli-on-swarm/agent/worker/worker.go index 4b3c6bdfa8..36ab3684d2 100644 --- a/hack/integration-cli-on-swarm/agent/worker/worker.go +++ b/hack/integration-cli-on-swarm/agent/worker/worker.go @@ -24,6 +24,7 @@ func validImageDigest(s string) bool { func xmain() error { workerImageDigest := flag.String("worker-image-digest", "", "Needs to be the digest of this worker image itself") dryRun := flag.Bool("dry-run", false, "Dry run") + keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm") flag.Parse() if !validImageDigest(*workerImageDigest) { // Because of issue #29582. @@ -31,9 +32,9 @@ func xmain() error { // So, `docker run localregistry.example.com/blahblah:latest` fails: `Unable to find image 'localregistry.example.com/blahblah:latest' locally` return fmt.Errorf("worker-image-digest must be a digest, got %q", *workerImageDigest) } - executor := privilegedTestChunkExecutor + executor := privilegedTestChunkExecutor(!*keepExecutor) if *dryRun { - executor = dryTestChunkExecutor + executor = dryTestChunkExecutor() } return handle(*workerImageDigest, executor) } diff --git a/hack/integration-cli-on-swarm/host/compose.go b/hack/integration-cli-on-swarm/host/compose.go index 7b5f955150..a92282a1a0 100644 --- a/hack/integration-cli-on-swarm/host/compose.go +++ b/hack/integration-cli-on-swarm/host/compose.go @@ -16,7 +16,7 @@ version: "3" services: worker: image: "{{.WorkerImage}}" - command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}"] + command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}", "-keep-executor={{.KeepExecutor}}"] networks: - net volumes: @@ -57,14 +57,15 @@ volumes: ` type composeOptions struct { - Replicas int - Chunks int - MasterImage string - WorkerImage string - Volume string - Shuffle bool - RandSeed int64 - DryRun bool + Replicas int + Chunks int + MasterImage string + WorkerImage string + Volume string + Shuffle bool + RandSeed int64 + DryRun bool + KeepExecutor bool } type composeTemplateOptions struct { diff --git a/hack/integration-cli-on-swarm/host/host.go b/hack/integration-cli-on-swarm/host/host.go index 254cf0dbbd..4d06651df3 100644 --- a/hack/integration-cli-on-swarm/host/host.go +++ b/hack/integration-cli-on-swarm/host/host.go @@ -41,6 +41,7 @@ func xmain() error { randSeed := flag.Int64("rand-seed", int64(0), "Random seed used for shuffling (0 == curent time)") filtersFile := flag.String("filters-file", "", "Path to optional file composed of `-check.f` filter strings") dryRun := flag.Bool("dry-run", false, "Dry run") + keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm") flag.Parse() if *chunks == 0 { *chunks = *replicas @@ -72,14 +73,15 @@ func xmain() error { workerImageForStack = *pushWorkerImage } compose, err := createCompose("", cli, composeOptions{ - Replicas: *replicas, - Chunks: *chunks, - MasterImage: defaultMasterImageName, - WorkerImage: workerImageForStack, - Volume: defaultVolumeName, - Shuffle: *shuffle, - RandSeed: *randSeed, - DryRun: *dryRun, + Replicas: *replicas, + Chunks: *chunks, + MasterImage: defaultMasterImageName, + WorkerImage: workerImageForStack, + Volume: defaultVolumeName, + Shuffle: *shuffle, + RandSeed: *randSeed, + DryRun: *dryRun, + KeepExecutor: *keepExecutor, }) if err != nil { return err