Merge pull request #30144 from dnephin/add-secrets-to-stack-deploy

Add secrets to stack deploy

cli/command/secret/utils.go

@@ -11,7 +11,8 @@ import (
 	"golang.org/x/net/context"
 )
 
-func getSecretsByNameOrIDPrefixes(ctx context.Context, client client.APIClient, terms []string) ([]swarm.Secret, error) {
+// GetSecretsByNameOrIDPrefixes returns secrets given a list of ids or names
+func GetSecretsByNameOrIDPrefixes(ctx context.Context, client client.APIClient, terms []string) ([]swarm.Secret, error) {
 	args := filters.NewArgs()
 	for _, n := range terms {
 		args.Add("names", n)
@@ -24,7 +25,7 @@ func getSecretsByNameOrIDPrefixes(ctx context.Context, client client.APIClient,
 }
 
 func getCliRequestedSecretIDs(ctx context.Context, client client.APIClient, terms []string) ([]string, error) {
-	secrets, err := getSecretsByNameOrIDPrefixes(ctx, client, terms)
+	secrets, err := GetSecretsByNameOrIDPrefixes(ctx, client, terms)
 	if err != nil {
 		return nil, err
 	}

cli/command/service/create.go

@@ -62,7 +62,7 @@ func runCreate(dockerCli *command.DockerCli, opts *serviceOptions) error {
 	specifiedSecrets := opts.secrets.Value()
 	if len(specifiedSecrets) > 0 {
 		// parse and validate secrets
-		secrets, err := parseSecrets(apiClient, specifiedSecrets)
+		secrets, err := ParseSecrets(apiClient, specifiedSecrets)
 		if err != nil {
 			return err
 		}

cli/command/service/parse.go

@@ -10,9 +10,9 @@ import (
 	"golang.org/x/net/context"
 )
 
-// parseSecrets retrieves the secrets from the requested names and converts
+// ParseSecrets retrieves the secrets from the requested names and converts
 // them to secret references to use with the spec
-func parseSecrets(client client.SecretAPIClient, requestedSecrets []*types.SecretRequestOption) ([]*swarmtypes.SecretReference, error) {
+func ParseSecrets(client client.SecretAPIClient, requestedSecrets []*types.SecretRequestOption) ([]*swarmtypes.SecretReference, error) {
 	secretRefs := make(map[string]*swarmtypes.SecretReference)
 	ctx := context.Background()
 

cli/command/service/update.go

@@ -443,7 +443,7 @@ func getUpdatedSecrets(apiClient client.SecretAPIClient, flags *pflag.FlagSet, s
 	if flags.Changed(flagSecretAdd) {
 		values := flags.Lookup(flagSecretAdd).Value.(*opts.SecretOpt).Value()
 
-		addSecrets, err := parseSecrets(apiClient, values)
+		addSecrets, err := ParseSecrets(apiClient, values)
 		if err != nil {
 			return nil, err
 		}

cli/command/stack/common.go

@@ -48,3 +48,13 @@ func getStackNetworks(
 		ctx,
 		types.NetworkListOptions{Filters: getStackFilter(namespace)})
 }
+
+func getStackSecrets(
+	ctx context.Context,
+	apiclient client.APIClient,
+	namespace string,
+) ([]swarm.Secret, error) {
+	return apiclient.SecretList(
+		ctx,
+		types.SecretListOptions{Filters: getStackFilter(namespace)})
+}

cli/command/stack/deploy.go

@@ -1,24 +1,24 @@
 package stack
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"sort"
 	"strings"
 
-	"github.com/spf13/cobra"
-	"golang.org/x/net/context"
-
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/cli"
 	"github.com/docker/docker/cli/command"
+	secretcli "github.com/docker/docker/cli/command/secret"
 	"github.com/docker/docker/cli/compose/convert"
 	"github.com/docker/docker/cli/compose/loader"
 	composetypes "github.com/docker/docker/cli/compose/types"
 	dockerclient "github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
 )
 
 const (
@@ -126,7 +126,16 @@ func deployCompose(ctx context.Context, dockerCli *command.DockerCli, opts deplo
 	if err := createNetworks(ctx, dockerCli, namespace, networks); err != nil {
 		return err
 	}
-	services, err := convert.Services(namespace, config)
+
+	secrets, err := convert.Secrets(namespace, config.Secrets)
+	if err != nil {
+		return err
+	}
+	if err := createSecrets(ctx, dockerCli, namespace, secrets); err != nil {
+		return err
+	}
+
+	services, err := convert.Services(namespace, config, dockerCli.Client())
 	if err != nil {
 		return err
 	}
@@ -211,6 +220,37 @@ func validateExternalNetworks(
 	return nil
 }
 
+func createSecrets(
+	ctx context.Context,
+	dockerCli *command.DockerCli,
+	namespace convert.Namespace,
+	secrets []swarm.SecretSpec,
+) error {
+	client := dockerCli.Client()
+
+	for _, secretSpec := range secrets {
+		// TODO: fix this after https://github.com/docker/docker/pull/29218
+		secrets, err := secretcli.GetSecretsByNameOrIDPrefixes(ctx, client, []string{secretSpec.Name})
+		switch {
+		case err != nil:
+			return err
+		case len(secrets) > 1:
+			return errors.Errorf("ambiguous secret name: %s", secretSpec.Name)
+		case len(secrets) == 0:
+			fmt.Fprintf(dockerCli.Out(), "Creating secret %s\n", secretSpec.Name)
+			_, err = client.SecretCreate(ctx, secretSpec)
+		default:
+			secret := secrets[0]
+			// Update secret to ensure that the local data hasn't changed
+			err = client.SecretUpdate(ctx, secret.ID, secret.Meta.Version, secretSpec)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func createNetworks(
 	ctx context.Context,
 	dockerCli *command.DockerCli,

cli/command/stack/remove.go

@@ -3,11 +3,12 @@ package stack
 import (
 	"fmt"
 
-	"golang.org/x/net/context"
+	"github.com/docker/docker/api/types"
-
+	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/cli"
 	"github.com/docker/docker/cli/command"
 	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
 )
 
 type removeOptions struct {
@@ -33,41 +34,79 @@ func newRemoveCommand(dockerCli *command.DockerCli) *cobra.Command {
 func runRemove(dockerCli *command.DockerCli, opts removeOptions) error {
 	namespace := opts.namespace
 	client := dockerCli.Client()
-	stderr := dockerCli.Err()
 	ctx := context.Background()
-	hasError := false
 
 	services, err := getServices(ctx, client, namespace)
 	if err != nil {
 		return err
 	}
-	for _, service := range services {
-		fmt.Fprintf(stderr, "Removing service %s\n", service.Spec.Name)
-		if err := client.ServiceRemove(ctx, service.ID); err != nil {
-			hasError = true
-			fmt.Fprintf(stderr, "Failed to remove service %s: %s", service.ID, err)
-		}
-	}
 
 	networks, err := getStackNetworks(ctx, client, namespace)
 	if err != nil {
 		return err
 	}
-	for _, network := range networks {
+
-		fmt.Fprintf(stderr, "Removing network %s\n", network.Name)
+	secrets, err := getStackSecrets(ctx, client, namespace)
-		if err := client.NetworkRemove(ctx, network.ID); err != nil {
+	if err != nil {
-			hasError = true
+		return err
-			fmt.Fprintf(stderr, "Failed to remove network %s: %s", network.ID, err)
-		}
 	}
 
-	if len(services) == 0 && len(networks) == 0 {
+	if len(services)+len(networks)+len(secrets) == 0 {
 		fmt.Fprintf(dockerCli.Out(), "Nothing found in stack: %s\n", namespace)
 		return nil
 	}
 
+	hasError := removeServices(ctx, dockerCli, services)
+	hasError = removeSecrets(ctx, dockerCli, secrets) || hasError
+	hasError = removeNetworks(ctx, dockerCli, networks) || hasError
+
 	if hasError {
 		return fmt.Errorf("Failed to remove some resources")
 	}
 	return nil
 }
+
+func removeServices(
+	ctx context.Context,
+	dockerCli *command.DockerCli,
+	services []swarm.Service,
+) bool {
+	var err error
+	for _, service := range services {
+		fmt.Fprintf(dockerCli.Err(), "Removing service %s\n", service.Spec.Name)
+		if err = dockerCli.Client().ServiceRemove(ctx, service.ID); err != nil {
+			fmt.Fprintf(dockerCli.Err(), "Failed to remove service %s: %s", service.ID, err)
+		}
+	}
+	return err != nil
+}
+
+func removeNetworks(
+	ctx context.Context,
+	dockerCli *command.DockerCli,
+	networks []types.NetworkResource,
+) bool {
+	var err error
+	for _, network := range networks {
+		fmt.Fprintf(dockerCli.Err(), "Removing network %s\n", network.Name)
+		if err = dockerCli.Client().NetworkRemove(ctx, network.ID); err != nil {
+			fmt.Fprintf(dockerCli.Err(), "Failed to remove network %s: %s", network.ID, err)
+		}
+	}
+	return err != nil
+}
+
+func removeSecrets(
+	ctx context.Context,
+	dockerCli *command.DockerCli,
+	secrets []swarm.Secret,
+) bool {
+	var err error
+	for _, secret := range secrets {
+		fmt.Fprintf(dockerCli.Err(), "Removing secret %s\n", secret.Spec.Name)
+		if err = dockerCli.Client().SecretRemove(ctx, secret.ID); err != nil {
+			fmt.Fprintf(dockerCli.Err(), "Failed to remove secret %s: %s", secret.ID, err)
+		}
+	}
+	return err != nil
+}

cli/compose/convert/compose.go

@@ -1,8 +1,11 @@
 package convert
 
 import (
+	"io/ioutil"
+
 	"github.com/docker/docker/api/types"
 	networktypes "github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/swarm"
 	composetypes "github.com/docker/docker/cli/compose/types"
 )
 
@@ -82,3 +85,27 @@ func Networks(namespace Namespace, networks networkMap, servicesNetworks map[str
 
 	return result, externalNetworks
 }
+
+// Secrets converts secrets from the Compose type to the engine API type
+func Secrets(namespace Namespace, secrets map[string]composetypes.SecretConfig) ([]swarm.SecretSpec, error) {
+	result := []swarm.SecretSpec{}
+	for name, secret := range secrets {
+		if secret.External.External {
+			continue
+		}
+
+		data, err := ioutil.ReadFile(secret.File)
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(result, swarm.SecretSpec{
+			Annotations: swarm.Annotations{
+				Name:   namespace.Scope(name),
+				Labels: AddStackLabel(namespace, secret.Labels),
+			},
+			Data: data,
+		})
+	}
+	return result, nil
+}

cli/compose/convert/compose_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/docker/api/types/network"
 	composetypes "github.com/docker/docker/cli/compose/types"
 	"github.com/docker/docker/pkg/testutil/assert"
+	"github.com/docker/docker/pkg/testutil/tempfile"
 )
 
 func TestNamespaceScope(t *testing.T) {
@@ -88,3 +89,34 @@ func TestNetworks(t *testing.T) {
 	assert.DeepEqual(t, networks, expected)
 	assert.DeepEqual(t, externals, []string{"special"})
 }
+
+func TestSecrets(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+
+	secretText := "this is the first secret"
+	secretFile := tempfile.NewTempFile(t, "convert-secrets", secretText)
+	defer secretFile.Remove()
+
+	source := map[string]composetypes.SecretConfig{
+		"one": {
+			File:   secretFile.Name(),
+			Labels: map[string]string{"monster": "mash"},
+		},
+		"ext": {
+			External: composetypes.External{
+				External: true,
+			},
+		},
+	}
+
+	specs, err := Secrets(namespace, source)
+	assert.NilError(t, err)
+	assert.Equal(t, len(specs), 1)
+	secret := specs[0]
+	assert.Equal(t, secret.Name, "foo_one")
+	assert.DeepEqual(t, secret.Labels, map[string]string{
+		"monster":      "mash",
+		LabelNamespace: "foo",
+	})
+	assert.DeepEqual(t, secret.Data, []byte(secretText))
+}

cli/compose/convert/service.go

@@ -2,20 +2,26 @@ package convert
 
 import (
 	"fmt"
+	"os"
 	"time"
 
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/swarm"
+	servicecli "github.com/docker/docker/cli/command/service"
 	composetypes "github.com/docker/docker/cli/compose/types"
+	"github.com/docker/docker/client"
 	"github.com/docker/docker/opts"
 	runconfigopts "github.com/docker/docker/runconfig/opts"
 	"github.com/docker/go-connections/nat"
 )
 
 // Services from compose-file types to engine API types
+// TODO: fix secrets API so that SecretAPIClient is not required here
 func Services(
 	namespace Namespace,
 	config *composetypes.Config,
+	client client.SecretAPIClient,
 ) (map[string]swarm.ServiceSpec, error) {
 	result := make(map[string]swarm.ServiceSpec)
 
@@ -24,7 +30,12 @@ func Services(
 	networks := config.Networks
 
 	for _, service := range services {
-		serviceSpec, err := convertService(namespace, service, networks, volumes)
+
+		secrets, err := convertServiceSecrets(client, namespace, service.Secrets, config.Secrets)
+		if err != nil {
+			return nil, err
+		}
+		serviceSpec, err := convertService(namespace, service, networks, volumes, secrets)
 		if err != nil {
 			return nil, err
 		}
@@ -39,6 +50,7 @@ func convertService(
 	service composetypes.ServiceConfig,
 	networkConfigs map[string]composetypes.NetworkConfig,
 	volumes map[string]composetypes.VolumeConfig,
+	secrets []*swarm.SecretReference,
 ) (swarm.ServiceSpec, error) {
 	name := namespace.Scope(service.Name)
 
@@ -108,6 +120,7 @@ func convertService(
 				StopGracePeriod: service.StopGracePeriod,
 				TTY:             service.Tty,
 				OpenStdin:       service.StdinOpen,
+				Secrets:         secrets,
 			},
 			LogDriver:     logDriver,
 			Resources:     resources,
@@ -163,6 +176,47 @@ func convertServiceNetworks(
 	return nets, nil
 }
 
+// TODO: fix secrets API so that SecretAPIClient is not required here
+func convertServiceSecrets(
+	client client.SecretAPIClient,
+	namespace Namespace,
+	secrets []composetypes.ServiceSecretConfig,
+	secretSpecs map[string]composetypes.SecretConfig,
+) ([]*swarm.SecretReference, error) {
+	opts := []*types.SecretRequestOption{}
+	for _, secret := range secrets {
+		target := secret.Target
+		if target == "" {
+			target = secret.Source
+		}
+
+		source := namespace.Scope(secret.Source)
+		secretSpec := secretSpecs[secret.Source]
+		if secretSpec.External.External {
+			source = secretSpec.External.Name
+		}
+
+		uid := secret.UID
+		gid := secret.GID
+		if uid == "" {
+			uid = "0"
+		}
+		if gid == "" {
+			gid = "0"
+		}
+
+		opts = append(opts, &types.SecretRequestOption{
+			Source: source,
+			Target: target,
+			UID:    uid,
+			GID:    gid,
+			Mode:   os.FileMode(secret.Mode),
+		})
+	}
+
+	return servicecli.ParseSecrets(client, opts)
+}
+
 func convertExtraHosts(extraHosts map[string]string) []string {
 	hosts := []string{}
 	for host, ip := range extraHosts {

cli/compose/loader/loader.go

@@ -62,16 +62,11 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 		}
 	}
 
-	if err := schema.Validate(configDict); err != nil {
+	if err := schema.Validate(configDict, schema.Version(configDict)); err != nil {
 		return nil, err
 	}
 
 	cfg := types.Config{}
-	version := configDict["version"].(string)
-	if version != "3" && version != "3.0" {
-		return nil, fmt.Errorf(`Unsupported Compose file version: %#v. The only version supported is "3" (or "3.0")`, version)
-	}
-
 	if services, ok := configDict["services"]; ok {
 		servicesConfig, err := interpolation.Interpolate(services.(types.Dict), "service", os.LookupEnv)
 		if err != nil {
@@ -114,6 +109,20 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 		cfg.Volumes = volumesMapping
 	}
 
+	if secrets, ok := configDict["secrets"]; ok {
+		secretsConfig, err := interpolation.Interpolate(secrets.(types.Dict), "secret", os.LookupEnv)
+		if err != nil {
+			return nil, err
+		}
+
+		secretsMapping, err := loadSecrets(secretsConfig, configDetails.WorkingDir)
+		if err != nil {
+			return nil, err
+		}
+
+		cfg.Secrets = secretsMapping
+	}
+
 	return &cfg, nil
 }
 
@@ -215,13 +224,15 @@ func transformHook(
 ) (interface{}, error) {
 	switch target {
 	case reflect.TypeOf(types.External{}):
-		return transformExternal(source, target, data)
+		return transformExternal(data)
 	case reflect.TypeOf(make(map[string]string, 0)):
 		return transformMapStringString(source, target, data)
 	case reflect.TypeOf(types.UlimitsConfig{}):
-		return transformUlimits(source, target, data)
+		return transformUlimits(data)
 	case reflect.TypeOf(types.UnitBytes(0)):
 		return loadSize(data)
+	case reflect.TypeOf(types.ServiceSecretConfig{}):
+		return transformServiceSecret(data)
 	}
 	switch target.Kind() {
 	case reflect.Struct:
@@ -316,7 +327,7 @@ func resolveEnvironment(serviceConfig *types.ServiceConfig, serviceDict types.Di
 		var envVars []string
 
 		for _, file := range envFiles {
-			filePath := path.Join(workingDir, file)
+			filePath := absPath(workingDir, file)
 			fileVars, err := opts.ParseEnvFile(filePath)
 			if err != nil {
 				return err
@@ -346,7 +357,7 @@ func resolveVolumePaths(volumes []string, workingDir string) error {
 		}
 
 		if strings.HasPrefix(parts[0], ".") {
-			parts[0] = path.Join(workingDir, parts[0])
+			parts[0] = absPath(workingDir, parts[0])
 		}
 		parts[0] = expandUser(parts[0])
 
@@ -364,11 +375,7 @@ func expandUser(path string) string {
 	return path
 }
 
-func transformUlimits(
+func transformUlimits(data interface{}) (interface{}, error) {
-	source reflect.Type,
-	target reflect.Type,
-	data interface{},
-) (interface{}, error) {
 	switch value := data.(type) {
 	case int:
 		return types.UlimitsConfig{Single: value}, nil
@@ -412,6 +419,31 @@ func loadVolumes(source types.Dict) (map[string]types.VolumeConfig, error) {
 	return volumes, nil
 }
 
+// TODO: remove duplicate with networks/volumes
+func loadSecrets(source types.Dict, workingDir string) (map[string]types.SecretConfig, error) {
+	secrets := make(map[string]types.SecretConfig)
+	if err := transform(source, &secrets); err != nil {
+		return secrets, err
+	}
+	for name, secret := range secrets {
+		if secret.External.External && secret.External.Name == "" {
+			secret.External.Name = name
+			secrets[name] = secret
+		}
+		if secret.File != "" {
+			secret.File = absPath(workingDir, secret.File)
+		}
+	}
+	return secrets, nil
+}
+
+func absPath(workingDir string, filepath string) string {
+	if path.IsAbs(filepath) {
+		return filepath
+	}
+	return path.Join(workingDir, filepath)
+}
+
 func transformStruct(
 	source reflect.Type,
 	target reflect.Type,
@@ -495,11 +527,7 @@ func convertField(
 	return data, nil
 }
 
-func transformExternal(
+func transformExternal(data interface{}) (interface{}, error) {
-	source reflect.Type,
-	target reflect.Type,
-	data interface{},
-) (interface{}, error) {
 	switch value := data.(type) {
 	case bool:
 		return map[string]interface{}{"external": value}, nil
@@ -512,6 +540,20 @@ func transformExternal(
 	}
 }
 
+func transformServiceSecret(data interface{}) (interface{}, error) {
+	switch value := data.(type) {
+	case string:
+		return map[string]interface{}{"source": value}, nil
+	case types.Dict:
+		return data, nil
+	case map[string]interface{}:
+		return data, nil
+	default:
+		return data, fmt.Errorf("invalid type %T for external", value)
+	}
+
+}
+
 func toYAMLName(name string) string {
 	nameParts := fieldNameRegexp.FindAllString(name, -1)
 	for i, p := range nameParts {

cli/compose/loader/loader_test.go

@@ -163,6 +163,24 @@ func TestLoad(t *testing.T) {
 	assert.Equal(t, sampleConfig.Volumes, actual.Volumes)
 }
 
+func TestLoadV31(t *testing.T) {
+	actual, err := loadYAML(`
+version: "3.1"
+services:
+  foo:
+    image: busybox
+    secrets: [super]
+secrets:
+  super:
+    external: true
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, len(actual.Services), 1)
+	assert.Equal(t, len(actual.Secrets), 1)
+}
+
 func TestParseAndLoad(t *testing.T) {
 	actual, err := loadYAML(sampleYAML)
 	if !assert.NoError(t, err) {

cli/compose/schema/data/config_schema_v3.1.json

@@ -0,0 +1,428 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "config_schema_v3.1.json",
+  "type": "object",
+  "required": ["version"],
+
+  "properties": {
+    "version": {
+      "type": "string"
+    },
+
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
+
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "secrets": {
+      "id": "#/properties/secrets",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/secret"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "additionalProperties": false,
+
+  "definitions": {
+
+    "service": {
+      "id": "#/definitions/service",
+      "type": "object",
+
+      "properties": {
+        "deploy": {"$ref": "#/definitions/deployment"},
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"}
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cgroup_parent": {"type": "string"},
+        "command": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "container_name": {"type": "string"},
+        "depends_on": {"$ref": "#/definitions/list_of_strings"},
+        "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "dns": {"$ref": "#/definitions/string_or_list"},
+        "dns_search": {"$ref": "#/definitions/string_or_list"},
+        "domainname": {"type": "string"},
+        "entrypoint": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "env_file": {"$ref": "#/definitions/string_or_list"},
+        "environment": {"$ref": "#/definitions/list_or_dict"},
+
+        "expose": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "expose"
+          },
+          "uniqueItems": true
+        },
+
+        "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+        "healthcheck": {"$ref": "#/definitions/healthcheck"},
+        "hostname": {"type": "string"},
+        "image": {"type": "string"},
+        "ipc": {"type": "string"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+
+        "logging": {
+            "type": "object",
+
+            "properties": {
+                "driver": {"type": "string"},
+                "options": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^.+$": {"type": ["string", "number", "null"]}
+                  }
+                }
+            },
+            "additionalProperties": false
+        },
+
+        "mac_address": {"type": "string"},
+        "network_mode": {"type": "string"},
+
+        "networks": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "aliases": {"$ref": "#/definitions/list_of_strings"},
+                        "ipv4_address": {"type": "string"},
+                        "ipv6_address": {"type": "string"}
+                      },
+                      "additionalProperties": false
+                    },
+                    {"type": "null"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "pid": {"type": ["string", "null"]},
+
+        "ports": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "ports"
+          },
+          "uniqueItems": true
+        },
+
+        "privileged": {"type": "boolean"},
+        "read_only": {"type": "boolean"},
+        "restart": {"type": "string"},
+        "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "shm_size": {"type": ["number", "string"]},
+        "secrets": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "properties": {
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "uid": {"type": "string"},
+                  "gid": {"type": "string"},
+                  "mode": {"type": "number"}
+                }
+              }
+            ]
+          }
+        },
+        "sysctls": {"$ref": "#/definitions/list_or_dict"},
+        "stdin_open": {"type": "boolean"},
+        "stop_grace_period": {"type": "string", "format": "duration"},
+        "stop_signal": {"type": "string"},
+        "tmpfs": {"$ref": "#/definitions/string_or_list"},
+        "tty": {"type": "boolean"},
+        "ulimits": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "oneOf": [
+                {"type": "integer"},
+                {
+                  "type":"object",
+                  "properties": {
+                    "hard": {"type": "integer"},
+                    "soft": {"type": "integer"}
+                  },
+                  "required": ["soft", "hard"],
+                  "additionalProperties": false
+                }
+              ]
+            }
+          }
+        },
+        "user": {"type": "string"},
+        "userns_mode": {"type": "string"},
+        "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "working_dir": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "healthcheck": {
+      "id": "#/definitions/healthcheck",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "disable": {"type": "boolean"},
+        "interval": {"type": "string"},
+        "retries": {"type": "number"},
+        "test": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "timeout": {"type": "string"}
+      }
+    },
+    "deployment": {
+      "id": "#/definitions/deployment",
+      "type": ["object", "null"],
+      "properties": {
+        "mode": {"type": "string"},
+        "replicas": {"type": "integer"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "update_config": {
+          "type": "object",
+          "properties": {
+            "parallelism": {"type": "integer"},
+            "delay": {"type": "string", "format": "duration"},
+            "failure_action": {"type": "string"},
+            "monitor": {"type": "string", "format": "duration"},
+            "max_failure_ratio": {"type": "number"}
+          },
+          "additionalProperties": false
+        },
+        "resources": {
+          "type": "object",
+          "properties": {
+            "limits": {"$ref": "#/definitions/resource"},
+            "reservations": {"$ref": "#/definitions/resource"}
+          }
+        },
+        "restart_policy": {
+          "type": "object",
+          "properties": {
+            "condition": {"type": "string"},
+            "delay": {"type": "string", "format": "duration"},
+            "max_attempts": {"type": "integer"},
+            "window": {"type": "string", "format": "duration"}
+          },
+          "additionalProperties": false
+        },
+        "placement": {
+          "type": "object",
+          "properties": {
+            "constraints": {"type": "array", "items": {"type": "string"}}
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "resource": {
+      "id": "#/definitions/resource",
+      "type": "object",
+      "properties": {
+        "cpus": {"type": "string"},
+        "memory": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "network": {
+      "id": "#/definitions/network",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "ipam": {
+          "type": "object",
+          "properties": {
+            "driver": {"type": "string"},
+            "config": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "subnet": {"type": "string"}
+                },
+                "additionalProperties": false
+              }
+            }
+          },
+          "additionalProperties": false
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "internal": {"type": "boolean"},
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "secret": {
+      "id": "#/definitions/secret",
+      "type": "object",
+      "properties": {
+        "file": {"type": "string"},
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          }
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "string_or_list": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/list_of_strings"}
+      ]
+    },
+
+    "list_of_strings": {
+      "type": "array",
+      "items": {"type": "string"},
+      "uniqueItems": true
+    },
+
+    "list_or_dict": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": ["string", "number", "null"]
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "constraints": {
+      "service": {
+        "id": "#/definitions/constraints/service",
+        "anyOf": [
+          {"required": ["build"]},
+          {"required": ["image"]}
+        ],
+        "properties": {
+          "build": {
+            "required": ["context"]
+          }
+        }
+      }
+    }
+  }
+}

cli/compose/schema/schema.go

@@ -7,9 +7,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/xeipuuv/gojsonschema"
 )
 
+const (
+	defaultVersion = "1.0"
+	versionField   = "version"
+)
+
 type portsFormatChecker struct{}
 
 func (checker portsFormatChecker) IsFormat(input string) bool {
@@ -30,11 +36,29 @@ func init() {
 	gojsonschema.FormatCheckers.Add("duration", durationFormatChecker{})
 }
 
+// Version returns the version of the config, defaulting to version 1.0
+func Version(config map[string]interface{}) string {
+	version, ok := config[versionField]
+	if !ok {
+		return defaultVersion
+	}
+	return normalizeVersion(fmt.Sprintf("%v", version))
+}
+
+func normalizeVersion(version string) string {
+	switch version {
+	case "3":
+		return "3.0"
+	default:
+		return version
+	}
+}
+
 // Validate uses the jsonschema to validate the configuration
-func Validate(config map[string]interface{}) error {
+func Validate(config map[string]interface{}, version string) error {
-	schemaData, err := Asset("data/config_schema_v3.0.json")
+	schemaData, err := Asset(fmt.Sprintf("data/config_schema_v%s.json", version))
 	if err != nil {
-		return err
+		return errors.Errorf("unsupported Compose file version: %s", version)
 	}
 
 	schemaLoader := gojsonschema.NewStringLoader(string(schemaData))

cli/compose/schema/schema_test.go

@@ -8,7 +8,35 @@ import (
 
 type dict map[string]interface{}
 
-func TestValid(t *testing.T) {
+func TestValidate(t *testing.T) {
+	config := dict{
+		"version": "3.0",
+		"services": dict{
+			"foo": dict{
+				"image": "busybox",
+			},
+		},
+	}
+
+	assert.NoError(t, Validate(config, "3.0"))
+}
+
+func TestValidateUndefinedTopLevelOption(t *testing.T) {
+	config := dict{
+		"version": "3.0",
+		"helicopters": dict{
+			"foo": dict{
+				"image": "busybox",
+			},
+		},
+	}
+
+	err := Validate(config, "3.0")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Additional property helicopters is not allowed")
+}
+
+func TestValidateInvalidVersion(t *testing.T) {
 	config := dict{
 		"version": "2.1",
 		"services": dict{
@@ -18,18 +46,7 @@ func TestValid(t *testing.T) {
 		},
 	}
 
-	assert.NoError(t, Validate(config))
+	err := Validate(config, "2.1")
-}
+	assert.Error(t, err)
-
+	assert.Contains(t, err.Error(), "unsupported Compose file version: 2.1")
-func TestUndefinedTopLevelOption(t *testing.T) {
-	config := dict{
-		"version": "2.1",
-		"helicopters": dict{
-			"foo": dict{
-				"image": "busybox",
-			},
-		},
-	}
-
-	assert.Error(t, Validate(config))
 }

cli/compose/types/types.go

@@ -71,6 +71,7 @@ type Config struct {
 	Services []ServiceConfig
 	Networks map[string]NetworkConfig
 	Volumes  map[string]VolumeConfig
+	Secrets  map[string]SecretConfig
 }
 
 // ServiceConfig is the configuration of one service
@@ -108,6 +109,7 @@ type ServiceConfig struct {
 	Privileged      bool
 	ReadOnly        bool `mapstructure:"read_only"`
 	Restart         string
+	Secrets         []ServiceSecretConfig
 	SecurityOpt     []string       `mapstructure:"security_opt"`
 	StdinOpen       bool           `mapstructure:"stdin_open"`
 	StopGracePeriod *time.Duration `mapstructure:"stop_grace_period"`
@@ -191,6 +193,15 @@ type ServiceNetworkConfig struct {
 	Ipv6Address string `mapstructure:"ipv6_address"`
 }
 
+// ServiceSecretConfig is the secret configuration for a service
+type ServiceSecretConfig struct {
+	Source string
+	Target string
+	UID    string
+	GID    string
+	Mode   uint32
+}
+
 // UlimitsConfig the ulimit configuration
 type UlimitsConfig struct {
 	Single int
@@ -233,3 +244,10 @@ type External struct {
 	Name     string
 	External bool
 }
+
+// SecretConfig for a secret
+type SecretConfig struct {
+	File     string
+	External External
+	Labels   map[string]string `compose:"list_or_dict_equals"`
+}

client/secret_update.go

@@ -8,8 +8,7 @@ import (
 	"golang.org/x/net/context"
 )
 
-// SecretUpdate updates a Secret. Currently, the only part of a secret spec
+// SecretUpdate attempts to updates a Secret
-// which can be updated is Labels.
 func (cli *Client) SecretUpdate(ctx context.Context, id string, version swarm.Version, secret swarm.SecretSpec) error {
 	query := url.Values{}
 	query.Set("version", strconv.FormatUint(version.Index, 10))

integration-cli/docker_cli_stack_test.go

@@ -1,15 +1,18 @@
 package main
 
 import (
+	"encoding/json"
 	"io/ioutil"
 	"os"
+	"sort"
+	"strings"
 
+	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/go-check/check"
 )
 
-func (s *DockerSwarmSuite) TestStackRemove(c *check.C) {
+func (s *DockerSwarmSuite) TestStackRemoveUnknown(c *check.C) {
-	testRequires(c, ExperimentalDaemon)
 	d := s.AddDaemon(c, true, true)
 
 	stackArgs := append([]string{"stack", "remove", "UNKNOWN_STACK"})
@@ -19,8 +22,7 @@ func (s *DockerSwarmSuite) TestStackRemove(c *check.C) {
 	c.Assert(out, check.Equals, "Nothing found in stack: UNKNOWN_STACK\n")
 }
 
-func (s *DockerSwarmSuite) TestStackTasks(c *check.C) {
+func (s *DockerSwarmSuite) TestStackPSUnknown(c *check.C) {
-	testRequires(c, ExperimentalDaemon)
 	d := s.AddDaemon(c, true, true)
 
 	stackArgs := append([]string{"stack", "ps", "UNKNOWN_STACK"})
@@ -30,8 +32,7 @@ func (s *DockerSwarmSuite) TestStackTasks(c *check.C) {
 	c.Assert(out, check.Equals, "Nothing found in stack: UNKNOWN_STACK\n")
 }
 
-func (s *DockerSwarmSuite) TestStackServices(c *check.C) {
+func (s *DockerSwarmSuite) TestStackServicesUnknown(c *check.C) {
-	testRequires(c, ExperimentalDaemon)
 	d := s.AddDaemon(c, true, true)
 
 	stackArgs := append([]string{"stack", "services", "UNKNOWN_STACK"})
@@ -42,7 +43,6 @@ func (s *DockerSwarmSuite) TestStackServices(c *check.C) {
 }
 
 func (s *DockerSwarmSuite) TestStackDeployComposeFile(c *check.C) {
-	testRequires(c, ExperimentalDaemon)
 	d := s.AddDaemon(c, true, true)
 
 	testStackName := "testdeploy"
@@ -54,17 +54,81 @@ func (s *DockerSwarmSuite) TestStackDeployComposeFile(c *check.C) {
 	out, err := d.Cmd(stackArgs...)
 	c.Assert(err, checker.IsNil, check.Commentf(out))
 
-	out, err = d.Cmd([]string{"stack", "ls"}...)
+	out, err = d.Cmd("stack", "ls")
 	c.Assert(err, checker.IsNil)
 	c.Assert(out, check.Equals, "NAME        SERVICES\n"+"testdeploy  2\n")
 
-	out, err = d.Cmd([]string{"stack", "rm", testStackName}...)
+	out, err = d.Cmd("stack", "rm", testStackName)
 	c.Assert(err, checker.IsNil)
-	out, err = d.Cmd([]string{"stack", "ls"}...)
+	out, err = d.Cmd("stack", "ls")
 	c.Assert(err, checker.IsNil)
 	c.Assert(out, check.Equals, "NAME  SERVICES\n")
 }
 
+func (s *DockerSwarmSuite) TestStackDeployWithSecretsTwice(c *check.C) {
+	d := s.AddDaemon(c, true, true)
+
+	out, err := d.Cmd("secret", "create", "outside", "fixtures/secrets/default")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	testStackName := "testdeploy"
+	stackArgs := []string{
+		"stack", "deploy",
+		"--compose-file", "fixtures/deploy/secrets.yaml",
+		testStackName,
+	}
+	out, err = d.Cmd(stackArgs...)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Secrets }}", "testdeploy_web")
+	c.Assert(err, checker.IsNil)
+
+	var refs []swarm.SecretReference
+	c.Assert(json.Unmarshal([]byte(out), &refs), checker.IsNil)
+	c.Assert(refs, checker.HasLen, 3)
+
+	sort.Sort(sortSecrets(refs))
+	c.Assert(refs[0].SecretName, checker.Equals, "outside")
+	c.Assert(refs[1].SecretName, checker.Equals, "testdeploy_special")
+	c.Assert(refs[1].File.Name, checker.Equals, "special")
+	c.Assert(refs[2].SecretName, checker.Equals, "testdeploy_super")
+	c.Assert(refs[2].File.Name, checker.Equals, "foo.txt")
+	c.Assert(refs[2].File.Mode, checker.Equals, os.FileMode(0400))
+
+	// Deploy again to ensure there are no errors when secret hasn't changed
+	out, err = d.Cmd(stackArgs...)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+}
+
+func (s *DockerSwarmSuite) TestStackRemove(c *check.C) {
+	d := s.AddDaemon(c, true, true)
+
+	stackName := "testdeploy"
+	stackArgs := []string{
+		"stack", "deploy",
+		"--compose-file", "fixtures/deploy/remove.yaml",
+		stackName,
+	}
+	out, err := d.Cmd(stackArgs...)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	out, err = d.Cmd("stack", "ps", stackName)
+	c.Assert(err, checker.IsNil)
+	c.Assert(strings.Split(strings.TrimSpace(out), "\n"), checker.HasLen, 2)
+
+	out, err = d.Cmd("stack", "rm", stackName)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	c.Assert(out, checker.Contains, "Removing service testdeploy_web")
+	c.Assert(out, checker.Contains, "Removing network testdeploy_default")
+	c.Assert(out, checker.Contains, "Removing secret testdeploy_special")
+}
+
+type sortSecrets []swarm.SecretReference
+
+func (s sortSecrets) Len() int           { return len(s) }
+func (s sortSecrets) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s sortSecrets) Less(i, j int) bool { return s[i].SecretName < s[j].SecretName }
+
 // testDAB is the DAB JSON used for testing.
 // TODO: Use template/text and substitute "Image" with the result of
 // `docker inspect --format '{{index .RepoDigests 0}}' busybox:latest`

integration-cli/fixtures/deploy/remove.yaml

@@ -0,0 +1,11 @@
+
+version: "3.1"
+services:
+  web:
+    image: busybox@sha256:e4f93f6ed15a0cdd342f5aae387886fba0ab98af0a102da6276eaf24d6e6ade0
+    command: top
+    secrets:
+      - special
+secrets:
+  special:
+    file: fixtures/secrets/default

integration-cli/fixtures/deploy/secrets.yaml

@@ -0,0 +1,20 @@
+
+version: "3.1"
+services:
+  web:
+    image: busybox@sha256:e4f93f6ed15a0cdd342f5aae387886fba0ab98af0a102da6276eaf24d6e6ade0
+    command: top
+    secrets:
+      - special
+      - source: super
+        target: foo.txt
+        mode: 0400
+      - star
+secrets:
+  special:
+    file: fixtures/secrets/default
+  super:
+    file: fixtures/secrets/default
+  star:
+    external:
+      name: outside

integration-cli/fixtures/secrets/default

@@ -0,0 +1 @@
+this is the secret