diff --git a/api/client/bundlefile/bundlefile.go b/api/client/bundlefile/bundlefile.go new file mode 100644 index 0000000000..0e31772148 --- /dev/null +++ b/api/client/bundlefile/bundlefile.go @@ -0,0 +1,62 @@ +// +build experimental + +package bundlefile + +import ( + "encoding/json" + "io" + "os" +) + +// Bundlefile stores the contents of a bundlefile +type Bundlefile struct { + Version string + Services map[string]Service +} + +// Service is a service from a bundlefile +type Service struct { + Image string + Command []string `json:",omitempty"` + Args []string `json:",omitempty"` + Env []string `json:",omitempty"` + Labels map[string]string `json:",omitempty"` + Ports []Port `json:",omitempty"` + WorkingDir *string `json:",omitempty"` + User *string `json:",omitempty"` + Networks []string `json:",omitempty"` +} + +// Port is a port as defined in a bundlefile +type Port struct { + Protocol string + Port uint32 +} + +// LoadFile loads a bundlefile from a path to the file +func LoadFile(path string) (*Bundlefile, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + + bundlefile := &Bundlefile{} + + if err := json.NewDecoder(reader).Decode(bundlefile); err != nil { + return nil, err + } + + return bundlefile, err +} + +// Print writes the contents of the bundlefile to the output writer +// as human readable json +func Print(out io.Writer, bundle *Bundlefile) error { + bytes, err := json.MarshalIndent(*bundle, "", " ") + if err != nil { + return err + } + + _, err = out.Write(bytes) + return err +} diff --git a/api/client/node/cmd.go b/api/client/node/cmd.go index d951043f78..9971745ede 100644 --- a/api/client/node/cmd.go +++ b/api/client/node/cmd.go @@ -16,7 +16,7 @@ import ( func NewNodeCommand(dockerCli *client.DockerCli) *cobra.Command { cmd := &cobra.Command{ Use: "node", - Short: "Manage docker swarm nodes", + Short: "Manage Docker Swarm nodes", Args: cli.NoArgs, Run: func(cmd *cobra.Command, args []string) { fmt.Fprintf(dockerCli.Err(), "\n"+cmd.UsageString()) diff --git a/api/client/service/cmd.go b/api/client/service/cmd.go index b660c19f6f..87c4a3e098 100644 --- a/api/client/service/cmd.go +++ b/api/client/service/cmd.go @@ -13,7 +13,7 @@ import ( func NewServiceCommand(dockerCli *client.DockerCli) *cobra.Command { cmd := &cobra.Command{ Use: "service", - Short: "Manage docker services", + Short: "Manage Docker services", Args: cli.NoArgs, Run: func(cmd *cobra.Command, args []string) { fmt.Fprintf(dockerCli.Err(), "\n"+cmd.UsageString()) diff --git a/api/client/stack/cmd.go b/api/client/stack/cmd.go new file mode 100644 index 0000000000..c095441b9c --- /dev/null +++ b/api/client/stack/cmd.go @@ -0,0 +1,38 @@ +// +build experimental + +package stack + +import ( + "fmt" + + "github.com/docker/docker/api/client" + "github.com/docker/docker/cli" + "github.com/spf13/cobra" +) + +// NewStackCommand returns a cobra command for `stack` subcommands +func NewStackCommand(dockerCli *client.DockerCli) *cobra.Command { + cmd := &cobra.Command{ + Use: "stack", + Short: "Manage Docker stacks", + Args: cli.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + fmt.Fprintf(dockerCli.Err(), "\n"+cmd.UsageString()) + }, + } + cmd.AddCommand( + newConfigCommand(dockerCli), + newDeployCommand(dockerCli), + newRemoveCommand(dockerCli), + newTasksCommand(dockerCli), + ) + return cmd +} + +// NewTopLevelDeployCommand return a command for `docker deploy` +func NewTopLevelDeployCommand(dockerCli *client.DockerCli) *cobra.Command { + cmd := newDeployCommand(dockerCli) + // Remove the aliases at the top level + cmd.Aliases = []string{} + return cmd +} diff --git a/api/client/stack/cmd_stub.go b/api/client/stack/cmd_stub.go new file mode 100644 index 0000000000..76b83e3646 --- /dev/null +++ b/api/client/stack/cmd_stub.go @@ -0,0 +1,18 @@ +// +build !experimental + +package stack + +import ( + "github.com/docker/docker/api/client" + "github.com/spf13/cobra" +) + +// NewStackCommand returns nocommand +func NewStackCommand(dockerCli *client.DockerCli) *cobra.Command { + return &cobra.Command{} +} + +// NewTopLevelDeployCommand return no command +func NewTopLevelDeployCommand(dockerCli *client.DockerCli) *cobra.Command { + return &cobra.Command{} +} diff --git a/api/client/stack/common.go b/api/client/stack/common.go new file mode 100644 index 0000000000..46c9957250 --- /dev/null +++ b/api/client/stack/common.go @@ -0,0 +1,50 @@ +// +build experimental + +package stack + +import ( + "golang.org/x/net/context" + + "github.com/docker/engine-api/client" + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/filters" + "github.com/docker/engine-api/types/swarm" +) + +const ( + labelNamespace = "com.docker.stack.namespace" +) + +func getStackLabels(namespace string, labels map[string]string) map[string]string { + if labels == nil { + labels = make(map[string]string) + } + labels[labelNamespace] = namespace + return labels +} + +func getStackFilter(namespace string) filters.Args { + filter := filters.NewArgs() + filter.Add("label", labelNamespace+"="+namespace) + return filter +} + +func getServices( + ctx context.Context, + apiclient client.APIClient, + namespace string, +) ([]swarm.Service, error) { + return apiclient.ServiceList( + ctx, + types.ServiceListOptions{Filter: getStackFilter(namespace)}) +} + +func getNetworks( + ctx context.Context, + apiclient client.APIClient, + namespace string, +) ([]types.NetworkResource, error) { + return apiclient.NetworkList( + ctx, + types.NetworkListOptions{Filters: getStackFilter(namespace)}) +} diff --git a/api/client/stack/config.go b/api/client/stack/config.go new file mode 100644 index 0000000000..696c0c3fc7 --- /dev/null +++ b/api/client/stack/config.go @@ -0,0 +1,41 @@ +// +build experimental + +package stack + +import ( + "github.com/docker/docker/api/client" + "github.com/docker/docker/api/client/bundlefile" + "github.com/docker/docker/cli" + "github.com/spf13/cobra" +) + +type configOptions struct { + bundlefile string + namespace string +} + +func newConfigCommand(dockerCli *client.DockerCli) *cobra.Command { + var opts configOptions + + cmd := &cobra.Command{ + Use: "config [OPTIONS] STACK", + Short: "Print the stack configuration", + Args: cli.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + opts.namespace = args[0] + return runConfig(dockerCli, opts) + }, + } + + flags := cmd.Flags() + addBundlefileFlag(&opts.bundlefile, flags) + return cmd +} + +func runConfig(dockerCli *client.DockerCli, opts configOptions) error { + bundle, err := loadBundlefile(dockerCli.Err(), opts.namespace, opts.bundlefile) + if err != nil { + return err + } + return bundlefile.Print(dockerCli.Out(), bundle) +} diff --git a/api/client/stack/deploy.go b/api/client/stack/deploy.go new file mode 100644 index 0000000000..8d13da5b47 --- /dev/null +++ b/api/client/stack/deploy.go @@ -0,0 +1,205 @@ +// +build experimental + +package stack + +import ( + "fmt" + + "github.com/spf13/cobra" + "golang.org/x/net/context" + + "github.com/docker/docker/api/client" + "github.com/docker/docker/api/client/bundlefile" + "github.com/docker/docker/cli" + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/network" + "github.com/docker/engine-api/types/swarm" +) + +const ( + defaultNetworkDriver = "overlay" +) + +type deployOptions struct { + bundlefile string + namespace string +} + +func newDeployCommand(dockerCli *client.DockerCli) *cobra.Command { + var opts deployOptions + + cmd := &cobra.Command{ + Use: "deploy [OPTIONS] STACK", + Aliases: []string{"up"}, + Short: "Create and update a stack", + Args: cli.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + opts.namespace = args[0] + return runDeploy(dockerCli, opts) + }, + } + + flags := cmd.Flags() + addBundlefileFlag(&opts.bundlefile, flags) + return cmd +} + +func runDeploy(dockerCli *client.DockerCli, opts deployOptions) error { + bundle, err := loadBundlefile(dockerCli.Err(), opts.namespace, opts.bundlefile) + if err != nil { + return err + } + + networks := getUniqueNetworkNames(bundle.Services) + ctx := context.Background() + + if err := updateNetworks(ctx, dockerCli, networks, opts.namespace); err != nil { + return err + } + return deployServices(ctx, dockerCli, bundle.Services, opts.namespace) +} + +func getUniqueNetworkNames(services map[string]bundlefile.Service) []string { + networkSet := make(map[string]bool) + for _, service := range services { + for _, network := range service.Networks { + networkSet[network] = true + } + } + + networks := []string{} + for network := range networkSet { + networks = append(networks, network) + } + return networks +} + +func updateNetworks( + ctx context.Context, + dockerCli *client.DockerCli, + networks []string, + namespace string, +) error { + client := dockerCli.Client() + + existingNetworks, err := getNetworks(ctx, client, namespace) + if err != nil { + return err + } + + existingNetworkMap := make(map[string]types.NetworkResource) + for _, network := range existingNetworks { + existingNetworkMap[network.Name] = network + } + + createOpts := types.NetworkCreate{ + Labels: getStackLabels(namespace, nil), + Driver: defaultNetworkDriver, + // TODO: remove when engine-api uses omitempty for IPAM + IPAM: network.IPAM{Driver: "default"}, + } + + for _, internalName := range networks { + name := fmt.Sprintf("%s_%s", namespace, internalName) + + if _, exists := existingNetworkMap[name]; exists { + continue + } + fmt.Fprintf(dockerCli.Out(), "Creating network %s\n", name) + if _, err := client.NetworkCreate(ctx, name, createOpts); err != nil { + return err + } + } + return nil +} + +func convertNetworks(networks []string, namespace string, name string) []swarm.NetworkAttachmentConfig { + nets := []swarm.NetworkAttachmentConfig{} + for _, network := range networks { + nets = append(nets, swarm.NetworkAttachmentConfig{ + Target: namespace + "_" + network, + Aliases: []string{name}, + }) + } + return nets +} + +func deployServices( + ctx context.Context, + dockerCli *client.DockerCli, + services map[string]bundlefile.Service, + namespace string, +) error { + apiClient := dockerCli.Client() + out := dockerCli.Out() + + existingServices, err := getServices(ctx, apiClient, namespace) + if err != nil { + return err + } + + existingServiceMap := make(map[string]swarm.Service) + for _, service := range existingServices { + existingServiceMap[service.Spec.Name] = service + } + + for internalName, service := range services { + name := fmt.Sprintf("%s_%s", namespace, internalName) + + var ports []swarm.PortConfig + for _, portSpec := range service.Ports { + ports = append(ports, swarm.PortConfig{ + Protocol: swarm.PortConfigProtocol(portSpec.Protocol), + TargetPort: portSpec.Port, + }) + } + + serviceSpec := swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: name, + Labels: getStackLabels(namespace, service.Labels), + }, + TaskTemplate: swarm.TaskSpec{ + ContainerSpec: swarm.ContainerSpec{ + Image: service.Image, + Command: service.Command, + Args: service.Args, + Env: service.Env, + }, + }, + EndpointSpec: &swarm.EndpointSpec{ + Ports: ports, + }, + Networks: convertNetworks(service.Networks, namespace, internalName), + } + + cspec := &serviceSpec.TaskTemplate.ContainerSpec + if service.WorkingDir != nil { + cspec.Dir = *service.WorkingDir + } + if service.User != nil { + cspec.User = *service.User + } + + if service, exists := existingServiceMap[name]; exists { + fmt.Fprintf(out, "Updating service %s (id: %s)\n", name, service.ID) + + if err := apiClient.ServiceUpdate( + ctx, + service.ID, + service.Version, + serviceSpec, + ); err != nil { + return err + } + } else { + fmt.Fprintf(out, "Creating service %s\n", name) + + if _, err := apiClient.ServiceCreate(ctx, serviceSpec); err != nil { + return err + } + } + } + + return nil +} diff --git a/api/client/stack/opts.go b/api/client/stack/opts.go new file mode 100644 index 0000000000..b4e12dc260 --- /dev/null +++ b/api/client/stack/opts.go @@ -0,0 +1,39 @@ +// +build experimental + +package stack + +import ( + "fmt" + "io" + "os" + + "github.com/docker/docker/api/client/bundlefile" + "github.com/spf13/pflag" +) + +func addBundlefileFlag(opt *string, flags *pflag.FlagSet) { + flags.StringVarP( + opt, + "bundle", "f", "", + "Path to a bundle (Default: STACK.dsb)") +} + +func loadBundlefile(stderr io.Writer, namespace string, path string) (*bundlefile.Bundlefile, error) { + defaultPath := fmt.Sprintf("%s.dsb", namespace) + + if path == "" { + path = defaultPath + } + if _, err := os.Stat(path); err != nil { + return nil, fmt.Errorf( + "Bundle %s not found. Specify the path with -f or --bundle", + path) + } + + fmt.Fprintf(stderr, "Loading bundle from %s\n", path) + bundle, err := bundlefile.LoadFile(path) + if err != nil { + return nil, fmt.Errorf("Error reading %s: %v\n", path, err) + } + return bundle, err +} diff --git a/api/client/stack/remove.go b/api/client/stack/remove.go new file mode 100644 index 0000000000..1d066d42e6 --- /dev/null +++ b/api/client/stack/remove.go @@ -0,0 +1,70 @@ +// +build experimental + +package stack + +import ( + "fmt" + + "golang.org/x/net/context" + + "github.com/docker/docker/api/client" + "github.com/docker/docker/cli" + "github.com/spf13/cobra" +) + +type removeOptions struct { + namespace string +} + +func newRemoveCommand(dockerCli *client.DockerCli) *cobra.Command { + var opts removeOptions + + cmd := &cobra.Command{ + Use: "rm STACK", + Aliases: []string{"remove", "down"}, + Short: "Remove the stack", + Args: cli.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + opts.namespace = args[0] + return runRemove(dockerCli, opts) + }, + } + return cmd +} + +func runRemove(dockerCli *client.DockerCli, opts removeOptions) error { + namespace := opts.namespace + client := dockerCli.Client() + stderr := dockerCli.Err() + ctx := context.Background() + hasError := false + + services, err := getServices(ctx, client, namespace) + if err != nil { + return err + } + for _, service := range services { + fmt.Fprintf(stderr, "Removing service %s\n", service.Spec.Name) + if err := client.ServiceRemove(ctx, service.ID); err != nil { + hasError = true + fmt.Fprintf(stderr, "Failed to remove service %s: %s", service.ID, err) + } + } + + networks, err := getNetworks(ctx, client, namespace) + if err != nil { + return err + } + for _, network := range networks { + fmt.Fprintf(stderr, "Removing network %s\n", network.Name) + if err := client.NetworkRemove(ctx, network.ID); err != nil { + hasError = true + fmt.Fprintf(stderr, "Failed to remove network %s: %s", network.ID, err) + } + } + + if hasError { + return fmt.Errorf("Failed to remove some resources") + } + return nil +} diff --git a/api/client/stack/tasks.go b/api/client/stack/tasks.go new file mode 100644 index 0000000000..5ed1fbae35 --- /dev/null +++ b/api/client/stack/tasks.go @@ -0,0 +1,62 @@ +// +build experimental + +package stack + +import ( + "golang.org/x/net/context" + + "github.com/docker/docker/api/client" + "github.com/docker/docker/api/client/idresolver" + "github.com/docker/docker/api/client/task" + "github.com/docker/docker/cli" + "github.com/docker/docker/opts" + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/swarm" + "github.com/spf13/cobra" +) + +type tasksOptions struct { + all bool + filter opts.FilterOpt + namespace string + noResolve bool +} + +func newTasksCommand(dockerCli *client.DockerCli) *cobra.Command { + opts := tasksOptions{filter: opts.NewFilterOpt()} + + cmd := &cobra.Command{ + Use: "tasks [OPTIONS] STACK", + Short: "List the tasks in the stack", + Args: cli.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + opts.namespace = args[0] + return runTasks(dockerCli, opts) + }, + } + flags := cmd.Flags() + flags.BoolVarP(&opts.all, "all", "a", false, "Display all tasks") + flags.BoolVarP(&opts.noResolve, "no-resolve", "n", false, "Do not map IDs to Names") + flags.VarP(&opts.filter, "filter", "f", "Filter output based on conditions provided") + + return cmd +} + +func runTasks(dockerCli *client.DockerCli, opts tasksOptions) error { + client := dockerCli.Client() + ctx := context.Background() + + filter := opts.filter.Value() + filter.Add("label", labelNamespace+"="+opts.namespace) + if !opts.all && !filter.Include("desired_state") { + filter.Add("desired_state", string(swarm.TaskStateRunning)) + filter.Add("desired_state", string(swarm.TaskStateAccepted)) + } + + tasks, err := client.TaskList(ctx, types.TaskListOptions{Filter: filter}) + if err != nil { + return err + } + + return task.Print(dockerCli, ctx, tasks, idresolver.New(client, opts.noResolve)) +} diff --git a/api/client/swarm/cmd.go b/api/client/swarm/cmd.go index 0c40d20d9c..03a4c0a1cb 100644 --- a/api/client/swarm/cmd.go +++ b/api/client/swarm/cmd.go @@ -13,7 +13,7 @@ import ( func NewSwarmCommand(dockerCli *client.DockerCli) *cobra.Command { cmd := &cobra.Command{ Use: "swarm", - Short: "Manage docker swarm", + Short: "Manage Docker Swarm", Args: cli.NoArgs, Run: func(cmd *cobra.Command, args []string) { fmt.Fprintf(dockerCli.Err(), "\n"+cmd.UsageString()) diff --git a/cli/cobraadaptor/adaptor.go b/cli/cobraadaptor/adaptor.go index a87c75206a..e8a29c6817 100644 --- a/cli/cobraadaptor/adaptor.go +++ b/cli/cobraadaptor/adaptor.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/api/client/plugin" "github.com/docker/docker/api/client/registry" "github.com/docker/docker/api/client/service" + "github.com/docker/docker/api/client/stack" "github.com/docker/docker/api/client/swarm" "github.com/docker/docker/api/client/system" "github.com/docker/docker/api/client/volume" @@ -42,6 +43,8 @@ func NewCobraAdaptor(clientFlags *cliflags.ClientFlags) CobraAdaptor { rootCmd.AddCommand( node.NewNodeCommand(dockerCli), service.NewServiceCommand(dockerCli), + stack.NewStackCommand(dockerCli), + stack.NewTopLevelDeployCommand(dockerCli), swarm.NewSwarmCommand(dockerCli), container.NewAttachCommand(dockerCli), container.NewCommitCommand(dockerCli), @@ -98,7 +101,9 @@ func NewCobraAdaptor(clientFlags *cliflags.ClientFlags) CobraAdaptor { func (c CobraAdaptor) Usage() []cli.Command { cmds := []cli.Command{} for _, cmd := range c.rootCmd.Commands() { - cmds = append(cmds, cli.Command{Name: cmd.Name(), Description: cmd.Short}) + if cmd.Name() != "" { + cmds = append(cmds, cli.Command{Name: cmd.Name(), Description: cmd.Short}) + } } return cmds } diff --git a/experimental/README.md b/experimental/README.md index 540769dbf3..67abbe1cdf 100644 --- a/experimental/README.md +++ b/experimental/README.md @@ -73,7 +73,7 @@ to build a Docker binary with the experimental features enabled: * [External graphdriver plugins](plugins_graphdriver.md) * [Macvlan and Ipvlan Network Drivers](vlan-networks.md) - * The user namespaces feature has graduated from experimental. + * [Docker stacks](docker-stacks.md) ## How to comment on an experimental feature diff --git a/experimental/docker-stacks.md b/experimental/docker-stacks.md new file mode 100644 index 0000000000..31a02b6f54 --- /dev/null +++ b/experimental/docker-stacks.md @@ -0,0 +1,98 @@ +# Docker Stacks + +## Overview + +Docker Stacks are an experimental feature introduced in Docker 1.12, alongside +the new concepts of Swarms and Services inside the Engine. + +A Dockerfile can be built into an image, and containers can be created from that +image. Similarly, a docker-compose.yml can be built into a **bundle**, and +**stacks** can be created from that bundle. In that sense, the bundle is a +multi-services distributable image format. + +As of 1.12, the feature is introduced as experimental, and Docker Engine doesn't +support distribution of bundles. + +## Producing a bundle + +The easiest way to produce a bundle is to generate it using `docker-compose` +from an existing `docker-compose.yml`. Of course, that's just *one* possible way +to proceed, in the same way that `docker build` isn't the only way to produce a +Docker image. + +From `docker-compose`: + + ```bash + $ docker-compose bundle + WARNING: Unsupported key 'network_mode' in services.nsqd - ignoring + WARNING: Unsupported key 'links' in services.nsqd - ignoring + WARNING: Unsupported key 'volumes' in services.nsqd - ignoring + [...] + Wrote bundle to vossibility-stack.dsb + ``` + +## Creating a stack from a bundle + +A stack is created using the `docker deploy` command: + + ```bash + # docker deploy --help + + Usage: docker deploy [OPTIONS] STACK + + Create and update a stack + + Options: + -f, --bundle string Path to a bundle (Default: STACK.dsb) + --help Print usage + ``` + +Let's deploy the stack created before: + + ```bash + # docker deploy vossibility-stack + Loading bundle from vossibility-stack.dsb + Creating service vossibility-stack_elasticsearch + Creating service vossibility-stack_kibana + Creating service vossibility-stack_logstash + Creating service vossibility-stack_lookupd + Creating service vossibility-stack_nsqd + Creating service vossibility-stack_vossibility-collector + ``` + +We can verify that services were correctly created: + + ```bash + # docker service ls + ID NAME SCALE IMAGE + COMMAND + 29bv0vnlm903 vossibility-stack_lookupd 1 nsqio/nsq@sha256:eeba05599f31eba418e96e71e0984c3dc96963ceb66924dd37a47bf7ce18a662 /nsqlookupd + 4awt47624qwh vossibility-stack_nsqd 1 nsqio/nsq@sha256:eeba05599f31eba418e96e71e0984c3dc96963ceb66924dd37a47bf7ce18a662 /nsqd --data-path=/data --lookupd-tcp-address=lookupd:4160 + 4tjx9biia6fs vossibility-stack_elasticsearch 1 elasticsearch@sha256:12ac7c6af55d001f71800b83ba91a04f716e58d82e748fa6e5a7359eed2301aa + 7563uuzr9eys vossibility-stack_kibana 1 kibana@sha256:6995a2d25709a62694a937b8a529ff36da92ebee74bafd7bf00e6caf6db2eb03 + 9gc5m4met4he vossibility-stack_logstash 1 logstash@sha256:2dc8bddd1bb4a5a34e8ebaf73749f6413c101b2edef6617f2f7713926d2141fe logstash -f /etc/logstash/conf.d/logstash.conf + axqh55ipl40h vossibility-stack_vossibility-collector 1 icecrime/vossibility-collector@sha256:f03f2977203ba6253988c18d04061c5ec7aab46bca9dfd89a9a1fa4500989fba --config /config/config.toml --debug + ``` + +## Managing stacks + +Tasks are managed using the `docker stack` command: + + ```bash + # docker stack --help + + Usage: docker stack COMMAND + + Manage Docker stacks + + Options: + --help Print usage + + Commands: + config Print the stack configuration + deploy Create and update a stack + rm Remove the stack + tasks List the tasks in the stack + + Run 'docker stack COMMAND --help' for more information on a command. + ```