From 87e1464c438f4163e2694a0bf29ae5a45903449b Mon Sep 17 00:00:00 2001 From: Renaud Gaubert Date: Tue, 30 May 2017 17:02:11 -0700 Subject: [PATCH] Added support for Generic Resources Signed-off-by: Renaud Gaubert --- api/swagger.yaml | 89 +++++++++++++------ api/types/swarm/task.go | 31 ++++++- api/types/types.go | 1 + cmd/dockerd/config.go | 2 + daemon/cluster/convert/node.go | 1 + daemon/cluster/convert/service.go | 50 ++++++++++- daemon/cluster/convert/task.go | 3 +- .../cluster/executor/container/container.go | 6 +- daemon/cluster/executor/container/executor.go | 2 + daemon/config/config.go | 7 ++ daemon/daemon.go | 17 ++++ daemon/info.go | 1 + opts/opts.go | 19 ++++ 13 files changed, 196 insertions(+), 33 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index 7e451167e8..a3f2e84e16 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -487,6 +487,41 @@ definitions: type: "integer" format: "int64" + ResourceObject: + description: "An object describing the resources which can be advertised by a node and requested by a task" + type: "object" + properties: + NanoCPUs: + type: "integer" + format: "int64" + MemoryBytes: + type: "integer" + format: "int64" + GenericResources: + $ref: "#/definitions/GenericResources" + + GenericResources: + description: "User defined Resources, can be either Integer resources (e.g: SSD=3) or String resources (e.g: GPU={UUID1, UUID2})" + type: "array" + items: + type: "object" + properties: + NamedResourceSpec: + type: "object" + properties: + Kind: + type: "string" + Value: + type: "string" + DiscreteResourceSpec: + type: "object" + properties: + Kind: + type: "string" + Value: + type: "integer" + format: "int64" + HealthConfig: description: "A test to perform to check that the container is healthy." type: "object" @@ -1702,14 +1737,7 @@ definitions: OS: type: "string" Resources: - type: "object" - properties: - NanoCPUs: - type: "integer" - format: "int64" - MemoryBytes: - type: "integer" - format: "int64" + $ref: "#/definitions/ResourceObject" Engine: type: "object" properties: @@ -1750,6 +1778,16 @@ definitions: Resources: NanoCPUs: 4000000000 MemoryBytes: 8272408576 + GenericResources: + - DiscreteResourceSpec: + Kind: "SSD" + Value: 3 + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID1" + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID2" Engine: EngineVersion: "17.04.0" Labels: @@ -2132,27 +2170,10 @@ definitions: properties: Limits: description: "Define resources limits." - type: "object" - properties: - NanoCPUs: - description: "CPU limit in units of 10-9 CPU shares." - type: "integer" - format: "int64" - MemoryBytes: - description: "Memory limit in Bytes." - type: "integer" - format: "int64" + $ref: "#/definitions/ResourceObject" Reservation: description: "Define resources reservation." - properties: - NanoCPUs: - description: "CPU reservation in units of 10-9 CPU shares." - type: "integer" - format: "int64" - MemoryBytes: - description: "Memory reservation in Bytes." - type: "integer" - format: "int64" + $ref: "#/definitions/ResourceObject" RestartPolicy: description: "Specification for the restart policy which applies to containers created as part of this service." type: "object" @@ -2283,6 +2304,8 @@ definitions: NodeID: description: "The ID of the node that this task is on." type: "string" + AssignedGenericResources: + $ref: "#/definitions/GenericResources" Status: type: "object" properties: @@ -2362,6 +2385,16 @@ definitions: Gateway: "10.255.0.1" Addresses: - "10.255.0.10/16" + AssignedGenericResources: + - DiscreteResourceSpec: + Kind: "SSD" + Value: 3 + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID1" + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID2" ServiceSpec: description: "User modifiable configuration for a service." properties: @@ -5392,6 +5425,8 @@ paths: type: "string" MemTotal: type: "integer" + GenericResources: + $ref: "#/definitions/GenericResources" MemoryLimit: type: "boolean" NCPU: diff --git a/api/types/swarm/task.go b/api/types/swarm/task.go index 1712c06cf6..ff11b07e74 100644 --- a/api/types/swarm/task.go +++ b/api/types/swarm/task.go @@ -51,6 +51,7 @@ type Task struct { Status TaskStatus `json:",omitempty"` DesiredState TaskState `json:",omitempty"` NetworksAttachments []NetworkAttachment `json:",omitempty"` + GenericResources []GenericResource `json:",omitempty"` } // TaskSpec represents the spec of a task. @@ -79,8 +80,34 @@ type TaskSpec struct { // Resources represents resources (CPU/Memory). type Resources struct { - NanoCPUs int64 `json:",omitempty"` - MemoryBytes int64 `json:",omitempty"` + NanoCPUs int64 `json:",omitempty"` + MemoryBytes int64 `json:",omitempty"` + GenericResources []GenericResource `json:",omitempty"` +} + +// GenericResource represents a "user defined" resource which can +// be either an integer (e.g: SSD=3) or a string (e.g: SSD=sda1) +type GenericResource struct { + NamedResourceSpec *NamedGenericResource `json:",omitempty"` + DiscreteResourceSpec *DiscreteGenericResource `json:",omitempty"` +} + +// NamedGenericResource represents a "user defined" resource which is defined +// as a string. +// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...) +// Value is used to identify the resource (GPU="UUID-1", FPGA="/dev/sdb5", ...) +type NamedGenericResource struct { + Kind string `json:",omitempty"` + Value string `json:",omitempty"` +} + +// DiscreteGenericResource represents a "user defined" resource which is defined +// as an integer +// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...) +// Value is used to count the resource (SSD=5, HDD=3, ...) +type DiscreteGenericResource struct { + Kind string `json:",omitempty"` + Value int64 `json:",omitempty"` } // ResourceRequirements represents resources requirements. diff --git a/api/types/types.go b/api/types/types.go index c96df27332..f7ac772971 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -168,6 +168,7 @@ type Info struct { RegistryConfig *registry.ServiceConfig NCPU int MemTotal int64 + GenericResources []swarm.GenericResource DockerRootDir string HTTPProxy string `json:"HttpProxy"` HTTPSProxy string `json:"HttpsProxy"` diff --git a/cmd/dockerd/config.go b/cmd/dockerd/config.go index 440497bfbc..11084ec8d6 100644 --- a/cmd/dockerd/config.go +++ b/cmd/dockerd/config.go @@ -62,6 +62,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.StringVar(&conf.MetricsAddress, "metrics-addr", "", "Set default address and port to serve the metrics api on") + flags.StringVar(&conf.NodeGenericResources, "node-generic-resources", "", "user defined resources (e.g. fpga=2;gpu={UUID1,UUID2,UUID3})") + // "--deprecated-key-path" is to allow configuration of the key used // for the daemon ID and the deprecated image signing. It was never // exposed as a command line option but is added here to allow diff --git a/daemon/cluster/convert/node.go b/daemon/cluster/convert/node.go index f075783e88..528ec54a60 100644 --- a/daemon/cluster/convert/node.go +++ b/daemon/cluster/convert/node.go @@ -42,6 +42,7 @@ func NodeFromGRPC(n swarmapi.Node) types.Node { if n.Description.Resources != nil { node.Description.Resources.NanoCPUs = n.Description.Resources.NanoCPUs node.Description.Resources.MemoryBytes = n.Description.Resources.MemoryBytes + node.Description.Resources.GenericResources = GenericResourcesFromGRPC(n.Description.Resources.Generic) } if n.Description.Engine != nil { node.Description.Engine.EngineVersion = n.Description.Engine.EngineVersion diff --git a/daemon/cluster/convert/service.go b/daemon/cluster/convert/service.go index 947debdf52..f0dee57bd4 100644 --- a/daemon/cluster/convert/service.go +++ b/daemon/cluster/convert/service.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types/swarm/runtime" "github.com/docker/docker/pkg/namesgenerator" swarmapi "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/api/genericresource" "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" "github.com/pkg/errors" @@ -301,6 +302,31 @@ func annotationsFromGRPC(ann swarmapi.Annotations) types.Annotations { return a } +// GenericResourcesFromGRPC converts a GRPC GenericResource to a GenericResource +func GenericResourcesFromGRPC(genericRes []*swarmapi.GenericResource) []types.GenericResource { + var generic []types.GenericResource + for _, res := range genericRes { + var current types.GenericResource + + switch r := res.Resource.(type) { + case *swarmapi.GenericResource_DiscreteResourceSpec: + current.DiscreteResourceSpec = &types.DiscreteGenericResource{ + Kind: r.DiscreteResourceSpec.Kind, + Value: r.DiscreteResourceSpec.Value, + } + case *swarmapi.GenericResource_NamedResourceSpec: + current.NamedResourceSpec = &types.NamedGenericResource{ + Kind: r.NamedResourceSpec.Kind, + Value: r.NamedResourceSpec.Value, + } + } + + generic = append(generic, current) + } + + return generic +} + func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequirements { var resources *types.ResourceRequirements if res != nil { @@ -313,8 +339,9 @@ func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequir } if res.Reservations != nil { resources.Reservations = &types.Resources{ - NanoCPUs: res.Reservations.NanoCPUs, - MemoryBytes: res.Reservations.MemoryBytes, + NanoCPUs: res.Reservations.NanoCPUs, + MemoryBytes: res.Reservations.MemoryBytes, + GenericResources: GenericResourcesFromGRPC(res.Reservations.Generic), } } } @@ -322,6 +349,24 @@ func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequir return resources } +// GenericResourcesToGRPC converts a GenericResource to a GRPC GenericResource +func GenericResourcesToGRPC(genericRes []types.GenericResource) []*swarmapi.GenericResource { + var generic []*swarmapi.GenericResource + for _, res := range genericRes { + var r *swarmapi.GenericResource + + if res.DiscreteResourceSpec != nil { + r = genericresource.NewDiscrete(res.DiscreteResourceSpec.Kind, res.DiscreteResourceSpec.Value) + } else if res.NamedResourceSpec != nil { + r = genericresource.NewString(res.NamedResourceSpec.Kind, res.NamedResourceSpec.Value) + } + + generic = append(generic, r) + } + + return generic +} + func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirements { var reqs *swarmapi.ResourceRequirements if res != nil { @@ -336,6 +381,7 @@ func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirem reqs.Reservations = &swarmapi.Resources{ NanoCPUs: res.Reservations.NanoCPUs, MemoryBytes: res.Reservations.MemoryBytes, + Generic: GenericResourcesToGRPC(res.Reservations.GenericResources), } } diff --git a/daemon/cluster/convert/task.go b/daemon/cluster/convert/task.go index e301415c6c..bedf2dba9e 100644 --- a/daemon/cluster/convert/task.go +++ b/daemon/cluster/convert/task.go @@ -30,7 +30,8 @@ func TaskFromGRPC(t swarmapi.Task) (types.Task, error) { Message: t.Status.Message, Err: t.Status.Err, }, - DesiredState: types.TaskState(strings.ToLower(t.DesiredState.String())), + DesiredState: types.TaskState(strings.ToLower(t.DesiredState.String())), + GenericResources: GenericResourcesFromGRPC(t.AssignedGenericResources), } // Meta diff --git a/daemon/cluster/executor/container/container.go b/daemon/cluster/executor/container/container.go index d25e59acf6..3ca7b5dcef 100644 --- a/daemon/cluster/executor/container/container.go +++ b/daemon/cluster/executor/container/container.go @@ -25,6 +25,7 @@ import ( netconst "github.com/docker/libnetwork/datastore" "github.com/docker/swarmkit/agent/exec" "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/api/genericresource" "github.com/docker/swarmkit/template" gogotypes "github.com/gogo/protobuf/types" ) @@ -186,13 +187,16 @@ func (c *containerConfig) exposedPorts() map[nat.Port]struct{} { } func (c *containerConfig) config() *enginecontainer.Config { + genericEnvs := genericresource.EnvFormat(c.task.AssignedGenericResources, "DOCKER_RESOURCE") + env := append(c.spec().Env, genericEnvs...) + config := &enginecontainer.Config{ Labels: c.labels(), StopSignal: c.spec().StopSignal, Tty: c.spec().TTY, OpenStdin: c.spec().OpenStdin, User: c.spec().User, - Env: c.spec().Env, + Env: env, Hostname: c.spec().Hostname, WorkingDir: c.spec().Dir, Image: c.image(), diff --git a/daemon/cluster/executor/container/executor.go b/daemon/cluster/executor/container/executor.go index a71a9412e3..f6fb6e55be 100644 --- a/daemon/cluster/executor/container/executor.go +++ b/daemon/cluster/executor/container/executor.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/api/types/network" swarmtypes "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/daemon/cluster/controllers/plugin" + "github.com/docker/docker/daemon/cluster/convert" executorpkg "github.com/docker/docker/daemon/cluster/executor" clustertypes "github.com/docker/docker/daemon/cluster/provider" networktypes "github.com/docker/libnetwork/types" @@ -119,6 +120,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) { Resources: &api.Resources{ NanoCPUs: int64(info.NCPU) * 1e9, MemoryBytes: info.MemTotal, + Generic: convert.GenericResourcesToGRPC(info.GenericResources), }, } diff --git a/daemon/config/config.go b/daemon/config/config.go index beb147224f..86a1356c57 100644 --- a/daemon/config/config.go +++ b/daemon/config/config.go @@ -168,6 +168,9 @@ type CommonConfig struct { ValuesSet map[string]interface{} Experimental bool `json:"experimental"` // Experimental indicates whether experimental features should be exposed or not + + // Exposed node Generic Resources + NodeGenericResources string `json:"node-generic-resources,omitempty"` } // IsValueSet returns true if a configuration value @@ -497,6 +500,10 @@ func Validate(config *Config) error { } } + if _, err := opts.ParseGenericResources(config.NodeGenericResources); err != nil { + return err + } + if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" && defaultRuntime != StockRuntimeName { runtimes := config.GetAllRuntimes() if _, ok := runtimes[defaultRuntime]; !ok { diff --git a/daemon/daemon.go b/daemon/daemon.go index 8359ef31ca..6d09c79bf1 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -23,12 +23,14 @@ import ( "github.com/docker/docker/api" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/container" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/opts" // register graph drivers _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/initlayer" @@ -111,6 +113,7 @@ type Daemon struct { defaultIsolation containertypes.Isolation // Default isolation mode on Windows clusterProvider cluster.Provider cluster Cluster + genericResources []swarm.GenericResource metricsPluginListener net.Listener machineMemory uint64 @@ -568,6 +571,9 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe } }() + if err := d.setGenericResources(config); err != nil { + return nil, err + } // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event // on Windows to dump Go routine stacks stackDumpDir := config.Root @@ -1036,6 +1042,17 @@ func (daemon *Daemon) setupInitLayer(initPath string) error { return initlayer.Setup(initPath, rootIDs) } +func (daemon *Daemon) setGenericResources(conf *config.Config) error { + genericResources, err := opts.ParseGenericResources(conf.NodeGenericResources) + if err != nil { + return err + } + + daemon.genericResources = genericResources + + return nil +} + func setDefaultMtu(conf *config.Config) { // do nothing if the config does not have the default 0 value. if conf.Mtu != 0 { diff --git a/daemon/info.go b/daemon/info.go index 2cb3e84798..1de899f797 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -123,6 +123,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { RegistryConfig: daemon.RegistryService.ServiceConfig(), NCPU: sysinfo.NumCPU(), MemTotal: meminfo.MemTotal, + GenericResources: daemon.genericResources, DockerRootDir: daemon.configStore.Root, Labels: daemon.configStore.Labels, ExperimentalBuild: daemon.configStore.Experimental, diff --git a/opts/opts.go b/opts/opts.go index 8d82f76792..300fb426ad 100644 --- a/opts/opts.go +++ b/opts/opts.go @@ -7,7 +7,10 @@ import ( "regexp" "strings" + "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/daemon/cluster/convert" units "github.com/docker/go-units" + "github.com/docker/swarmkit/api/genericresource" ) var ( @@ -325,3 +328,19 @@ func (m *MemBytes) UnmarshalJSON(s []byte) error { *m = MemBytes(val) return err } + +// ParseGenericResources parses and validates the specified string as a list of GenericResource +func ParseGenericResources(value string) ([]swarm.GenericResource, error) { + if value == "" { + return nil, nil + } + + resources, err := genericresource.Parse(value) + if err != nil { + return nil, err + } + + obj := convert.GenericResourcesFromGRPC(resources) + + return obj, nil +}