Added support for Generic Resources

Signed-off-by: Renaud Gaubert <rgaubert@nvidia.com>

api/swagger.yaml

@@ -487,6 +487,41 @@ definitions:
         type: "integer"
         format: "int64"
 
+  ResourceObject:
+    description: "An object describing the resources which can be advertised by a node and requested by a task"
+    type: "object"
+    properties:
+      NanoCPUs:
+        type: "integer"
+        format: "int64"
+      MemoryBytes:
+        type: "integer"
+        format: "int64"
+      GenericResources:
+        $ref: "#/definitions/GenericResources"
+
+  GenericResources:
+    description: "User defined Resources, can be either Integer resources (e.g: SSD=3) or String resources (e.g: GPU={UUID1, UUID2})"
+    type: "array"
+    items:
+      type: "object"
+      properties:
+        NamedResourceSpec:
+          type: "object"
+          properties:
+            Kind:
+              type: "string"
+            Value:
+              type: "string"
+        DiscreteResourceSpec:
+          type: "object"
+          properties:
+            Kind:
+              type: "string"
+            Value:
+              type: "integer"
+              format: "int64"
+
   HealthConfig:
     description: "A test to perform to check that the container is healthy."
     type: "object"
@@ -1702,14 +1737,7 @@ definitions:
               OS:
                 type: "string"
           Resources:
-            type: "object"
-            properties:
-              NanoCPUs:
-                type: "integer"
-                format: "int64"
-              MemoryBytes:
-                type: "integer"
-                format: "int64"
+            $ref: "#/definitions/ResourceObject"
           Engine:
             type: "object"
             properties:
@@ -1750,6 +1778,16 @@ definitions:
         Resources:
           NanoCPUs: 4000000000
           MemoryBytes: 8272408576
+          GenericResources:
+            - DiscreteResourceSpec:
+                Kind: "SSD"
+                Value: 3
+            - NamedResourceSpec:
+                Kind: "GPU"
+                Value: "UUID1"
+            - NamedResourceSpec:
+                Kind: "GPU"
+                Value: "UUID2"
         Engine:
           EngineVersion: "17.04.0"
           Labels:
@@ -2132,27 +2170,10 @@ definitions:
         properties:
           Limits:
             description: "Define resources limits."
-            type: "object"
-            properties:
-              NanoCPUs:
-                description: "CPU limit in units of 10<sup>-9</sup> CPU shares."
-                type: "integer"
-                format: "int64"
-              MemoryBytes:
-                description: "Memory limit in Bytes."
-                type: "integer"
-                format: "int64"
+            $ref: "#/definitions/ResourceObject"
           Reservation:
             description: "Define resources reservation."
-            properties:
-              NanoCPUs:
-                description: "CPU reservation in units of 10<sup>-9</sup> CPU shares."
-                type: "integer"
-                format: "int64"
-              MemoryBytes:
-                description: "Memory reservation in Bytes."
-                type: "integer"
-                format: "int64"
+            $ref: "#/definitions/ResourceObject"
       RestartPolicy:
         description: "Specification for the restart policy which applies to containers created as part of this service."
         type: "object"
@@ -2283,6 +2304,8 @@ definitions:
       NodeID:
         description: "The ID of the node that this task is on."
         type: "string"
+      AssignedGenericResources:
+        $ref: "#/definitions/GenericResources"
       Status:
         type: "object"
         properties:
@@ -2362,6 +2385,16 @@ definitions:
                   Gateway: "10.255.0.1"
           Addresses:
             - "10.255.0.10/16"
+      AssignedGenericResources:
+        - DiscreteResourceSpec:
+            Kind: "SSD"
+            Value: 3
+        - NamedResourceSpec:
+            Kind: "GPU"
+            Value: "UUID1"
+        - NamedResourceSpec:
+            Kind: "GPU"
+            Value: "UUID2"
   ServiceSpec:
     description: "User modifiable configuration for a service."
     properties:
@@ -5392,6 +5425,8 @@ paths:
                   type: "string"
               MemTotal:
                 type: "integer"
+              GenericResources:
+                $ref: "#/definitions/GenericResources"
               MemoryLimit:
                 type: "boolean"
               NCPU:

api/types/swarm/task.go

@@ -51,6 +51,7 @@ type Task struct {
 	Status              TaskStatus          `json:",omitempty"`
 	DesiredState        TaskState           `json:",omitempty"`
 	NetworksAttachments []NetworkAttachment `json:",omitempty"`
+	GenericResources    []GenericResource   `json:",omitempty"`
 }
 
 // TaskSpec represents the spec of a task.
@@ -79,8 +80,34 @@ type TaskSpec struct {
 
 // Resources represents resources (CPU/Memory).
 type Resources struct {
-	NanoCPUs    int64 `json:",omitempty"`
-	MemoryBytes int64 `json:",omitempty"`
+	NanoCPUs         int64             `json:",omitempty"`
+	MemoryBytes      int64             `json:",omitempty"`
+	GenericResources []GenericResource `json:",omitempty"`
+}
+
+// GenericResource represents a "user defined" resource which can
+// be either an integer (e.g: SSD=3) or a string (e.g: SSD=sda1)
+type GenericResource struct {
+	NamedResourceSpec    *NamedGenericResource    `json:",omitempty"`
+	DiscreteResourceSpec *DiscreteGenericResource `json:",omitempty"`
+}
+
+// NamedGenericResource represents a "user defined" resource which is defined
+// as a string.
+// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...)
+// Value is used to identify the resource (GPU="UUID-1", FPGA="/dev/sdb5", ...)
+type NamedGenericResource struct {
+	Kind  string `json:",omitempty"`
+	Value string `json:",omitempty"`
+}
+
+// DiscreteGenericResource represents a "user defined" resource which is defined
+// as an integer
+// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...)
+// Value is used to count the resource (SSD=5, HDD=3, ...)
+type DiscreteGenericResource struct {
+	Kind  string `json:",omitempty"`
+	Value int64  `json:",omitempty"`
 }
 
 // ResourceRequirements represents resources requirements.

api/types/types.go

@@ -168,6 +168,7 @@ type Info struct {
 	RegistryConfig     *registry.ServiceConfig
 	NCPU               int
 	MemTotal           int64
+	GenericResources   []swarm.GenericResource
 	DockerRootDir      string
 	HTTPProxy          string `json:"HttpProxy"`
 	HTTPSProxy         string `json:"HttpsProxy"`

cmd/dockerd/config.go

@@ -62,6 +62,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) {
 
 	flags.StringVar(&conf.MetricsAddress, "metrics-addr", "", "Set default address and port to serve the metrics api on")
 
+	flags.StringVar(&conf.NodeGenericResources, "node-generic-resources", "", "user defined resources (e.g. fpga=2;gpu={UUID1,UUID2,UUID3})")
+
 	// "--deprecated-key-path" is to allow configuration of the key used
 	// for the daemon ID and the deprecated image signing. It was never
 	// exposed as a command line option but is added here to allow

daemon/cluster/convert/node.go

@@ -42,6 +42,7 @@ func NodeFromGRPC(n swarmapi.Node) types.Node {
 		if n.Description.Resources != nil {
 			node.Description.Resources.NanoCPUs = n.Description.Resources.NanoCPUs
 			node.Description.Resources.MemoryBytes = n.Description.Resources.MemoryBytes
+			node.Description.Resources.GenericResources = GenericResourcesFromGRPC(n.Description.Resources.Generic)
 		}
 		if n.Description.Engine != nil {
 			node.Description.Engine.EngineVersion = n.Description.Engine.EngineVersion

daemon/cluster/convert/service.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/docker/api/types/swarm/runtime"
 	"github.com/docker/docker/pkg/namesgenerator"
 	swarmapi "github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/api/genericresource"
 	"github.com/gogo/protobuf/proto"
 	gogotypes "github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
@@ -301,6 +302,31 @@ func annotationsFromGRPC(ann swarmapi.Annotations) types.Annotations {
 	return a
 }
 
+// GenericResourcesFromGRPC converts a GRPC GenericResource to a GenericResource
+func GenericResourcesFromGRPC(genericRes []*swarmapi.GenericResource) []types.GenericResource {
+	var generic []types.GenericResource
+	for _, res := range genericRes {
+		var current types.GenericResource
+
+		switch r := res.Resource.(type) {
+		case *swarmapi.GenericResource_DiscreteResourceSpec:
+			current.DiscreteResourceSpec = &types.DiscreteGenericResource{
+				Kind:  r.DiscreteResourceSpec.Kind,
+				Value: r.DiscreteResourceSpec.Value,
+			}
+		case *swarmapi.GenericResource_NamedResourceSpec:
+			current.NamedResourceSpec = &types.NamedGenericResource{
+				Kind:  r.NamedResourceSpec.Kind,
+				Value: r.NamedResourceSpec.Value,
+			}
+		}
+
+		generic = append(generic, current)
+	}
+
+	return generic
+}
+
 func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequirements {
 	var resources *types.ResourceRequirements
 	if res != nil {
@@ -313,8 +339,9 @@ func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequir
 		}
 		if res.Reservations != nil {
 			resources.Reservations = &types.Resources{
-				NanoCPUs:    res.Reservations.NanoCPUs,
-				MemoryBytes: res.Reservations.MemoryBytes,
+				NanoCPUs:         res.Reservations.NanoCPUs,
+				MemoryBytes:      res.Reservations.MemoryBytes,
+				GenericResources: GenericResourcesFromGRPC(res.Reservations.Generic),
 			}
 		}
 	}
@@ -322,6 +349,24 @@ func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequir
 	return resources
 }
 
+// GenericResourcesToGRPC converts a GenericResource to a GRPC GenericResource
+func GenericResourcesToGRPC(genericRes []types.GenericResource) []*swarmapi.GenericResource {
+	var generic []*swarmapi.GenericResource
+	for _, res := range genericRes {
+		var r *swarmapi.GenericResource
+
+		if res.DiscreteResourceSpec != nil {
+			r = genericresource.NewDiscrete(res.DiscreteResourceSpec.Kind, res.DiscreteResourceSpec.Value)
+		} else if res.NamedResourceSpec != nil {
+			r = genericresource.NewString(res.NamedResourceSpec.Kind, res.NamedResourceSpec.Value)
+		}
+
+		generic = append(generic, r)
+	}
+
+	return generic
+}
+
 func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirements {
 	var reqs *swarmapi.ResourceRequirements
 	if res != nil {
@@ -336,6 +381,7 @@ func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirem
 			reqs.Reservations = &swarmapi.Resources{
 				NanoCPUs:    res.Reservations.NanoCPUs,
 				MemoryBytes: res.Reservations.MemoryBytes,
+				Generic:     GenericResourcesToGRPC(res.Reservations.GenericResources),
 			}
 
 		}

daemon/cluster/convert/task.go

@@ -30,7 +30,8 @@ func TaskFromGRPC(t swarmapi.Task) (types.Task, error) {
 			Message: t.Status.Message,
 			Err:     t.Status.Err,
 		},
-		DesiredState: types.TaskState(strings.ToLower(t.DesiredState.String())),
+		DesiredState:     types.TaskState(strings.ToLower(t.DesiredState.String())),
+		GenericResources: GenericResourcesFromGRPC(t.AssignedGenericResources),
 	}
 
 	// Meta

daemon/cluster/executor/container/container.go

@@ -25,6 +25,7 @@ import (
 	netconst "github.com/docker/libnetwork/datastore"
 	"github.com/docker/swarmkit/agent/exec"
 	"github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/api/genericresource"
 	"github.com/docker/swarmkit/template"
 	gogotypes "github.com/gogo/protobuf/types"
 )
@@ -186,13 +187,16 @@ func (c *containerConfig) exposedPorts() map[nat.Port]struct{} {
 }
 
 func (c *containerConfig) config() *enginecontainer.Config {
+	genericEnvs := genericresource.EnvFormat(c.task.AssignedGenericResources, "DOCKER_RESOURCE")
+	env := append(c.spec().Env, genericEnvs...)
+
 	config := &enginecontainer.Config{
 		Labels:       c.labels(),
 		StopSignal:   c.spec().StopSignal,
 		Tty:          c.spec().TTY,
 		OpenStdin:    c.spec().OpenStdin,
 		User:         c.spec().User,
-		Env:          c.spec().Env,
+		Env:          env,
 		Hostname:     c.spec().Hostname,
 		WorkingDir:   c.spec().Dir,
 		Image:        c.image(),

daemon/cluster/executor/container/executor.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/docker/api/types/network"
 	swarmtypes "github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/daemon/cluster/controllers/plugin"
+	"github.com/docker/docker/daemon/cluster/convert"
 	executorpkg "github.com/docker/docker/daemon/cluster/executor"
 	clustertypes "github.com/docker/docker/daemon/cluster/provider"
 	networktypes "github.com/docker/libnetwork/types"
@@ -119,6 +120,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
 		Resources: &api.Resources{
 			NanoCPUs:    int64(info.NCPU) * 1e9,
 			MemoryBytes: info.MemTotal,
+			Generic:     convert.GenericResourcesToGRPC(info.GenericResources),
 		},
 	}
 

daemon/config/config.go

@@ -168,6 +168,9 @@ type CommonConfig struct {
 	ValuesSet map[string]interface{}
 
 	Experimental bool `json:"experimental"` // Experimental indicates whether experimental features should be exposed or not
+
+	// Exposed node Generic Resources
+	NodeGenericResources string `json:"node-generic-resources,omitempty"`
 }
 
 // IsValueSet returns true if a configuration value
@@ -497,6 +500,10 @@ func Validate(config *Config) error {
 		}
 	}
 
+	if _, err := opts.ParseGenericResources(config.NodeGenericResources); err != nil {
+		return err
+	}
+
 	if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" && defaultRuntime != StockRuntimeName {
 		runtimes := config.GetAllRuntimes()
 		if _, ok := runtimes[defaultRuntime]; !ok {

daemon/daemon.go

@@ -23,12 +23,14 @@ import (
 	"github.com/docker/docker/api"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/discovery"
 	"github.com/docker/docker/daemon/events"
 	"github.com/docker/docker/daemon/exec"
 	"github.com/docker/docker/daemon/logger"
+	"github.com/docker/docker/opts"
 	// register graph drivers
 	_ "github.com/docker/docker/daemon/graphdriver/register"
 	"github.com/docker/docker/daemon/initlayer"
@@ -111,6 +113,7 @@ type Daemon struct {
 	defaultIsolation      containertypes.Isolation // Default isolation mode on Windows
 	clusterProvider       cluster.Provider
 	cluster               Cluster
+	genericResources      []swarm.GenericResource
 	metricsPluginListener net.Listener
 
 	machineMemory uint64
@@ -568,6 +571,9 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
 		}
 	}()
 
+	if err := d.setGenericResources(config); err != nil {
+		return nil, err
+	}
 	// set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
 	// on Windows to dump Go routine stacks
 	stackDumpDir := config.Root
@@ -1036,6 +1042,17 @@ func (daemon *Daemon) setupInitLayer(initPath string) error {
 	return initlayer.Setup(initPath, rootIDs)
 }
 
+func (daemon *Daemon) setGenericResources(conf *config.Config) error {
+	genericResources, err := opts.ParseGenericResources(conf.NodeGenericResources)
+	if err != nil {
+		return err
+	}
+
+	daemon.genericResources = genericResources
+
+	return nil
+}
+
 func setDefaultMtu(conf *config.Config) {
 	// do nothing if the config does not have the default 0 value.
 	if conf.Mtu != 0 {

daemon/info.go

@@ -123,6 +123,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
 		RegistryConfig:     daemon.RegistryService.ServiceConfig(),
 		NCPU:               sysinfo.NumCPU(),
 		MemTotal:           meminfo.MemTotal,
+		GenericResources:   daemon.genericResources,
 		DockerRootDir:      daemon.configStore.Root,
 		Labels:             daemon.configStore.Labels,
 		ExperimentalBuild:  daemon.configStore.Experimental,

opts/opts.go

@@ -7,7 +7,10 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/daemon/cluster/convert"
 	units "github.com/docker/go-units"
+	"github.com/docker/swarmkit/api/genericresource"
 )
 
 var (
@@ -325,3 +328,19 @@ func (m *MemBytes) UnmarshalJSON(s []byte) error {
 	*m = MemBytes(val)
 	return err
 }
+
+// ParseGenericResources parses and validates the specified string as a list of GenericResource
+func ParseGenericResources(value string) ([]swarm.GenericResource, error) {
+	if value == "" {
+		return nil, nil
+	}
+
+	resources, err := genericresource.Parse(value)
+	if err != nil {
+		return nil, err
+	}
+
+	obj := convert.GenericResourcesFromGRPC(resources)
+
+	return obj, nil
+}