Allow user to modify ingress network

Signed-off-by: Alessandro Boch <aboch@docker.com>

api/server/router/network/network_routes.go

@@ -294,6 +294,7 @@ func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.Netwo
 	r.EnableIPv6 = info.IPv6Enabled()
 	r.Internal = info.Internal()
 	r.Attachable = info.Attachable()
+	r.Ingress = info.Ingress()
 	r.Options = info.DriverOptions()
 	r.Containers = make(map[string]types.EndpointResource)
 	buildIpamResources(r, info)

api/swagger.yaml

@@ -1117,6 +1117,8 @@ definitions:
         type: "boolean"
       Attachable:
         type: "boolean"
+      Ingress:
+        type: "boolean"
       Containers:
         type: "object"
         additionalProperties:
@@ -1145,6 +1147,7 @@ definitions:
           foo: "bar"
       Internal: false
       Attachable: false
+      Ingress: false
       Containers:
         19a4d5d687db25203351ed79d478946f861258f018fe384f229f2efa4b23513c:
           Name: "test"
@@ -6211,6 +6214,7 @@ paths:
                 EnableIPv6: false
                 Internal: false
                 Attachable: false
+                Ingress: false
                 IPAM:
                   Driver: "default"
                   Config:
@@ -6237,6 +6241,7 @@ paths:
                 EnableIPv6: false
                 Internal: false
                 Attachable: false
+                Ingress: false
                 IPAM:
                   Driver: "default"
                   Config: []
@@ -6250,6 +6255,7 @@ paths:
                 EnableIPv6: false
                 Internal: false
                 Attachable: false
+                Ingress: false
                 IPAM:
                   Driver: "default"
                   Config: []
@@ -6383,6 +6389,9 @@ paths:
               Attachable:
                 description: "Globally scoped network is manually attachable by regular containers from workers in swarm mode."
                 type: "boolean"
+              Ingress:
+                description: "Ingress network is the network which provides the routing-mesh in swarm mode."
+                type: "boolean"
               IPAM:
                 description: "Optional custom IP scheme for the network."
                 $ref: "#/definitions/IPAM"
@@ -6416,6 +6425,7 @@ paths:
                   foo: "bar"
               Internal: true
               Attachable: false
+              Ingress: false
               Options:
                 com.docker.network.bridge.default_bridge: "true"
                 com.docker.network.bridge.enable_icc: "true"

api/types/swarm/network.go

@@ -82,6 +82,7 @@ type NetworkSpec struct {
 	IPv6Enabled         bool         `json:",omitempty"`
 	Internal            bool         `json:",omitempty"`
 	Attachable          bool         `json:",omitempty"`
+	Ingress             bool         `json:",omitempty"`
 	IPAMOptions         *IPAMOptions `json:",omitempty"`
 }
 

api/types/types.go

@@ -400,6 +400,7 @@ type NetworkResource struct {
 	IPAM       network.IPAM                   // IPAM is the network's IP Address Management
 	Internal   bool                           // Internal represents if the network is used internal only
 	Attachable bool                           // Attachable represents if the global scope is manually attachable by regular containers from workers in swarm mode.
+	Ingress    bool                           // Ingress indicates the network is providing the routing-mesh for the swarm cluster.
 	Containers map[string]EndpointResource    // Containers contains endpoints belonging to the network
 	Options    map[string]string              // Options holds the network specific options to use for when creating the network
 	Labels     map[string]string              // Labels holds metadata specific to the network being created
@@ -431,6 +432,7 @@ type NetworkCreate struct {
 	IPAM           *network.IPAM
 	Internal       bool
 	Attachable     bool
+	Ingress        bool
 	Options        map[string]string
 	Labels         map[string]string
 }

cli/command/network/create.go

@@ -24,6 +24,7 @@ type createOptions struct {
 	internal   bool
 	ipv6       bool
 	attachable bool
+	ingress    bool
 
 	ipamDriver  string
 	ipamSubnet  []string
@@ -59,6 +60,8 @@ func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
 	flags.BoolVar(&opts.ipv6, "ipv6", false, "Enable IPv6 networking")
 	flags.BoolVar(&opts.attachable, "attachable", false, "Enable manual container attachment")
 	flags.SetAnnotation("attachable", "version", []string{"1.25"})
+	flags.BoolVar(&opts.ingress, "ingress", false, "Create swarm routing-mesh network")
+	flags.SetAnnotation("ingress", "version", []string{"1.29"})
 
 	flags.StringVar(&opts.ipamDriver, "ipam-driver", "default", "IP Address Management Driver")
 	flags.StringSliceVar(&opts.ipamSubnet, "subnet", []string{}, "Subnet in CIDR format that represents a network segment")
@@ -92,6 +95,7 @@ func runCreate(dockerCli *command.DockerCli, opts createOptions) error {
 		Internal:       opts.internal,
 		EnableIPv6:     opts.ipv6,
 		Attachable:     opts.attachable,
+		Ingress:        opts.ingress,
 		Labels:         runconfigopts.ConvertKVStringsToMap(opts.labels.GetAll()),
 	}
 

cli/command/network/remove.go

@@ -22,12 +22,22 @@ func newRemoveCommand(dockerCli *command.DockerCli) *cobra.Command {
 	}
 }
 
+const ingressWarning = "WARNING! Before removing the routing-mesh network, " +
+	"make sure all the nodes in your swarm run the same docker engine version. " +
+	"Otherwise, removal may not be effective and functionality of newly create " +
+	"ingress networks will be impaired.\nAre you sure you want to continue?"
+
 func runRemove(dockerCli *command.DockerCli, networks []string) error {
 	client := dockerCli.Client()
 	ctx := context.Background()
 	status := 0
 
 	for _, name := range networks {
+		if nw, _, err := client.NetworkInspectWithRaw(ctx, name, false); err == nil &&
+			nw.Ingress &&
+			!command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), ingressWarning) {
+			continue
+		}
 		if err := client.NetworkRemove(ctx, name); err != nil {
 			fmt.Fprintf(dockerCli.Err(), "%s\n", err)
 			status = 1

daemon/cluster/convert/network.go

@@ -28,6 +28,7 @@ func networkFromGRPC(n *swarmapi.Network) types.Network {
 				IPv6Enabled: n.Spec.Ipv6Enabled,
 				Internal:    n.Spec.Internal,
 				Attachable:  n.Spec.Attachable,
+				Ingress:     n.Spec.Ingress,
 				IPAMOptions: ipamFromGRPC(n.Spec.IPAM),
 			},
 			IPAMOptions: ipamFromGRPC(n.IPAM),
@@ -156,6 +157,7 @@ func BasicNetworkFromGRPC(n swarmapi.Network) basictypes.NetworkResource {
 		IPAM:       ipam,
 		Internal:   spec.Internal,
 		Attachable: spec.Attachable,
+		Ingress:    spec.Ingress,
 		Labels:     n.Spec.Annotations.Labels,
 	}
 
@@ -181,6 +183,7 @@ func BasicNetworkCreateToGRPC(create basictypes.NetworkCreateRequest) swarmapi.N
 		Ipv6Enabled: create.EnableIPv6,
 		Internal:    create.Internal,
 		Attachable:  create.Attachable,
+		Ingress:     create.Ingress,
 	}
 	if create.IPAM != nil {
 		driver := create.IPAM.Driver

daemon/cluster/executor/backend.go

@@ -28,6 +28,7 @@ type Backend interface {
 	DeleteManagedNetwork(name string) error
 	FindNetwork(idName string) (libnetwork.Network, error)
 	SetupIngress(req clustertypes.NetworkCreateRequest, nodeIP string) error
+	ReleaseIngress() error
 	PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
 	CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
 	ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error

daemon/cluster/executor/container/container.go

@@ -575,6 +575,7 @@ func (c *containerConfig) networkCreateRequest(name string) (clustertypes.Networ
 		Labels:         na.Network.Spec.Annotations.Labels,
 		Internal:       na.Network.Spec.Internal,
 		Attachable:     na.Network.Spec.Attachable,
+		Ingress:        na.Network.Spec.Ingress,
 		EnableIPv6:     na.Network.Spec.Ipv6Enabled,
 		CheckDuplicate: true,
 	}

daemon/cluster/executor/container/executor.go

@@ -116,6 +116,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
 func (e *executor) Configure(ctx context.Context, node *api.Node) error {
 	na := node.Attachment
 	if na == nil {
+		e.backend.ReleaseIngress()
 		return nil
 	}
 
@@ -125,6 +126,7 @@ func (e *executor) Configure(ctx context.Context, node *api.Node) error {
 			Driver: na.Network.IPAM.Driver.Name,
 		},
 		Options:        na.Network.DriverState.Options,
+		Ingress:        true,
 		CheckDuplicate: true,
 	}
 

daemon/network.go

@@ -6,6 +6,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/Sirupsen/logrus"
 	apierrors "github.com/docker/docker/api/errors"
@@ -99,15 +100,40 @@ func (daemon *Daemon) getAllNetworks() []libnetwork.Network {
 	return daemon.netController.Networks()
 }
 
-func isIngressNetwork(name string) bool {
+type ingressJob struct {
-	return name == "ingress"
+	create *clustertypes.NetworkCreateRequest
+	ip     net.IP
 }
 
-var ingressChan = make(chan struct{}, 1)
+var (
+	ingressWorkerOnce  sync.Once
+	ingressJobsChannel chan *ingressJob
+	ingressID          string
+)
 
-func ingressWait() func() {
+func (daemon *Daemon) startIngressWorker() {
-	ingressChan <- struct{}{}
+	ingressJobsChannel = make(chan *ingressJob, 100)
-	return func() { <-ingressChan }
+	go func() {
+		for {
+			select {
+			case r := <-ingressJobsChannel:
+				if r.create != nil {
+					daemon.setupIngress(r.create, r.ip, ingressID)
+					ingressID = r.create.ID
+				} else {
+					daemon.releaseIngress(ingressID)
+					ingressID = ""
+				}
+			}
+		}
+	}()
+}
+
+// enqueueIngressJob adds a ingress add/rm request to the worker queue.
+// It guarantees the worker is started.
+func (daemon *Daemon) enqueueIngressJob(job *ingressJob) {
+	ingressWorkerOnce.Do(daemon.startIngressWorker)
+	ingressJobsChannel <- job
 }
 
 // SetupIngress setups ingress networking.
@@ -116,74 +142,95 @@ func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nod
 	if err != nil {
 		return err
 	}
-
+	daemon.enqueueIngressJob(&ingressJob{&create, ip})
-	go func() {
-		controller := daemon.netController
-		controller.AgentInitWait()
-
-		if n, err := daemon.GetNetworkByName(create.Name); err == nil && n != nil && n.ID() != create.ID {
-			if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
-				logrus.Errorf("Failed to delete stale ingress sandbox: %v", err)
-				return
-			}
-
-			// Cleanup any stale endpoints that might be left over during previous iterations
-			epList := n.Endpoints()
-			for _, ep := range epList {
-				if err := ep.Delete(true); err != nil {
-					logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
-				}
-			}
-
-			if err := n.Delete(); err != nil {
-				logrus.Errorf("Failed to delete stale ingress network %s: %v", n.ID(), err)
-				return
-			}
-		}
-
-		if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
-			// If it is any other error other than already
-			// exists error log error and return.
-			if _, ok := err.(libnetwork.NetworkNameError); !ok {
-				logrus.Errorf("Failed creating ingress network: %v", err)
-				return
-			}
-
-			// Otherwise continue down the call to create or recreate sandbox.
-		}
-
-		n, err := daemon.GetNetworkByID(create.ID)
-		if err != nil {
-			logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
-			return
-		}
-
-		sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
-		if err != nil {
-			if _, ok := err.(networktypes.ForbiddenError); !ok {
-				logrus.Errorf("Failed creating ingress sandbox: %v", err)
-			}
-			return
-		}
-
-		ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
-		if err != nil {
-			logrus.Errorf("Failed creating ingress endpoint: %v", err)
-			return
-		}
-
-		if err := ep.Join(sb, nil); err != nil {
-			logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
-		}
-
-		if err := sb.EnableService(); err != nil {
-			logrus.WithError(err).Error("Failed enabling service for ingress sandbox")
-		}
-	}()
-
 	return nil
 }
 
+// ReleaseIngress releases the ingress networking.
+func (daemon *Daemon) ReleaseIngress() error {
+	daemon.enqueueIngressJob(&ingressJob{nil, nil})
+	return nil
+}
+
+func (daemon *Daemon) setupIngress(create *clustertypes.NetworkCreateRequest, ip net.IP, staleID string) {
+	controller := daemon.netController
+	controller.AgentInitWait()
+
+	if staleID != "" && staleID != create.ID {
+		daemon.releaseIngress(staleID)
+	}
+
+	if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
+		// If it is any other error other than already
+		// exists error log error and return.
+		if _, ok := err.(libnetwork.NetworkNameError); !ok {
+			logrus.Errorf("Failed creating ingress network: %v", err)
+			return
+		}
+		// Otherwise continue down the call to create or recreate sandbox.
+	}
+
+	n, err := daemon.GetNetworkByID(create.ID)
+	if err != nil {
+		logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
+	}
+
+	sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
+	if err != nil {
+		if _, ok := err.(networktypes.ForbiddenError); !ok {
+			logrus.Errorf("Failed creating ingress sandbox: %v", err)
+		}
+		return
+	}
+
+	ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
+	if err != nil {
+		logrus.Errorf("Failed creating ingress endpoint: %v", err)
+		return
+	}
+
+	if err := ep.Join(sb, nil); err != nil {
+		logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
+		return
+	}
+
+	if err := sb.EnableService(); err != nil {
+		logrus.Errorf("Failed enabling service for ingress sandbox")
+	}
+}
+
+func (daemon *Daemon) releaseIngress(id string) {
+	controller := daemon.netController
+
+	if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
+		logrus.Errorf("Failed to delete ingress sandbox: %v", err)
+	}
+
+	if id == "" {
+		return
+	}
+
+	n, err := controller.NetworkByID(id)
+	if err != nil {
+		logrus.Errorf("failed to retrieve ingress network %s: %v", id, err)
+		return
+	}
+
+	for _, ep := range n.Endpoints() {
+		if err := ep.Delete(true); err != nil {
+			logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
+			return
+		}
+	}
+
+	if err := n.Delete(); err != nil {
+		logrus.Errorf("Failed to delete ingress network %s: %v", n.ID(), err)
+		return
+	}
+
+	return
+}
+
 // SetNetworkBootstrapKeys sets the bootstrap keys.
 func (daemon *Daemon) SetNetworkBootstrapKeys(keys []*networktypes.EncryptionKey) error {
 	return daemon.netController.SetKeys(keys)
@@ -228,13 +275,6 @@ func (daemon *Daemon) CreateNetwork(create types.NetworkCreateRequest) (*types.N
 }
 
 func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string, agent bool) (*types.NetworkCreateResponse, error) {
-	// If there is a pending ingress network creation wait here
-	// since ingress network creation can happen via node download
-	// from manager or task download.
-	if isIngressNetwork(create.Name) {
-		defer ingressWait()()
-	}
-
 	if runconfig.IsPreDefinedNetwork(create.Name) && !agent {
 		err := fmt.Errorf("%s is a pre-defined network and cannot be created", create.Name)
 		return nil, apierrors.NewRequestForbiddenError(err)
@@ -267,6 +307,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 		libnetwork.NetworkOptionDriverOpts(create.Options),
 		libnetwork.NetworkOptionLabels(create.Labels),
 		libnetwork.NetworkOptionAttachable(create.Attachable),
+		libnetwork.NetworkOptionIngress(create.Ingress),
 	}
 
 	if create.IPAM != nil {
@@ -286,10 +327,6 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 		nwOptions = append(nwOptions, libnetwork.NetworkOptionPersist(false))
 	}
 
-	if isIngressNetwork(create.Name) {
-		nwOptions = append(nwOptions, libnetwork.NetworkOptionIngress())
-	}
-
 	n, err := c.NewNetwork(driver, create.Name, id, nwOptions...)
 	if err != nil {
 		return nil, err

daemon/prune.go

@@ -231,7 +231,8 @@ func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.Ne
 	}
 	networkIsInUse := regexp.MustCompile(`network ([[:alnum:]]+) is in use`)
 	for _, nw := range networks {
-		if nw.Name == "ingress" {
+		if nw.Ingress {
+			// Routing-mesh network removal has to be explicitly invoked by user
 			continue
 		}
 		if !until.IsZero() && nw.Created.After(until) {

docs/api/version-history.md

@@ -17,6 +17,10 @@ keywords: "API, Docker, rcli, REST, documentation"
 
 [Docker Engine API v1.29](https://docs.docker.com/engine/api/v1.29/) documentation
 
+
+* `DELETE /networks/(name)` now allows to remove the ingress network, the one used to provide the routing-mesh.
+* `POST /networks/create` now supports creating the ingress network, by specifying an `Ingress` boolean field. As of now this is supported only when using the overlay network driver.
+* `GET /networks/(name)` now returns an `Ingress` field showing whether the network is the ingress one.
 * `GET /networks/` now supports a `scope` filter to filter networks based on the network mode (`swarm`, `global`, or `local`).
 
 ## v1.28 API changes

docs/reference/commandline/network_create.md

@@ -22,6 +22,7 @@ Create a network
 
 Options:
       --attachable           Enable manual container attachment
+      --ingress              Specify the network provides the routing-mesh
       --aux-address value    Auxiliary IPv4 or IPv6 addresses used by Network
                              driver (default map[])
   -d, --driver string        Driver to manage the Network (default "bridge")
@@ -195,6 +196,23 @@ connects a bridge network to it to provide external connectivity. If you want
 to create an externally isolated `overlay` network, you can specify the
 `--internal` option.
 
+### Network ingress mode
+
+You can create the network which will be used to provide the routing-mesh in the
+swarm cluster. You do so by specifying `--ingress` when creating the network. Only
+one ingress network can be created at the time. The network can be removed only
+if no services depend on it. Any option available when creating a overlay network
+is also available when creating the ingress network, besides the `--attachable` option.
+
+```bash
+$ docker network create -d overlay \
+  --subnet=10.11.0.0/16 \
+  --ingress \
+  --opt com.docker.network.mtu=9216 \
+  --opt encrypted=true \
+  my-ingress-network
+```
+
 ## Related commands
 
 * [network inspect](network_inspect.md)

integration-cli/docker_cli_swarm_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -19,6 +20,7 @@ import (
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/cli"
 	"github.com/docker/docker/integration-cli/daemon"
+	"github.com/docker/docker/pkg/testutil"
 	icmd "github.com/docker/docker/pkg/testutil/cmd"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/ipamapi"
@@ -413,14 +415,57 @@ func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *che
 	c.Assert(err, checker.IsNil, check.Commentf(out))
 }
 
-func (s *DockerSwarmSuite) TestSwarmRemoveInternalNetwork(c *check.C) {
+func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *check.C) {
 	d := s.AddDaemon(c, true, true)
 
-	name := "ingress"
+	// Ingress network can be removed
-	out, err := d.Cmd("network", "rm", name)
+	out, _, err := testutil.RunCommandPipelineWithOutput(
+		exec.Command("echo", "Y"),
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "ingress"),
+	)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	// And recreated
+	out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	// But only one is allowed
+	out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress")
 	c.Assert(err, checker.NotNil)
-	c.Assert(strings.TrimSpace(out), checker.Contains, name)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "is already present")
-	c.Assert(strings.TrimSpace(out), checker.Contains, "is a pre-defined network and cannot be removed")
+
+	// It cannot be removed if it is being used
+	out, err = d.Cmd("service", "create", "--name", "srv1", "-p", "9000:8000", "busybox", "top")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	out, _, err = testutil.RunCommandPipelineWithOutput(
+		exec.Command("echo", "Y"),
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
+	)
+	c.Assert(err, checker.NotNil)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "ingress network cannot be removed because service")
+
+	// But it can be removed once no more services depend on it
+	out, err = d.Cmd("service", "update", "--publish-rm", "9000:8000", "srv1")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+	out, _, err = testutil.RunCommandPipelineWithOutput(
+		exec.Command("echo", "Y"),
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
+	)
+	c.Assert(err, checker.IsNil, check.Commentf(out))
+
+	// A service which needs the ingress network cannot be created if no ingress is present
+	out, err = d.Cmd("service", "create", "--name", "srv2", "-p", "500:500", "busybox", "top")
+	c.Assert(err, checker.NotNil)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
+
+	// An existing service cannot be updated to use the ingress nw if the nw is not present
+	out, err = d.Cmd("service", "update", "--publish-add", "9000:8000", "srv1")
+	c.Assert(err, checker.NotNil)
+	c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
+
+	// But services which do not need routing mesh can be created regardless
+	out, err = d.Cmd("service", "create", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top")
+	c.Assert(err, checker.IsNil, check.Commentf(out))
 }
 
 // Test case for #24108, also the case from:

man/src/network/create.md

@@ -117,3 +117,20 @@ By default, when you connect a container to an `overlay` network, Docker also
 connects a bridge network to it to provide external connectivity. If you want
 to create an externally isolated `overlay` network, you can specify the
 `--internal` option.
+
+### Network ingress mode
+
+You can create the network which will be used to provide the routing-mesh in the
+swarm cluster. You do so by specifying `--ingress` when creating the network. Only
+one ingress network can be created at the time. The network can be removed only
+if no services depend on it. Any option available when creating a overlay network
+is also available when creating the ingress network, besides the `--attachable` option.
+
+```bash
+$ docker network create -d overlay \
+  --subnet=10.11.0.0/16 \
+  --ingress \
+  --opt com.docker.network.mtu=9216 \
+  --opt encrypted=true \
+  my-ingress-network
+```

man/src/network/inspect.md

@@ -32,6 +32,7 @@ $ sudo docker network inspect bridge
             ]
         },
         "Internal": false,
+        "Ingress": false,
         "Containers": {
             "bda12f8922785d1f160be70736f26c1e331ab8aaf8ed8d56728508f2e2fd4727": {
                 "Name": "container2",
@@ -116,6 +117,7 @@ $ docker network inspect --verbose ov1
         },
         "Internal": false,
         "Attachable": false,
+        "Ingress": false,
         "Containers": {
             "020403bd88a15f60747fd25d1ad5fa1272eb740e8a97fc547d8ad07b2f721c5e": {
                 "Name": "s1.1.pjn2ik0sfgkfzed3h0s00gs9o",

runconfig/hostconfig_unix.go

@@ -19,7 +19,7 @@ func DefaultDaemonNetworkMode() container.NetworkMode {
 // IsPreDefinedNetwork indicates if a network is predefined by the daemon
 func IsPreDefinedNetwork(network string) bool {
 	n := container.NetworkMode(network)
-	return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault() || network == "ingress"
+	return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault()
 }
 
 // validateNetMode ensures that the various combinations of requested