From 1285c6d1257af7536654251bcec888e520602933 Mon Sep 17 00:00:00 2001 From: Olli Janatuinen Date: Mon, 21 Sep 2020 22:21:22 +0300 Subject: [PATCH] Windows CI: Add support for testing with containerd Signed-off-by: Olli Janatuinen --- Dockerfile.windows | 15 ++++++++- Jenkinsfile | 2 +- cmd/dockerd/daemon_windows.go | 2 +- daemon/start_windows.go | 4 +-- hack/ci/windows.ps1 | 40 +++++++++++++++++++++++ integration-cli/docker_api_stats_test.go | 3 ++ integration-cli/docker_cli_commit_test.go | 2 ++ integration-cli/docker_cli_ps_test.go | 1 + integration-cli/docker_cli_run_test.go | 2 ++ integration-cli/requirements_test.go | 4 +++ integration/container/exec_test.go | 1 + pkg/system/init_windows.go | 21 +++--------- testutil/environment/environment.go | 5 +++ 13 files changed, 81 insertions(+), 21 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 7e1ae9617d..1f2a625a7a 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -166,12 +166,15 @@ FROM microsoft/windowsservercore SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] ARG GO_VERSION=1.16.7 +ARG CONTAINERD_VERSION=1.5.5 ARG GOTESTSUM_COMMIT=v0.5.3 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. +# - CONTAINERD_VERSION must be consistent with 'hack/dockerfile/install/containerd.installer' used by Linux. # - FROM_DOCKERFILE is used for detection of building within a container. ENV GO_VERSION=${GO_VERSION} ` + CONTAINERD_VERSION=${CONTAINERD_VERSION} ` GIT_VERSION=2.11.1 ` GOPATH=C:\gopath ` GO111MODULE=off ` @@ -211,7 +214,7 @@ RUN ` } ` } ` ` - setx /M PATH $('C:\git\cmd;C:\git\usr\bin;'+$Env:PATH+';C:\gcc\bin;C:\go\bin'); ` + setx /M PATH $('C:\git\cmd;C:\git\usr\bin;'+$Env:PATH+';C:\gcc\bin;C:\go\bin;C:\containerd\bin'); ` ` Write-Host INFO: Downloading git...; ` $location='https://www.nuget.org/api/v2/package/GitForWindows/'+$Env:GIT_VERSION; ` @@ -252,6 +255,16 @@ RUN ` Remove-Item C:\binutils.zip; ` Remove-Item C:\gitsetup.zip; ` ` + Write-Host INFO: Downloading containerd; ` + Install-Package -Force 7Zip4PowerShell; ` + $location='https://github.com/containerd/containerd/releases/download/v'+$Env:CONTAINERD_VERSION+'/containerd-'+$Env:CONTAINERD_VERSION+'-windows-amd64.tar.gz'; ` + Download-File $location C:\containerd.tar.gz; ` + New-Item -Path C:\containerd -ItemType Directory; ` + Expand-7Zip C:\containerd.tar.gz C:\; ` + Expand-7Zip C:\containerd.tar C:\containerd; ` + Remove-Item C:\containerd.tar.gz; ` + Remove-Item C:\containerd.tar; ` + ` # Ensure all directories exist that we will require below.... $srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; ` Write-Host INFO: Ensuring existence of directory $srcDir...; ` diff --git a/Jenkinsfile b/Jenkinsfile index 8ed9ce4115..b60dbbc5c4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1308,7 +1308,7 @@ pipeline { Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip" # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location - Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip" + Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/containerd.out", "bundles/containerd.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip" ''' archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true diff --git a/cmd/dockerd/daemon_windows.go b/cmd/dockerd/daemon_windows.go index 1a9c50b21f..ef06b0eb6d 100644 --- a/cmd/dockerd/daemon_windows.go +++ b/cmd/dockerd/daemon_windows.go @@ -94,6 +94,6 @@ func newCgroupParent(config *config.Config) string { } func (cli *DaemonCli) initContainerD(_ context.Context) (func(time.Duration) error, error) { - system.InitContainerdRuntime(cli.Config.Experimental, cli.Config.ContainerdAddr) + system.InitContainerdRuntime(cli.Config.ContainerdAddr) return nil, nil } diff --git a/daemon/start_windows.go b/daemon/start_windows.go index 5d01b84df5..cbdffb3466 100644 --- a/daemon/start_windows.go +++ b/daemon/start_windows.go @@ -8,8 +8,8 @@ import ( func (daemon *Daemon) getLibcontainerdCreateOptions(_ *container.Container) (string, interface{}, error) { if system.ContainerdRuntimeSupported() { - // Set the runtime options to debug regardless of current logging level. - return "", &options.Options{Debug: true}, nil + opts := &options.Options{} + return "io.containerd.runhcs.v1", opts, nil } return "", nil, nil } diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1 index 2f2272f2fe..c2b553d5c4 100644 --- a/hack/ci/windows.ps1 +++ b/hack/ci/windows.ps1 @@ -179,6 +179,13 @@ Function Nuke-Everything { } } + # Kill any spurious containerd. + $pids=$(get-process | where-object {$_.ProcessName -like 'containerd'}).id + foreach ($p in $pids) { + Write-Host "INFO: Killing containerd with PID $p" + Stop-Process -Id $p -Force -ErrorAction SilentlyContinue + } + Stop-Process -name "cc1" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null Stop-Process -name "link" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null Stop-Process -name "compile" -Force -ErrorAction SilentlyContinue 2>&1 | Out-Null @@ -521,6 +528,15 @@ Try { Throw "ERROR: gotestsum.exe not found...." ` } + docker cp "$COMMITHASH`:c`:\containerd\bin\containerd.exe" $env:TEMP\binary\ + if (-not (Test-Path "$env:TEMP\binary\containerd.exe")) { + Throw "ERROR: containerd.exe not found...." ` + } + docker cp "$COMMITHASH`:c`:\containerd\bin\containerd-shim-runhcs-v1.exe" $env:TEMP\binary\ + if (-not (Test-Path "$env:TEMP\binary\containerd-shim-runhcs-v1.exe")) { + Throw "ERROR: containerd-shim-runhcs-v1.exe not found...." ` + } + $ErrorActionPreference = "Stop" # Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager. @@ -594,6 +610,12 @@ Try { $dutArgs += "-D" } + # Arguments: Are we starting the daemon under test in containerd mode? + if (-not ("$env:DOCKER_WINDOWS_CONTAINERD_RUNTIME" -eq "")) { + Write-Host -ForegroundColor Green "INFO: Running the daemon under test in containerd mode" + $dutArgs += "--containerd \\.\pipe\containerd-containerd" + } + # Arguments: Are we starting the daemon under test with Hyper-V containers as the default isolation? if (-not ("$env:DOCKER_DUT_HYPERV" -eq "")) { Write-Host -ForegroundColor Green "INFO: Running the daemon under test with Hyper-V containers as the default" @@ -616,6 +638,15 @@ Try { Write-Host -ForegroundColor Green "INFO: Args: $dutArgs" New-Item -ItemType Directory $env:TEMP\daemon -ErrorAction SilentlyContinue | Out-Null + # Start containerd first + if (-not ("$env:DOCKER_WINDOWS_CONTAINERD_RUNTIME" -eq "")) { + Start-Process "$env:TEMP\binary\containerd.exe" ` + -ArgumentList "--log-level debug" ` + -RedirectStandardOutput "$env:TEMP\containerd.out" ` + -RedirectStandardError "$env:TEMP\containerd.err" + Write-Host -ForegroundColor Green "INFO: Containerd started successfully." + } + # Cannot fathom why, but always writes to stderr.... Start-Process "$env:TEMP\binary\dockerd-$COMMITHASH" ` -ArgumentList $dutArgs ` @@ -943,6 +974,15 @@ Finally { Copy-Item "$env:TEMP\dut.out" "bundles\CIDUT.out" -Force -ErrorAction SilentlyContinue Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.err) to bundles\CIDUT.err" Copy-Item "$env:TEMP\dut.err" "bundles\CIDUT.err" -Force -ErrorAction SilentlyContinue + + Write-Host -ForegroundColor Green "INFO: Saving containerd logs to bundles" + if (Test-Path -Path "$env:TEMP\containerd.out") { + Copy-Item "$env:TEMP\containerd.out" "bundles\containerd.out" -Force -ErrorAction SilentlyContinue + Copy-Item "$env:TEMP\containerd.err" "bundles\containerd.err" -Force -ErrorAction SilentlyContinue + } else { + "" | Out-File -FilePath "bundles\containerd.out" + "" | Out-File -FilePath "bundles\containerd.err" + } } Set-Location "$env:SOURCES_DRIVE\$env:SOURCES_SUBDIR" -ErrorAction SilentlyContinue diff --git a/integration-cli/docker_api_stats_test.go b/integration-cli/docker_api_stats_test.go index cd614a5e20..bbb37bbd97 100644 --- a/integration-cli/docker_api_stats_test.go +++ b/integration-cli/docker_api_stats_test.go @@ -18,11 +18,13 @@ import ( "github.com/docker/docker/client" "github.com/docker/docker/testutil/request" "gotest.tools/v3/assert" + "gotest.tools/v3/skip" ) var expectedNetworkInterfaceStats = strings.Split("rx_bytes rx_dropped rx_errors rx_packets tx_bytes tx_dropped tx_errors tx_packets", " ") func (s *DockerSuite) TestAPIStatsNoStreamGetCpu(c *testing.T) { + skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination") out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "while true;usleep 100; do echo 'Hello'; done") id := strings.TrimSpace(out) @@ -98,6 +100,7 @@ func (s *DockerSuite) TestAPIStatsStoppedContainerInGoroutines(c *testing.T) { } func (s *DockerSuite) TestAPIStatsNetworkStats(c *testing.T) { + skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination") testRequires(c, testEnv.IsLocalDaemon) out := runSleepingContainer(c) diff --git a/integration-cli/docker_cli_commit_test.go b/integration-cli/docker_cli_commit_test.go index b03708aa01..d5c4b8ca59 100644 --- a/integration-cli/docker_cli_commit_test.go +++ b/integration-cli/docker_cli_commit_test.go @@ -7,9 +7,11 @@ import ( "github.com/docker/docker/api/types/versions" "github.com/docker/docker/integration-cli/cli" "gotest.tools/v3/assert" + "gotest.tools/v3/skip" ) func (s *DockerSuite) TestCommitAfterContainerIsDone(c *testing.T) { + skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination") out := cli.DockerCmd(c, "run", "-i", "-a", "stdin", "busybox", "echo", "foo").Combined() cleanedContainerID := strings.TrimSpace(out) diff --git a/integration-cli/docker_cli_ps_test.go b/integration-cli/docker_cli_ps_test.go index 1677fc00ea..fe625263bc 100644 --- a/integration-cli/docker_cli_ps_test.go +++ b/integration-cli/docker_cli_ps_test.go @@ -227,6 +227,7 @@ func (s *DockerSuite) TestPsListContainersFilterStatus(c *testing.T) { } func (s *DockerSuite) TestPsListContainersFilterHealth(c *testing.T) { + skip.If(c, RuntimeIsWindowsContainerd(), "FIXME. Hang on Windows + containerd combination") existingContainers := ExistingContainerIDs(c) // Test legacy no health check out := runSleepingContainer(c, "--name=none_legacy") diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index c60e7badde..5c6a4d8fdd 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -37,6 +37,7 @@ import ( "github.com/moby/sys/mountinfo" "gotest.tools/v3/assert" "gotest.tools/v3/icmd" + "gotest.tools/v3/skip" ) // "test123" should be printed by docker run @@ -1984,6 +1985,7 @@ func (s *DockerSuite) TestRunCidFileCheckIDLength(c *testing.T) { } func (s *DockerSuite) TestRunSetMacAddress(c *testing.T) { + skip.If(c, RuntimeIsWindowsContainerd(), "FIXME: Broken on Windows + containerd combination") mac := "12:34:56:78:9a:bc" var out string if testEnv.OSType == "windows" { diff --git a/integration-cli/requirements_test.go b/integration-cli/requirements_test.go index 8d5c64fe18..767ad13d56 100644 --- a/integration-cli/requirements_test.go +++ b/integration-cli/requirements_test.go @@ -181,6 +181,10 @@ func RegistryHosting() bool { return err == nil } +func RuntimeIsWindowsContainerd() bool { + return os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME") == "1" +} + func SwarmInactive() bool { return testEnv.DaemonInfo.Swarm.LocalNodeState == swarm.LocalNodeStateInactive } diff --git a/integration/container/exec_test.go b/integration/container/exec_test.go index 0394f52963..26645ba8cc 100644 --- a/integration/container/exec_test.go +++ b/integration/container/exec_test.go @@ -17,6 +17,7 @@ import ( // TestExecWithCloseStdin adds case for moby#37870 issue. func TestExecWithCloseStdin(t *testing.T) { + skip.If(t, testEnv.RuntimeIsWindowsContainerd(), "FIXME. Hang on Windows + containerd combination") skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.39"), "broken in earlier versions") defer setupTest(t)() diff --git a/pkg/system/init_windows.go b/pkg/system/init_windows.go index a91288c60b..3c2a43ddbd 100644 --- a/pkg/system/init_windows.go +++ b/pkg/system/init_windows.go @@ -1,29 +1,18 @@ package system // import "github.com/docker/docker/pkg/system" -import ( - "os" - - "github.com/sirupsen/logrus" -) - var ( - // containerdRuntimeSupported determines if ContainerD should be the runtime. - // As of March 2019, this is an experimental feature. + // containerdRuntimeSupported determines if containerd should be the runtime. containerdRuntimeSupported = false ) -// InitContainerdRuntime sets whether to use ContainerD for runtime -// on Windows. This is an experimental feature still in development, and -// also requires an environment variable to be set (so as not to turn the -// feature on from simply experimental which would also mean LCOW. -func InitContainerdRuntime(experimental bool, cdPath string) { - if experimental && len(cdPath) > 0 && len(os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME")) > 0 { - logrus.Warnf("Using ContainerD runtime. This feature is experimental") +// InitContainerdRuntime sets whether to use containerd for runtime on Windows. +func InitContainerdRuntime(cdPath string) { + if len(cdPath) > 0 { containerdRuntimeSupported = true } } -// ContainerdRuntimeSupported returns true if the use of ContainerD runtime is supported. +// ContainerdRuntimeSupported returns true if the use of containerd runtime is supported. func ContainerdRuntimeSupported() bool { return containerdRuntimeSupported } diff --git a/testutil/environment/environment.go b/testutil/environment/environment.go index 23ab20451b..8af8ca8d6f 100644 --- a/testutil/environment/environment.go +++ b/testutil/environment/environment.go @@ -162,6 +162,11 @@ func (e *Execution) IsUserNamespace() bool { return root != "" } +// RuntimeIsWindowsContainerd returns whether containerd runtime is used on Windows +func (e *Execution) RuntimeIsWindowsContainerd() bool { + return os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME") == "1" +} + // IsRootless returns whether the rootless mode is enabled func (e *Execution) IsRootless() bool { return os.Getenv("DOCKER_ROOTLESS") != ""