1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00

Build and use gotestsum for running all tests

1. Dockerfile.Windows modified to build gotestsum.exe

2. Use gotestsum.exe in invoking the execution of:

   (a) Unit tests (run in containers),
   (b) Integration tests (run outside containers)
   (c) Integration-cli (run outside containers)

No changes made to other categories of tests (e.g.
LCOW).

3. Copy .xml files produced by gotestsum in
   appropriate paths where Jenkins can ingest them

4. Modify Jenkinsfile to mark results output as
   being jUnit "type" as well as to archive the
   .xml test result files as artifacts.

Signed-off-by: Vikram bir Singh <vikrambir.singh@docker.com>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Vikram bir Singh 2019-10-08 01:14:51 +00:00 committed by Sebastiaan van Stijn
parent 5f83f15ecb
commit 8d2e1ee546
No known key found for this signature in database
GPG key ID: 76698F39D527CE8C
4 changed files with 120 additions and 24 deletions

View file

@ -166,6 +166,7 @@ FROM microsoft/windowsservercore
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
ARG GO_VERSION=1.13.4
ARG GOTESTSUM_COMMIT=v0.3.5
# Environment variable notes:
# - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
@ -174,7 +175,8 @@ ENV GO_VERSION=${GO_VERSION} `
GIT_VERSION=2.11.1 `
GOPATH=C:\gopath `
GO111MODULE=off `
FROM_DOCKERFILE=1
FROM_DOCKERFILE=1 `
GOTESTSUM_COMMIT=${GOTESTSUM_COMMIT}
RUN `
Function Test-Nano() { `
@ -249,13 +251,46 @@ RUN `
Remove-Item C:\binutils.zip; `
Remove-Item C:\gitsetup.zip; `
`
Write-Host INFO: Creating source directory...; `
New-Item -ItemType Directory -Path ${GOPATH}\src\github.com\docker\docker | Out-Null; `
# Ensure all directories exist that we will require below....
$srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; `
Write-Host INFO: Ensuring existence of directory $srcDir...; `
New-Item -Force -ItemType Directory -Path $srcDir | Out-Null; `
`
Write-Host INFO: Configuring git core.autocrlf...; `
C:\git\cmd\git config --global core.autocrlf true; `
C:\git\cmd\git config --global core.autocrlf true;
RUN `
Function Build-GoTestSum() { `
Write-Host "INFO: Building gotestsum version $Env:GOTESTSUM_COMMIT in $Env:GOPATH"; `
$optsForGet = @('"get"', '"-d"', '"gotest.tools/gotestsum"'); `
&go $optsForGet; `
$savedExitCode = $LASTEXITCODE; `
if ($savedExitCode -ne 0) { `
Throw '"Failed getting gotestsum sources..."' `
}; `
Write-Host "INFO: Sources obtained for gotestsum..."; `
$GotestsumPath=Join-Path -Path $Env:GOPATH -ChildPath "src\gotest.tools\gotestsum"; `
Push-Location $GotestsumPath; `
$optsForCheckout = @('"checkout"', '"-q"', """$Env:GOTESTSUM_COMMIT"""); `
&git $optsForCheckout; `
$savedExitCode = $LASTEXITCODE; `
if ($savedExitCode -eq 0) { `
Write-Host "INFO: Checkout done for gotestsum..."; `
$optsForBuild = @('"build"', '"-buildmode=exe"'); `
&go $optsForBuild; `
$savedExitCode = $LASTEXITCODE; `
} else { `
Throw '"gotestsum checkout failed..."'; `
} `
Pop-Location; `
`
if ($savedExitCode -ne 0) { `
Throw '"gotestsum build failed..."'; `
} `
Write-Host "INFO: Build done for gotestsum..."; `
} `
`
Write-Host INFO: Completed
Build-GoTestSum
# Make PowerShell the default entrypoint
ENTRYPOINT ["powershell.exe"]

6
Jenkinsfile vendored
View file

@ -928,13 +928,14 @@ pipeline {
}
post {
always {
junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
powershell '''
$bundleName="windowsRS1-integration"
Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
# archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/testresults/unittests/junit-report-unit-tests.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
'''
archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
@ -988,13 +989,14 @@ pipeline {
}
post {
always {
junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
powershell '''
$bundleName="windowsRS5-integration"
Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
# archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
'''
archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true

View file

@ -447,6 +447,8 @@ Try {
$errorActionPreference='Stop'
New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null
New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue | Out-Null
New-Item -ItemType Directory "$env:TEMP\testresults" -ErrorAction SilentlyContinue | Out-Null
New-Item -ItemType Directory "$env:TEMP\testresults\unittests" -ErrorAction SilentlyContinue | Out-Null
New-Item -ItemType Directory "$env:TEMP\localappdata" -ErrorAction SilentlyContinue | Out-Null
New-Item -ItemType Directory "$env:TEMP\binary" -ErrorAction SilentlyContinue | Out-Null
New-Item -ItemType Directory "$env:TEMP\installer" -ErrorAction SilentlyContinue | Out-Null
@ -525,6 +527,16 @@ Try {
if (-not($LastExitCode -eq 0)) {
Throw "ERROR: Failed to docker cp the daemon binary (dockerd.exe) to $env:TEMP\binary"
}
$GotestsumBinRelLocationInContainer="src\gotest.tools\gotestsum\gotestsum.exe"
if (-not($LastExitCode -eq 0)) {
Throw "ERROR: Failed to docker cp the gotestsum binary (gotestsum.exe) to $env:TEMP\binary"
}
docker cp "$COMMITHASH`:c`:\gopath\$GotestsumBinRelLocationInContainer" $env:TEMP\binary\
if (-not (Test-Path "$env:TEMP\binary\gotestsum.exe")) {
Throw "ERROR: gotestsum.exe not found...." `
}
$ErrorActionPreference = "Stop"
# Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager.
@ -774,11 +786,36 @@ Try {
# Run the unit tests inside a container unless SKIP_UNIT_TESTS is defined
if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
if ($null -eq $env:SKIP_UNIT_TESTS) {
$ContainerNameForUnitTests = $COMMITHASH + "_UnitTests"
Write-Host -ForegroundColor Cyan "INFO: Running unit tests at $(Get-Date)..."
$ErrorActionPreference = "SilentlyContinue"
$Duration=$(Measure-Command {docker run -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
$Duration=$(Measure-Command {docker run --name $ContainerNameForUnitTests -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
$TestRunExitCode = $LastExitCode
$ErrorActionPreference = "Stop"
# Saving for artifacts......
$unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles"
$JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml"
docker cp $JunitExpectedContFilePath "$TEMPORIG"
if (-not($LastExitCode -eq 0)) {
Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $TEMPORIG"
}
if (Test-Path "$TEMPORIG\junit-report-unit-tests.xml") {
Write-Host -ForegroundColor Magenta "INFO: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) exist. pwd=$pwd"
} else {
Write-Host -ForegroundColor Magenta "ERROR: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
}
# Saving where jenkins will take a look at.....
$bundlesDir = "bundles"
New-Item -Force -ItemType Directory $bundlesDir | Out-Null
docker cp $JunitExpectedContFilePath "$bundlesDir"
if (-not($LastExitCode -eq 0)) {
Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $bundlesDir"
}
if (-not($TestRunExitCode -eq 0)) {
Throw "ERROR: Unit tests failed"
}
Write-Host -ForegroundColor Green "INFO: Unit tests ended at $(Get-Date). Duration`:$Duration"
@ -830,7 +867,9 @@ Try {
$env:OrigDOCKER_HOST="$env:DOCKER_HOST"
#https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
$c = "go test "
$jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json"
$xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml"
$c = "gotestsum --format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
$c += "`"-test.v`" "
if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
$c += "`"-test.run`" "
@ -864,10 +903,12 @@ Try {
if (!($env:INTEGRATION_TESTFLAGS)) {
$env:INTEGRATION_TESTFLAGS = "-test.v"
}
Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
$start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
$IntTestsRunResult = $LastExitCode
$ErrorActionPreference = "Stop"
if (-not($LastExitCode -eq 0)) {
# Copy all the test results to TEMPORIG for archival
Copy-Item -Path "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\bundles\junit-report*xml" -Destination $TEMPORIG
if (-not($IntTestsRunResult -eq 0)) {
Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
}
@ -877,6 +918,7 @@ Try {
Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
# Explicit to not use measure-command otherwise don't get output as it goes
$start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
Copy-Item -Path $xmlFilePath -Destination $TEMPORIG
}
$ErrorActionPreference = "Stop"
if (-not($LastExitCode -eq 0)) {

View file

@ -94,6 +94,7 @@ param(
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$pushed=$False # To restore the directory if we have temporarily pushed to one.
Set-Variable GOTESTSUM_LOCATION -option Constant -value "$env:GOPATH/src/gotest.tools/gotestsum"
# Utility function to get the commit ID of the repository
Function Get-GitCommit() {
@ -319,31 +320,42 @@ Function Run-UnitTests() {
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
$pkgList = $pkgList -replace "`r`n", " "
$goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
$goTestCommand = "$GOTESTSUM_LOCATION\gotestsum.exe --format=standard-quiet --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
Write-Host "INFO: Invoking unit tests run with $goTestCommand"
Invoke-Expression $goTestCommand
if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
}
# Run the integration tests
Function Run-IntegrationTests() {
$env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
$escRoot = [Regex]::Escape($root)
$env:DOCKER_INTEGRATION_DAEMON_DEST = $bundlesDir + "\tmp"
$dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
$integration_api_dirs = @()
ForEach($dir in $dirs) {
$integration_api_dirs += $dir
Write-Host "Building test suite binary $dir"
go test -c -o "$dir\test.exe" $dir
}
ForEach($dir in $integration_api_dirs) {
# Normalize directory name for using in the test results files.
$normDir = $dir.Trim()
$normDir = $normDir -replace $escRoot, ""
$normDir = $normDir -replace "\\", "-"
$normDir = $normDir -replace "\/", "-"
$normDir = $normDir -replace "\.", "-"
if ($normDir.StartsWith("-"))
{
$normDir = $normDir.TrimStart("-")
}
if ($normDir.EndsWith("-"))
{
$normDir = $normDir.TrimEnd("-")
}
$jsonFilePath = $bundlesDir + "\go-test-report-int-tests-$normDir" + ".json"
$xmlFilePath = $bundlesDir + "\junit-report-int-tests-$normDir" + ".xml"
Set-Location $dir
Write-Host "Running $($PWD.Path)"
$pinfo = New-Object System.Diagnostics.ProcessStartInfo
$pinfo.FileName = "$($PWD.Path)\test.exe"
$pinfo.FileName = "gotestsum.exe"
$pinfo.WorkingDirectory = "$($PWD.Path)"
$pinfo.RedirectStandardError = $true
$pinfo.UseShellExecute = $false
$pinfo.Arguments = $env:INTEGRATION_TESTFLAGS
$pinfo.Arguments = "--format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS"
$p = New-Object System.Diagnostics.Process
$p.StartInfo = $pinfo
$p.Start() | Out-Null
@ -351,6 +363,8 @@ Function Run-IntegrationTests() {
$err = $p.StandardError.ReadToEnd()
if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) {
Throw "Integration tests failed: $err"
} else {
Write-Host "$err"
}
}
}
@ -363,6 +377,11 @@ Try {
$root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
Push-Location $root
# Ensure the bundles directory exists
$bundlesDir = $root + "\bundles"
Set-Variable bundlesDir -option ReadOnly
New-Item -Force $bundlesDir -ItemType Directory | Out-Null
# Handle the "-All" shortcut to turn on all things we can handle.
# Note we expressly only include the items which can run in a container - the validations tests cannot
# as they require the .git directory which is excluded from the image by .dockerignore
@ -424,8 +443,6 @@ Try {
# Build the binaries
if ($Client -or $Daemon) {
# Create the bundles directory if it doesn't exist
if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
# Perform the actual build
if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }