kotovalexarian-likes-github
/
moby--moby
mirror of https://github.com/moby/moby.git
1
0
Fork 0
Code Releases Activity

integ-cli: Implement remote FakeStorage server for build via URL tests 

Implemented a FakeStorage alternative that supports spinning
up a remote container on DOCKER_TEST_HOST to serve files over
an offline-compiled Go static web server image so that tests which
use URLs in Dockerfile can build them over at the daemon side.

`fakeStorage` function now automatically chooses if it should
use a local httptest.Server or a remote container.

This fixes the following tests when running against a remote
daemon:

- `TestBuildCacheADD`
- `TestBuildCopyWildcardNoFind`
- `TestBuildCopyWildcardCache`
- `TestBuildADDRemoteFileWithCache`
- `TestBuildADDRemoteFileWithoutCache`
- `TestBuildADDRemoteFileMTime`
- `TestBuildADDLocalAndRemoteFilesWithCache`
- `TestBuildADDLocalAndRemoteFilesWithoutCache`
- `TestBuildFromURLWithF`
- `TestBuildApiDockerFileRemote`

Signed-off-by: Ahmet Alp Balkan <ahmetalpbalkan@gmail.com>
This commit is contained in:
Ahmet Alp Balkan 2015-02-19 02:01:27 -08:00
parent 6f0733a9e3
commit 2e95bb5f1a
7 changed files with 179 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/httpserver/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM busybox
EXPOSE 80/tcp
COPY httpserver .
CMD ["./httpserver"]

12
contrib/httpserver/server.go Normal file
View File

@ -0,0 +1,12 @@
package main
import (
"log"
"net/http"
)
func main() {
fs := http.FileServer(http.Dir("/static"))
http.Handle("/", fs)
log.Panic(http.ListenAndServe(":80", nil))
}

2
integration-cli/docker_api_containers_test.go
View File

@ -364,7 +364,7 @@ RUN find /tmp/`,
}
defer server.Close()
buf, err := sockRequestRaw("POST", "/build?dockerfile=baz&remote="+server.URL+"/testD", nil, "application/json")
buf, err := sockRequestRaw("POST", "/build?dockerfile=baz&remote="+server.URL()+"/testD", nil, "application/json")
if err != nil {
t.Fatalf("Build failed: %s", err)
}

62
integration-cli/docker_cli_build_test.go
View File

@ -8,14 +8,12 @@ import (
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strconv"
"strings"
"syscall"
"testing"
"text/template"
"time"
@ -645,9 +643,10 @@ func TestBuildCacheADD(t *testing.T) {
t.Fatal(err)
}
defer server.Close()
if _, err := buildImage(name,
fmt.Sprintf(`FROM scratch
ADD %s/robots.txt /`, server.URL),
ADD %s/robots.txt /`, server.URL()),
true); err != nil {
t.Fatal(err)
}
@ -657,7 +656,7 @@ func TestBuildCacheADD(t *testing.T) {
deleteImages(name)
_, out, err := buildImageWithOut(name,
fmt.Sprintf(`FROM scratch
ADD %s/index.html /`, server.URL),
ADD %s/index.html /`, server.URL()),
true)
if err != nil {
t.Fatal(err)
@ -797,7 +796,7 @@ RUN [ $(ls -l /exists/test_file4 | awk '{print $3":"$4}') = 'root:root' ]
RUN [ $(ls -l /exists/robots.txt | awk '{print $3":"$4}') = 'root:root' ]
RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
`, server.URL),
`, server.URL()),
map[string]string{
"test_file1": "test1",
"test_file2": "test2",
@ -1084,6 +1083,7 @@ func TestBuildCopyWildcard(t *testing.T) {
t.Fatal(err)
}
defer server.Close()
ctx, err := fakeContext(fmt.Sprintf(`FROM busybox
COPY file*.txt /tmp/
RUN ls /tmp/file1.txt /tmp/file2.txt
@ -1093,7 +1093,7 @@ func TestBuildCopyWildcard(t *testing.T) {
RUN mkdir /tmp2
ADD dir/*dir %s/robots.txt /tmp2/
RUN ls /tmp2/nest_nest_file /tmp2/robots.txt
`, server.URL),
`, server.URL()),
map[string]string{
"file1.txt": "test1",
"file2.txt": "test2",
@ -2831,10 +2831,11 @@ func TestBuildADDRemoteFileWithCache(t *testing.T) {
t.Fatal(err)
}
defer server.Close()
id1, err := buildImage(name,
fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD %s/baz /usr/lib/baz/quux`, server.URL),
ADD %s/baz /usr/lib/baz/quux`, server.URL()),
true)
if err != nil {
t.Fatal(err)
@ -2842,7 +2843,7 @@ func TestBuildADDRemoteFileWithCache(t *testing.T) {
id2, err := buildImage(name,
fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD %s/baz /usr/lib/baz/quux`, server.URL),
ADD %s/baz /usr/lib/baz/quux`, server.URL()),
true)
if err != nil {
t.Fatal(err)
@ -2864,10 +2865,11 @@ func TestBuildADDRemoteFileWithoutCache(t *testing.T) {
t.Fatal(err)
}
defer server.Close()
id1, err := buildImage(name,
fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD %s/baz /usr/lib/baz/quux`, server.URL),
ADD %s/baz /usr/lib/baz/quux`, server.URL()),
true)
if err != nil {
t.Fatal(err)
@ -2875,7 +2877,7 @@ func TestBuildADDRemoteFileWithoutCache(t *testing.T) {
id2, err := buildImage(name2,
fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD %s/baz /usr/lib/baz/quux`, server.URL),
ADD %s/baz /usr/lib/baz/quux`, server.URL()),
false)
if err != nil {
t.Fatal(err)
@ -2894,7 +2896,8 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
defer deleteImages(name, name2, name3, name4)
server, err := fakeStorage(map[string]string{"baz": "hello"})
files := map[string]string{"baz": "hello"}
server, err := fakeStorage(files)
if err != nil {
t.Fatal(err)
}
@ -2902,7 +2905,7 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
ctx, err := fakeContext(fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD %s/baz /usr/lib/baz/quux`, server.URL), nil)
ADD %s/baz /usr/lib/baz/quux`, server.URL()), nil)
if err != nil {
t.Fatal(err)
}
@ -2921,15 +2924,26 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
t.Fatal("The cache should have been used but wasn't - #1")
}
// Now set baz's times to anything else and redo the build
// Now create a different server withsame contents (causes different mtim)
// This time the cache should not be used
bazPath := path.Join(server.FakeContext.Dir, "baz")
err = syscall.UtimesNano(bazPath, make([]syscall.Timespec, 2))
if err != nil {
t.Fatalf("Error setting mtime on %q: %v", bazPath, err)
}
id3, err := buildImageFromContext(name3, ctx, true)
// allow some time for clock to pass as mtime precision is only 1s
time.Sleep(2 * time.Second)
server2, err := fakeStorage(files)
if err != nil {
t.Fatal(err)
}
defer server2.Close()
ctx2, err := fakeContext(fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD %s/baz /usr/lib/baz/quux`, server2.URL()), nil)
if err != nil {
t.Fatal(err)
}
defer ctx2.Close()
id3, err := buildImageFromContext(name3, ctx2, true)
if err != nil {
t.Fatal(err)
}
@ -2938,7 +2952,7 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
}
// And for good measure do it again and make sure cache is used this time
id4, err := buildImageFromContext(name4, ctx, true)
id4, err := buildImageFromContext(name4, ctx2, true)
if err != nil {
t.Fatal(err)
}
@ -2958,10 +2972,11 @@ func TestBuildADDLocalAndRemoteFilesWithCache(t *testing.T) {
t.Fatal(err)
}
defer server.Close()
ctx, err := fakeContext(fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD foo /usr/lib/bla/bar
ADD %s/baz /usr/lib/baz/quux`, server.URL),
ADD %s/baz /usr/lib/baz/quux`, server.URL()),
map[string]string{
"foo": "hello world",
})
@ -3047,10 +3062,11 @@ func TestBuildADDLocalAndRemoteFilesWithoutCache(t *testing.T) {
t.Fatal(err)
}
defer server.Close()
ctx, err := fakeContext(fmt.Sprintf(`FROM scratch
MAINTAINER dockerio
ADD foo /usr/lib/bla/bar
ADD %s/baz /usr/lib/baz/quux`, server.URL),
ADD %s/baz /usr/lib/baz/quux`, server.URL()),
map[string]string{
"foo": "hello world",
})
@ -4773,7 +4789,7 @@ RUN echo from Dockerfile`,
// Make sure that -f is ignored and that we don't use the Dockerfile
// that's in the current dir
out, _, err := dockerCmdInDir(t, ctx.Dir, "build", "-f", "baz", "-t", "test1", server.URL+"/baz")
out, _, err := dockerCmdInDir(t, ctx.Dir, "build", "-f", "baz", "-t", "test1", server.URL()+"/baz")
if err != nil {
t.Fatalf("Failed to build: %s\n%s", out, err)
}

130
integration-cli/docker_utils.go
View File

@ -581,17 +581,42 @@ func fakeContext(dockerfile string, files map[string]string) (*FakeContext, erro
return ctx, nil
}
type FakeStorage struct {
// FakeStorage is a static file server. It might be running locally or remotely
// on test host.
type FakeStorage interface {
Close() error
URL() string
CtxDir() string
}
// fakeStorage returns either a local or remote (at daemon machine) file server
func fakeStorage(files map[string]string) (FakeStorage, error) {
if isLocalDaemon {
return newLocalFakeStorage(files)
}
return newRemoteFileServer(files)
}
// localFileStorage is a file storage on the running machine
type localFileStorage struct {
*FakeContext
*httptest.Server
}
func (f *FakeStorage) Close() error {
f.Server.Close()
return f.FakeContext.Close()
func (s *localFileStorage) URL() string {
return s.Server.URL
}
func fakeStorage(files map[string]string) (*FakeStorage, error) {
func (s *localFileStorage) CtxDir() string {
return s.FakeContext.Dir
}
func (s *localFileStorage) Close() error {
defer s.Server.Close()
return s.FakeContext.Close()
}
func newLocalFakeStorage(files map[string]string) (*localFileStorage, error) {
tmp, err := ioutil.TempDir("", "fake-storage")
if err != nil {
return nil, err
@ -605,40 +630,99 @@ func fakeStorage(files map[string]string) (*FakeStorage, error) {
}
handler := http.FileServer(http.Dir(ctx.Dir))
server := httptest.NewServer(handler)
return &FakeStorage{
return &localFileStorage{
FakeContext: ctx,
Server: server,
}, nil
}
func inspectField(name, field string) (string, error) {
format := fmt.Sprintf("{{.%s}}", field)
// remoteFileServer is a containerized static file server started on the remote
// testing machine to be used in URL-accepting docker build functionality.
type remoteFileServer struct {
host string // hostname/port web server is listening to on docker host e.g. 0.0.0.0:43712
container string
image string
ctx *FakeContext
}
func (f *remoteFileServer) URL() string {
u := url.URL{
Scheme: "http",
Host: f.host}
return u.String()
}
func (f *remoteFileServer) CtxDir() string {
return f.ctx.Dir
}
func (f *remoteFileServer) Close() error {
defer func() {
if f.ctx != nil {
f.ctx.Close()
}
if f.image != "" {
deleteImages(f.image)
}
}()
if f.container == "" {
return nil
}
return deleteContainer(f.container)
}
func newRemoteFileServer(files map[string]string) (*remoteFileServer, error) {
var (
image = fmt.Sprintf("fileserver-img-%s", strings.ToLower(makeRandomString(10)))
container = fmt.Sprintf("fileserver-cnt-%s", strings.ToLower(makeRandomString(10)))
)
// Build the image
ctx, err := fakeContext(`FROM httpserver
COPY . /static`, files)
if _, err := buildImageFromContext(image, ctx, false); err != nil {
return nil, fmt.Errorf("failed building file storage container image: %v", err)
}
// Start the container
runCmd := exec.Command(dockerBinary, "run", "-d", "-P", "--name", container, image)
if out, ec, err := runCommandWithOutput(runCmd); err != nil {
return nil, fmt.Errorf("failed to start file storage container. ec=%v\nout=%s\nerr=%v", ec, out, err)
}
// Find out the system assigned port
out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "port", container, "80/tcp"))
if err != nil {
return nil, fmt.Errorf("failed to find container port: err=%v\nout=%s", err, out)
}
return &remoteFileServer{
container: container,
image: image,
host: strings.Trim(out, "\n"),
ctx: ctx}, nil
}
func inspectFilter(name, filter string) (string, error) {
format := fmt.Sprintf("{{%s}}", filter)
inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
out, exitCode, err := runCommandWithOutput(inspectCmd)
if err != nil || exitCode != 0 {
return "", fmt.Errorf("failed to inspect %s: %s", name, out)
return "", fmt.Errorf("failed to inspect container %s: %s", name, out)
}
return strings.TrimSpace(out), nil
}
func inspectField(name, field string) (string, error) {
return inspectFilter(name, fmt.Sprintf(".%s", field))
}
func inspectFieldJSON(name, field string) (string, error) {
format := fmt.Sprintf("{{json .%s}}", field)
inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
out, exitCode, err := runCommandWithOutput(inspectCmd)
if err != nil || exitCode != 0 {
return "", fmt.Errorf("failed to inspect %s: %s", name, out)
}
return strings.TrimSpace(out), nil
return inspectFilter(name, fmt.Sprintf("json .%s", field))
}
func inspectFieldMap(name, path, field string) (string, error) {
format := fmt.Sprintf("{{index .%s %q}}", path, field)
inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
out, exitCode, err := runCommandWithOutput(inspectCmd)
if err != nil || exitCode != 0 {
return "", fmt.Errorf("failed to inspect %s: %s", name, out)
}
return strings.TrimSpace(out), nil
return inspectFilter(name, fmt.Sprintf("index .%s %q", path, field))
}
func getIDByName(name string) (string, error) {

15
project/make/.ensure-httpserver Normal file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -e
# Build a Go static web server on top of busybox image
# and compile it for target daemon
dir="$DEST/httpserver"
mkdir -p "$dir"
(
cd "$dir"
GOOS=linux GOARCH=amd64 go build -o httpserver github.com/docker/docker/contrib/httpserver
cp ../../../../contrib/httpserver/Dockerfile .
docker build -qt httpserver . > /dev/null
)
rm -rf "$dir"

1
project/make/test-integration-cli
View File

@ -17,6 +17,7 @@ bundle_test_integration_cli() {
didFail=
if ! {
source "$(dirname "$BASH_SOURCE")/.ensure-busybox"
source "$(dirname "$BASH_SOURCE")/.ensure-httpserver"
source "$(dirname "$BASH_SOURCE")/.ensure-emptyfs"
bundle_test_integration_cli