builder: add support for building from tarball

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

api/server/backend/build/backend.go

@@ -3,7 +3,6 @@ package build // import "github.com/docker/docker/api/server/backend/build"
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
@@ -44,11 +43,7 @@ func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSC
 // Build builds an image from a Source
 func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
 	options := config.Options
-	useBuildKit := false
+	useBuildKit := options.Version == types.BuilderBuildKit
-	if strings.HasPrefix(options.SessionID, "buildkit:") {
-		useBuildKit = true
-		options.SessionID = strings.TrimPrefix(options.SessionID, "buildkit:")
-	}
 
 	tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
 	if err != nil {

api/server/router/build/build_routes.go

@@ -146,10 +146,25 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	}
 	options.SessionID = r.FormValue("session")
 	options.BuildID = r.FormValue("buildid")
+	builderVersion, err := parseVersion(r.FormValue("version"))
+	if err != nil {
+		return nil, err
+	}
+	options.Version = builderVersion
 
 	return options, nil
 }
 
+func parseVersion(s string) (types.BuilderVersion, error) {
+	if s == "" || s == string(types.BuilderV1) {
+		return types.BuilderV1, nil
+	}
+	if s == string(types.BuilderBuildKit) {
+		return types.BuilderBuildKit, nil
+	}
+	return "", errors.Errorf("invalid version %s", s)
+}
+
 func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	report, err := br.backend.PruneCache(ctx)
 	if err != nil {

api/types/client.go

@@ -181,9 +181,18 @@ type ImageBuildOptions struct {
 	Target      string
 	SessionID   string
 	Platform    string
+	Version     BuilderVersion
 	BuildID     string
 }
 
+// BuilderVersion sets the version of underlying builder to use
+type BuilderVersion string
+
+const (
+	BuilderV1       BuilderVersion = "1"
+	BuilderBuildKit                = "2"
+)
+
 // ImageBuildResponse holds information
 // returned by a server after building
 // an image.

builder/builder-next/builder.go

@@ -17,6 +17,7 @@ import (
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/util/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 	grpcmetadata "google.golang.org/grpc/metadata"
@@ -29,20 +30,24 @@ type Opt struct {
 }
 
 type Builder struct {
-	controller *control.Controller
+	controller     *control.Controller
+	reqBodyHandler *reqBodyHandler
 
 	mu   sync.Mutex
 	jobs map[string]func()
 }
 
 func New(opt Opt) (*Builder, error) {
-	c, err := newController(opt)
+	reqHandler := newReqBodyHandler(tracing.DefaultTransport)
+
+	c, err := newController(reqHandler, opt)
 	if err != nil {
 		return nil, err
 	}
 	b := &Builder{
-		controller: c,
+		controller:     c,
-		jobs:       map[string]func(){},
+		reqBodyHandler: reqHandler,
+		jobs:           map[string]func(){},
 	}
 	return b, nil
 }
@@ -133,7 +138,13 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
 	}
 
 	if opt.Options.RemoteContext != "" {
-		frontendAttrs["context"] = opt.Options.RemoteContext
+		if opt.Options.RemoteContext != "client-session" {
+			frontendAttrs["context"] = opt.Options.RemoteContext
+		}
+	} else {
+		url, cancel := b.reqBodyHandler.newRequest(opt.Source)
+		defer cancel()
+		frontendAttrs["context"] = url
 	}
 
 	var cacheFrom []string

builder/builder-next/controller.go

@@ -1,6 +1,7 @@
 package buildkit
 
 import (
+	"net/http"
 	"os"
 	"path/filepath"
 
@@ -24,7 +25,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-func newController(opt Opt) (*control.Controller, error) {
+func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 	if err := os.MkdirAll(opt.Root, 0700); err != nil {
 		return nil, err
 	}
@@ -133,6 +134,7 @@ func newController(opt Opt) (*control.Controller, error) {
 		Exporters: map[string]exporter.Exporter{
 			"moby": exp,
 		},
+		Transport: rt,
 	}
 
 	wc := &worker.Controller{}

builder/builder-next/reqbodyhandler.go

@@ -0,0 +1,74 @@
+package buildkit
+
+import (
+	"bufio"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/moby/buildkit/identity"
+	"github.com/pkg/errors"
+)
+
+const urlPrefix = "build-context-"
+
+type reqBodyHandler struct {
+	mu sync.Mutex
+	rt http.RoundTripper
+
+	requests map[string]io.ReadCloser
+}
+
+func newReqBodyHandler(rt http.RoundTripper) *reqBodyHandler {
+	return &reqBodyHandler{
+		rt:       rt,
+		requests: map[string]io.ReadCloser{},
+	}
+}
+
+func (h *reqBodyHandler) newRequest(rc io.ReadCloser) (string, func()) {
+	// handle expect-continue vs chunked output
+	r := bufio.NewReader(rc)
+	r.Peek(1)
+	id := identity.NewID()
+	h.mu.Lock()
+	h.requests[id] = &readCloser{Reader: r, Closer: rc}
+	h.mu.Unlock()
+	return "http://" + urlPrefix + id, func() {
+		h.mu.Lock()
+		delete(h.requests, id)
+		h.mu.Unlock()
+	}
+}
+
+func (h *reqBodyHandler) RoundTrip(req *http.Request) (*http.Response, error) {
+	host := req.URL.Host
+	if strings.HasPrefix(host, urlPrefix) {
+		if req.Method != "GET" {
+			return nil, errors.Errorf("invalid request")
+		}
+		id := strings.TrimPrefix(host, urlPrefix)
+		h.mu.Lock()
+		rc, ok := h.requests[id]
+		delete(h.requests, id)
+		h.mu.Unlock()
+
+		if !ok {
+			return nil, errors.Errorf("context not found")
+		}
+
+		return &http.Response{
+			Status:        "200 OK",
+			StatusCode:    200,
+			Body:          rc,
+			ContentLength: -1,
+		}, nil
+	}
+	return h.rt.RoundTrip(req)
+}
+
+type readCloser struct {
+	io.Reader
+	io.Closer
+}

builder/builder-next/worker/worker.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	nethttp "net/http"
 	"runtime"
 	"time"
 
@@ -53,6 +54,7 @@ type WorkerOpt struct {
 	Exporters         map[string]exporter.Exporter
 	DownloadManager   distribution.RootFSDownloadManager
 	V2MetadataService distmetadata.V2MetadataService
+	Transport         nethttp.RoundTripper
 }
 
 // Worker is a local worker instance with dedicated snapshotter, cache, and so on.
@@ -85,6 +87,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
 	hs, err := http.NewSource(http.Opt{
 		CacheAccessor: cm,
 		MetadataStore: opt.MetadataStore,
+		Transport:     opt.Transport,
 	})
 	if err != nil {
 		return nil, err
@@ -125,7 +128,7 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solve
 	case *pb.Op_Source:
 		return ops.NewSourceOp(v, op, w.SourceManager, w)
 	case *pb.Op_Exec:
-		return ops.NewExecOp(v, op, w.CacheManager, w.Executor, w)
+		return ops.NewExecOp(v, op, w.CacheManager, w.MetadataStore, w.Executor, w)
 	case *pb.Op_Build:
 		return ops.NewBuildOp(v, op, s, w)
 	default:

client/image_build.go

@@ -136,5 +136,6 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur
 	if options.BuildID != "" {
 		query.Set("buildid", options.BuildID)
 	}
+	query.Set("version", string(options.Version))
 	return query, nil
 }