From 92395261b05610ca0a8ae799220ce29f8ee00fac Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 17 May 2018 22:47:34 -0700 Subject: [PATCH] builder: add support for building from tarball Signed-off-by: Tonis Tiigi --- api/server/backend/build/backend.go | 7 +-- api/server/router/build/build_routes.go | 15 +++++ api/types/client.go | 9 +++ builder/builder-next/builder.go | 21 +++++-- builder/builder-next/controller.go | 4 +- builder/builder-next/reqbodyhandler.go | 74 +++++++++++++++++++++++++ builder/builder-next/worker/worker.go | 5 +- client/image_build.go | 1 + 8 files changed, 123 insertions(+), 13 deletions(-) create mode 100644 builder/builder-next/reqbodyhandler.go diff --git a/api/server/backend/build/backend.go b/api/server/backend/build/backend.go index ae5b9c72f9..3415cfbd8e 100644 --- a/api/server/backend/build/backend.go +++ b/api/server/backend/build/backend.go @@ -3,7 +3,6 @@ package build // import "github.com/docker/docker/api/server/backend/build" import ( "context" "fmt" - "strings" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" @@ -44,11 +43,7 @@ func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSC // Build builds an image from a Source func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) { options := config.Options - useBuildKit := false - if strings.HasPrefix(options.SessionID, "buildkit:") { - useBuildKit = true - options.SessionID = strings.TrimPrefix(options.SessionID, "buildkit:") - } + useBuildKit := options.Version == types.BuilderBuildKit tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags) if err != nil { diff --git a/api/server/router/build/build_routes.go b/api/server/router/build/build_routes.go index 7acef8ba68..11062d69e0 100644 --- a/api/server/router/build/build_routes.go +++ b/api/server/router/build/build_routes.go @@ -146,10 +146,25 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui } options.SessionID = r.FormValue("session") options.BuildID = r.FormValue("buildid") + builderVersion, err := parseVersion(r.FormValue("version")) + if err != nil { + return nil, err + } + options.Version = builderVersion return options, nil } +func parseVersion(s string) (types.BuilderVersion, error) { + if s == "" || s == string(types.BuilderV1) { + return types.BuilderV1, nil + } + if s == string(types.BuilderBuildKit) { + return types.BuilderBuildKit, nil + } + return "", errors.Errorf("invalid version %s", s) +} + func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { report, err := br.backend.PruneCache(ctx) if err != nil { diff --git a/api/types/client.go b/api/types/client.go index 76e589ed84..d825e5ae5c 100644 --- a/api/types/client.go +++ b/api/types/client.go @@ -181,9 +181,18 @@ type ImageBuildOptions struct { Target string SessionID string Platform string + Version BuilderVersion BuildID string } +// BuilderVersion sets the version of underlying builder to use +type BuilderVersion string + +const ( + BuilderV1 BuilderVersion = "1" + BuilderBuildKit = "2" +) + // ImageBuildResponse holds information // returned by a server after building // an image. diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go index 307d355051..141b441fdb 100644 --- a/builder/builder-next/builder.go +++ b/builder/builder-next/builder.go @@ -17,6 +17,7 @@ import ( "github.com/moby/buildkit/control" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/util/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" grpcmetadata "google.golang.org/grpc/metadata" @@ -29,20 +30,24 @@ type Opt struct { } type Builder struct { - controller *control.Controller + controller *control.Controller + reqBodyHandler *reqBodyHandler mu sync.Mutex jobs map[string]func() } func New(opt Opt) (*Builder, error) { - c, err := newController(opt) + reqHandler := newReqBodyHandler(tracing.DefaultTransport) + + c, err := newController(reqHandler, opt) if err != nil { return nil, err } b := &Builder{ - controller: c, - jobs: map[string]func(){}, + controller: c, + reqBodyHandler: reqHandler, + jobs: map[string]func(){}, } return b, nil } @@ -133,7 +138,13 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. } if opt.Options.RemoteContext != "" { - frontendAttrs["context"] = opt.Options.RemoteContext + if opt.Options.RemoteContext != "client-session" { + frontendAttrs["context"] = opt.Options.RemoteContext + } + } else { + url, cancel := b.reqBodyHandler.newRequest(opt.Source) + defer cancel() + frontendAttrs["context"] = url } var cacheFrom []string diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index cda45e88d3..9116065d0c 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -1,6 +1,7 @@ package buildkit import ( + "net/http" "os" "path/filepath" @@ -24,7 +25,7 @@ import ( "github.com/pkg/errors" ) -func newController(opt Opt) (*control.Controller, error) { +func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) { if err := os.MkdirAll(opt.Root, 0700); err != nil { return nil, err } @@ -133,6 +134,7 @@ func newController(opt Opt) (*control.Controller, error) { Exporters: map[string]exporter.Exporter{ "moby": exp, }, + Transport: rt, } wc := &worker.Controller{} diff --git a/builder/builder-next/reqbodyhandler.go b/builder/builder-next/reqbodyhandler.go new file mode 100644 index 0000000000..393e0cf33a --- /dev/null +++ b/builder/builder-next/reqbodyhandler.go @@ -0,0 +1,74 @@ +package buildkit + +import ( + "bufio" + "io" + "net/http" + "strings" + "sync" + + "github.com/moby/buildkit/identity" + "github.com/pkg/errors" +) + +const urlPrefix = "build-context-" + +type reqBodyHandler struct { + mu sync.Mutex + rt http.RoundTripper + + requests map[string]io.ReadCloser +} + +func newReqBodyHandler(rt http.RoundTripper) *reqBodyHandler { + return &reqBodyHandler{ + rt: rt, + requests: map[string]io.ReadCloser{}, + } +} + +func (h *reqBodyHandler) newRequest(rc io.ReadCloser) (string, func()) { + // handle expect-continue vs chunked output + r := bufio.NewReader(rc) + r.Peek(1) + id := identity.NewID() + h.mu.Lock() + h.requests[id] = &readCloser{Reader: r, Closer: rc} + h.mu.Unlock() + return "http://" + urlPrefix + id, func() { + h.mu.Lock() + delete(h.requests, id) + h.mu.Unlock() + } +} + +func (h *reqBodyHandler) RoundTrip(req *http.Request) (*http.Response, error) { + host := req.URL.Host + if strings.HasPrefix(host, urlPrefix) { + if req.Method != "GET" { + return nil, errors.Errorf("invalid request") + } + id := strings.TrimPrefix(host, urlPrefix) + h.mu.Lock() + rc, ok := h.requests[id] + delete(h.requests, id) + h.mu.Unlock() + + if !ok { + return nil, errors.Errorf("context not found") + } + + return &http.Response{ + Status: "200 OK", + StatusCode: 200, + Body: rc, + ContentLength: -1, + }, nil + } + return h.rt.RoundTrip(req) +} + +type readCloser struct { + io.Reader + io.Closer +} diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index e1c5653d33..6aa72c5e5e 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + nethttp "net/http" "runtime" "time" @@ -53,6 +54,7 @@ type WorkerOpt struct { Exporters map[string]exporter.Exporter DownloadManager distribution.RootFSDownloadManager V2MetadataService distmetadata.V2MetadataService + Transport nethttp.RoundTripper } // Worker is a local worker instance with dedicated snapshotter, cache, and so on. @@ -85,6 +87,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) { hs, err := http.NewSource(http.Opt{ CacheAccessor: cm, MetadataStore: opt.MetadataStore, + Transport: opt.Transport, }) if err != nil { return nil, err @@ -125,7 +128,7 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solve case *pb.Op_Source: return ops.NewSourceOp(v, op, w.SourceManager, w) case *pb.Op_Exec: - return ops.NewExecOp(v, op, w.CacheManager, w.Executor, w) + return ops.NewExecOp(v, op, w.CacheManager, w.MetadataStore, w.Executor, w) case *pb.Op_Build: return ops.NewBuildOp(v, op, s, w) default: diff --git a/client/image_build.go b/client/image_build.go index 18a510415a..dff19b989f 100644 --- a/client/image_build.go +++ b/client/image_build.go @@ -136,5 +136,6 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur if options.BuildID != "" { query.Set("buildid", options.BuildID) } + query.Set("version", string(options.Version)) return query, nil }