168 lines
4.3 KiB
Go
168 lines
4.3 KiB
Go
package artifacts
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
"gitlab.com/gitlab-org/labkit/log"
|
|
|
|
"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
|
|
"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
|
|
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
|
|
"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
|
|
"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
|
|
)
|
|
|
|
// Sent by the runner: https://gitlab.com/gitlab-org/gitlab-runner/blob/c24da19ecce8808d9d2950896f70c94f5ea1cc2e/network/gitlab.go#L580
|
|
const (
|
|
ArtifactFormatKey = "artifact_format"
|
|
ArtifactFormatZip = "zip"
|
|
ArtifactFormatDefault = ""
|
|
)
|
|
|
|
var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Name: "gitlab_workhorse_zip_subcommand_errors_total",
|
|
Help: "Errors comming from subcommands used for processing ZIP archives",
|
|
}, []string{"error"})
|
|
|
|
type artifactsUploadProcessor struct {
|
|
opts *filestore.SaveFileOpts
|
|
format string
|
|
|
|
upload.SavedFileTracker
|
|
}
|
|
|
|
func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *filestore.FileHandler) (*filestore.FileHandler, error) {
|
|
metaReader, metaWriter := io.Pipe()
|
|
defer metaWriter.Close()
|
|
|
|
metaOpts := &filestore.SaveFileOpts{
|
|
LocalTempPath: a.opts.LocalTempPath,
|
|
TempFilePrefix: "metadata.gz",
|
|
}
|
|
if metaOpts.LocalTempPath == "" {
|
|
metaOpts.LocalTempPath = os.TempDir()
|
|
}
|
|
|
|
fileName := file.LocalPath
|
|
if fileName == "" {
|
|
fileName = file.RemoteURL
|
|
}
|
|
|
|
zipMd := exec.CommandContext(ctx, "gitlab-zip-metadata", fileName)
|
|
zipMd.Stderr = log.ContextLogger(ctx).Writer()
|
|
zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
zipMd.Stdout = metaWriter
|
|
|
|
if err := zipMd.Start(); err != nil {
|
|
return nil, err
|
|
}
|
|
defer helper.CleanUpProcessGroup(zipMd)
|
|
|
|
type saveResult struct {
|
|
error
|
|
*filestore.FileHandler
|
|
}
|
|
done := make(chan saveResult)
|
|
go func() {
|
|
var result saveResult
|
|
result.FileHandler, result.error = filestore.SaveFileFromReader(ctx, metaReader, -1, metaOpts)
|
|
|
|
done <- result
|
|
}()
|
|
|
|
if err := zipMd.Wait(); err != nil {
|
|
st, ok := helper.ExitStatus(err)
|
|
|
|
if !ok {
|
|
return nil, err
|
|
}
|
|
|
|
zipSubcommandsErrorsCounter.WithLabelValues(zipartifacts.ErrorLabelByCode(st)).Inc()
|
|
|
|
if st == zipartifacts.CodeNotZip {
|
|
return nil, nil
|
|
}
|
|
|
|
if st == zipartifacts.CodeLimitsReached {
|
|
return nil, zipartifacts.ErrBadMetadata
|
|
}
|
|
}
|
|
|
|
metaWriter.Close()
|
|
result := <-done
|
|
return result.FileHandler, result.error
|
|
}
|
|
|
|
func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error {
|
|
// ProcessFile for artifacts requires file form-data field name to eq `file`
|
|
|
|
if formName != "file" {
|
|
return fmt.Errorf("invalid form field: %q", formName)
|
|
}
|
|
if a.Count() > 0 {
|
|
return fmt.Errorf("artifacts request contains more than one file")
|
|
}
|
|
a.Track(formName, file.LocalPath)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("ProcessFile: context done")
|
|
default:
|
|
}
|
|
|
|
if !strings.EqualFold(a.format, ArtifactFormatZip) && a.format != ArtifactFormatDefault {
|
|
return nil
|
|
}
|
|
|
|
// TODO: can we rely on disk for shipping metadata? Not if we split workhorse and rails in 2 different PODs
|
|
metadata, err := a.generateMetadataFromZip(ctx, file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if metadata != nil {
|
|
fields, err := metadata.GitLabFinalizeFields("metadata")
|
|
if err != nil {
|
|
return fmt.Errorf("finalize metadata field error: %v", err)
|
|
}
|
|
|
|
for k, v := range fields {
|
|
writer.WriteField(k, v)
|
|
}
|
|
|
|
a.Track("metadata", metadata.LocalPath)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (a *artifactsUploadProcessor) Name() string {
|
|
return "artifacts"
|
|
}
|
|
|
|
func UploadArtifacts(myAPI *api.API, h http.Handler, p upload.Preparer) http.Handler {
|
|
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
|
|
opts, _, err := p.Prepare(a)
|
|
if err != nil {
|
|
helper.Fail500(w, r, fmt.Errorf("UploadArtifacts: error preparing file storage options"))
|
|
return
|
|
}
|
|
|
|
format := r.URL.Query().Get(ArtifactFormatKey)
|
|
|
|
mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: upload.SavedFileTracker{Request: r}}
|
|
upload.HandleFileUploads(w, r, h, a, mg, opts)
|
|
}, "/authorize")
|
|
}
|