gitlab-org--gitlab-foss/workhorse/internal/upload/uploads_test.go

589 lines
17 KiB
Go

package upload
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"os"
"regexp"
"strconv"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test"
"gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper"
)
var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
type testFormProcessor struct{ SavedFileTracker }
func (a *testFormProcessor) ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error {
if formName != "token" && !strings.HasPrefix(formName, "file.") && !strings.HasPrefix(formName, "other.") {
return fmt.Errorf("illegal field: %v", formName)
}
return nil
}
func (a *testFormProcessor) Finalize(ctx context.Context) error {
return nil
}
func TestUploadTempPathRequirement(t *testing.T) {
apiResponse := &api.Response{}
preparer := &DefaultPreparer{}
_, _, err := preparer.Prepare(apiResponse)
require.Error(t, err)
}
func TestUploadHandlerForwardingRawData(t *testing.T) {
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "PATCH", r.Method, "method")
body, err := ioutil.ReadAll(r.Body)
require.NoError(t, err)
require.Equal(t, "REQUEST", string(body), "request body")
w.WriteHeader(202)
fmt.Fprint(w, "RESPONSE")
})
defer ts.Close()
httpRequest, err := http.NewRequest("PATCH", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
require.NoError(t, err)
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
response := httptest.NewRecorder()
handler := newProxy(ts.URL)
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, handler, apiResponse, nil, opts)
require.Equal(t, 202, response.Code)
require.Equal(t, "RESPONSE", response.Body.String(), "response body")
}
func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
var filePath string
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "PUT", r.Method, "method")
require.NoError(t, r.ParseMultipartForm(100000))
require.Empty(t, r.MultipartForm.File, "Expected to not receive any files")
require.Equal(t, "test", r.FormValue("token"), "Expected to receive token")
require.Equal(t, "my.file", r.FormValue("file.name"), "Expected to receive a filename")
filePath = r.FormValue("file.path")
require.True(t, strings.HasPrefix(filePath, tempPath), "Expected to the file to be in tempPath")
require.Empty(t, r.FormValue("file.remote_url"), "Expected to receive empty remote_url")
require.Empty(t, r.FormValue("file.remote_id"), "Expected to receive empty remote_id")
require.Equal(t, "4", r.FormValue("file.size"), "Expected to receive the file size")
hashes := map[string]string{
"md5": "098f6bcd4621d373cade4e832627b4f6",
"sha1": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
"sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
"sha512": "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff",
}
for algo, hash := range hashes {
require.Equal(t, hash, r.FormValue("file."+algo), "file hash %s", algo)
}
require.Len(t, r.MultipartForm.Value, 12, "multipart form values")
w.WriteHeader(202)
fmt.Fprint(w, "RESPONSE")
})
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
writer.WriteField("token", "test")
file, err := writer.CreateFormFile("file", "my.file")
require.NoError(t, err)
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", nil)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
httpRequest = httpRequest.WithContext(ctx)
httpRequest.Body = ioutil.NopCloser(&buffer)
httpRequest.ContentLength = int64(buffer.Len())
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
handler := newProxy(ts.URL)
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 202, response.Code)
cancel() // this will trigger an async cleanup
waitUntilDeleted(t, filePath)
}
func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) {
var filePath string
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
tests := []struct {
name string
field string
response int
}{
{
name: "injected file.path",
field: "file.path",
response: 400,
},
{
name: "injected file.remote_id",
field: "file.remote_id",
response: 400,
},
{
name: "field with other prefix",
field: "other.path",
response: 202,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "PUT", r.Method, "method")
w.WriteHeader(202)
fmt.Fprint(w, "RESPONSE")
})
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
require.NoError(t, err)
fmt.Fprint(file, "test")
writer.WriteField(test.field, "value")
writer.Close()
httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", &buffer)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
httpRequest = httpRequest.WithContext(ctx)
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
handler := newProxy(ts.URL)
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, test.response, response.Code)
cancel() // this will trigger an async cleanup
waitUntilDeleted(t, filePath)
})
}
}
func TestUploadProcessingField(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
writer.WriteField("token2", "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
require.NoError(t, err)
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 500, response.Code)
}
func TestUploadingMultipleFiles(t *testing.T) {
testhelper.ConfigureSecret()
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
for i := 0; i < 11; i++ {
_, err = writer.CreateFormFile(fmt.Sprintf("file %v", i), "my.file")
require.NoError(t, err)
}
require.NoError(t, writer.Close())
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
require.NoError(t, err)
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 400, response.Code)
require.Equal(t, "upload request contains more than 10 files\n", response.Body.String())
}
func TestUploadProcessingFile(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
_, testServer := test.StartObjectStore()
defer testServer.Close()
storeUrl := testServer.URL + test.ObjectPath
tests := []struct {
name string
preauth *api.Response
}{
{
name: "FileStore Upload",
preauth: &api.Response{TempPath: tempPath},
},
{
name: "ObjectStore Upload",
preauth: &api.Response{RemoteObject: api.RemoteObject{StoreURL: storeUrl}},
},
{
name: "ObjectStore and FileStore Upload",
preauth: &api.Response{
TempPath: tempPath,
RemoteObject: api.RemoteObject{StoreURL: storeUrl},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
require.NoError(t, err)
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
require.NoError(t, err)
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, 200, response.Code)
})
}
}
func TestInvalidFileNames(t *testing.T) {
testhelper.ConfigureSecret()
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
for _, testCase := range []struct {
filename string
code int
expectedPrefix string
}{
{"foobar", 200, "foobar"}, // sanity check for test setup below
{"foo/bar", 200, "bar"},
{"foo/bar/baz", 200, "baz"},
{"/../../foobar", 200, "foobar"},
{".", 500, ""},
{"..", 500, ""},
{"./", 500, ""},
} {
buffer := &bytes.Buffer{}
writer := multipart.NewWriter(buffer)
file, err := writer.CreateFormFile("file", testCase.filename)
require.NoError(t, err)
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("POST", "/example", buffer)
require.NoError(t, err)
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts)
require.Equal(t, testCase.code, response.Code)
require.Equal(t, testCase.expectedPrefix, opts.TempFilePrefix)
}
}
func TestContentDispositionRewrite(t *testing.T) {
testhelper.ConfigureSecret()
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
tests := []struct {
desc string
header string
code int
sanitizedHeader string
}{
{
desc: "with name",
header: `form-data; name="foo"`,
code: 200,
sanitizedHeader: `form-data; name=foo`,
},
{
desc: "with name and name*",
header: `form-data; name="foo"; name*=UTF-8''bar`,
code: 200,
sanitizedHeader: `form-data; name=bar`,
},
{
desc: "with name and invalid name*",
header: `form-data; name="foo"; name*=UTF-16''bar`,
code: 200,
sanitizedHeader: `form-data; name=foo`,
},
}
for _, testCase := range tests {
t.Run(testCase.desc, func(t *testing.T) {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", testCase.header)
h.Set("Content-Type", "application/octet-stream")
buffer := &bytes.Buffer{}
writer := multipart.NewWriter(buffer)
file, err := writer.CreatePart(h)
require.NoError(t, err)
fmt.Fprint(file, "test")
writer.Close()
httpRequest := httptest.NewRequest("POST", "/example", buffer)
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
var upstreamRequestBuffer bytes.Buffer
customHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
r.Write(&upstreamRequestBuffer)
})
response := httptest.NewRecorder()
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, customHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts)
upstreamRequest, err := http.ReadRequest(bufio.NewReader(&upstreamRequestBuffer))
require.NoError(t, err)
reader, err := upstreamRequest.MultipartReader()
require.NoError(t, err)
for i := 0; ; i++ {
p, err := reader.NextPart()
if err == io.EOF {
require.Equal(t, i, 1)
break
}
require.NoError(t, err)
require.Equal(t, testCase.sanitizedHeader, p.Header.Get("Content-Disposition"))
}
require.Equal(t, testCase.code, response.Code)
})
}
}
func TestUploadHandlerRemovingExif(t *testing.T) {
content, err := ioutil.ReadFile("exif/testdata/sample_exif.jpg")
require.NoError(t, err)
runUploadTest(t, content, "sample_exif.jpg", 200, func(w http.ResponseWriter, r *http.Request) {
err := r.ParseMultipartForm(100000)
require.NoError(t, err)
size, err := strconv.Atoi(r.FormValue("file.size"))
require.NoError(t, err)
require.True(t, size < len(content), "Expected the file to be smaller after removal of exif")
require.True(t, size > 0, "Expected to receive not empty file")
w.WriteHeader(200)
fmt.Fprint(w, "RESPONSE")
})
}
func TestUploadHandlerRemovingExifTiff(t *testing.T) {
content, err := ioutil.ReadFile("exif/testdata/sample_exif.tiff")
require.NoError(t, err)
runUploadTest(t, content, "sample_exif.tiff", 200, func(w http.ResponseWriter, r *http.Request) {
err := r.ParseMultipartForm(100000)
require.NoError(t, err)
size, err := strconv.Atoi(r.FormValue("file.size"))
require.NoError(t, err)
require.True(t, size < len(content), "Expected the file to be smaller after removal of exif")
require.True(t, size > 0, "Expected to receive not empty file")
w.WriteHeader(200)
fmt.Fprint(w, "RESPONSE")
})
}
func TestUploadHandlerRemovingExifInvalidContentType(t *testing.T) {
content, err := ioutil.ReadFile("exif/testdata/sample_exif_invalid.jpg")
require.NoError(t, err)
runUploadTest(t, content, "sample_exif_invalid.jpg", 200, func(w http.ResponseWriter, r *http.Request) {
err := r.ParseMultipartForm(100000)
require.NoError(t, err)
output, err := ioutil.ReadFile(r.FormValue("file.path"))
require.NoError(t, err)
require.Equal(t, content, output, "Expected the file to be same as before")
w.WriteHeader(200)
fmt.Fprint(w, "RESPONSE")
})
}
func TestUploadHandlerRemovingExifCorruptedFile(t *testing.T) {
content, err := ioutil.ReadFile("exif/testdata/sample_exif_corrupted.jpg")
require.NoError(t, err)
runUploadTest(t, content, "sample_exif_corrupted.jpg", 422, func(w http.ResponseWriter, r *http.Request) {
err := r.ParseMultipartForm(100000)
require.Error(t, err)
})
}
func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, tsHandler func(http.ResponseWriter, *http.Request)) {
tempPath, err := ioutil.TempDir("", "uploads")
require.NoError(t, err)
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", filename)
require.NoError(t, err)
_, err = file.Write(image)
require.NoError(t, err)
err = writer.Close()
require.NoError(t, err)
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`/url/path\z`), tsHandler)
defer ts.Close()
httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", &buffer)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
httpRequest = httpRequest.WithContext(ctx)
httpRequest.ContentLength = int64(buffer.Len())
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
handler := newProxy(ts.URL)
apiResponse := &api.Response{TempPath: tempPath}
preparer := &DefaultPreparer{}
opts, _, err := preparer.Prepare(apiResponse)
require.NoError(t, err)
interceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts)
require.Equal(t, httpCode, response.Code)
}
func newProxy(url string) *proxy.Proxy {
parsedURL := helper.URLMustParse(url)
return proxy.NewProxy(parsedURL, "123", roundtripper.NewTestBackendRoundTripper(parsedURL))
}
func waitUntilDeleted(t *testing.T, path string) {
var err error
require.Eventually(t, func() bool {
_, err = os.Stat(path)
return err != nil
}, 10*time.Second, 10*time.Millisecond)
require.True(t, os.IsNotExist(err), "expected the file to be deleted")
}