1
0
Fork 0
peertube/server/helpers/express-utils.ts
kontrollanten f6d6e7f861
Resumable video uploads (#3933)
* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00

138 lines
3.7 KiB
TypeScript

import * as express from 'express'
import * as multer from 'multer'
import { REMOTE_SCHEME } from '../initializers/constants'
import { logger } from './logger'
import { deleteFileAndCatch, generateRandomString } from './utils'
import { extname } from 'path'
import { isArray } from './custom-validators/misc'
import { CONFIG } from '../initializers/config'
import { getExtFromMimetype } from './video'
import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
if (paramNSFW === 'true') return true
if (paramNSFW === 'false') return false
if (paramNSFW === 'both') return undefined
if (res?.locals.oauth) {
const user = res.locals.oauth.token.User
// User does not want NSFW videos
if (user.nsfwPolicy === 'do_not_list') return false
// Both
return undefined
}
if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false
// Display all
return null
}
function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) {
const files = req.files
if (!files) return
if (isArray(files)) {
(files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
return
}
for (const key of Object.keys(files)) {
const file = files[key]
if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
else deleteFileAndCatch(file.path)
}
}
function getHostWithPort (host: string) {
const splitted = host.split(':')
// The port was not specified
if (splitted.length === 1) {
if (REMOTE_SCHEME.HTTP === 'https') return host + ':443'
return host + ':80'
}
return host
}
function badRequest (req: express.Request, res: express.Response) {
return res.type('json')
.status(HttpStatusCode.BAD_REQUEST_400)
.end()
}
function createReqFiles (
fieldNames: string[],
mimeTypes: { [id: string]: string | string[] },
destinations: { [fieldName: string]: string }
) {
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, destinations[file.fieldname])
},
filename: async (req, file, cb) => {
let extension: string
const fileExtension = extname(file.originalname)
const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
// Take the file extension if we don't understand the mime type
if (!extensionFromMimetype) {
extension = fileExtension
} else {
// Take the first available extension for this mimetype
extension = extensionFromMimetype
}
let randomString = ''
try {
randomString = await generateRandomString(16)
} catch (err) {
logger.error('Cannot generate random string for file name.', { err })
randomString = 'fake-random-string'
}
cb(null, randomString + extension)
}
})
const fields: { name: string, maxCount: number }[] = []
for (const fieldName of fieldNames) {
fields.push({
name: fieldName,
maxCount: 1
})
}
return multer({ storage }).fields(fields)
}
function isUserAbleToSearchRemoteURI (res: express.Response) {
const user = res.locals.oauth ? res.locals.oauth.token.User : undefined
return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
(CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
}
function getCountVideos (req: express.Request) {
return req.query.skipCount !== true
}
// ---------------------------------------------------------------------------
export {
buildNSFWFilter,
getHostWithPort,
isUserAbleToSearchRemoteURI,
badRequest,
createReqFiles,
cleanUpReqFiles,
getCountVideos
}