f6d6e7f861
* WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
149 lines
4.9 KiB
TypeScript
149 lines
4.9 KiB
TypeScript
import { UploadFiles } from 'express'
|
|
import { Transaction } from 'sequelize/types'
|
|
import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
|
|
import { sequelizeTypescript } from '@server/initializers/database'
|
|
import { TagModel } from '@server/models/video/tag'
|
|
import { VideoModel } from '@server/models/video/video'
|
|
import { FilteredModelAttributes } from '@server/types'
|
|
import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
|
|
import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
|
|
import { federateVideoIfNeeded } from './activitypub/videos'
|
|
import { JobQueue } from './job-queue/job-queue'
|
|
import { Notifier } from './notifier'
|
|
import { createVideoMiniatureFromExisting } from './thumbnail'
|
|
|
|
function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
|
|
return {
|
|
name: videoInfo.name,
|
|
remote: false,
|
|
category: videoInfo.category,
|
|
licence: videoInfo.licence,
|
|
language: videoInfo.language,
|
|
commentsEnabled: videoInfo.commentsEnabled !== false, // If the value is not "false", the default is "true"
|
|
downloadEnabled: videoInfo.downloadEnabled !== false,
|
|
waitTranscoding: videoInfo.waitTranscoding || false,
|
|
nsfw: videoInfo.nsfw || false,
|
|
description: videoInfo.description,
|
|
support: videoInfo.support,
|
|
privacy: videoInfo.privacy || VideoPrivacy.PRIVATE,
|
|
channelId: channelId,
|
|
originallyPublishedAt: videoInfo.originallyPublishedAt
|
|
}
|
|
}
|
|
|
|
async function buildVideoThumbnailsFromReq (options: {
|
|
video: MVideoThumbnail
|
|
files: UploadFiles
|
|
fallback: (type: ThumbnailType) => Promise<MThumbnail>
|
|
automaticallyGenerated?: boolean
|
|
}) {
|
|
const { video, files, fallback, automaticallyGenerated } = options
|
|
|
|
const promises = [
|
|
{
|
|
type: ThumbnailType.MINIATURE,
|
|
fieldName: 'thumbnailfile'
|
|
},
|
|
{
|
|
type: ThumbnailType.PREVIEW,
|
|
fieldName: 'previewfile'
|
|
}
|
|
].map(p => {
|
|
const fields = files?.[p.fieldName]
|
|
|
|
if (fields) {
|
|
return createVideoMiniatureFromExisting({
|
|
inputPath: fields[0].path,
|
|
video,
|
|
type: p.type,
|
|
automaticallyGenerated: automaticallyGenerated || false
|
|
})
|
|
}
|
|
|
|
return fallback(p.type)
|
|
})
|
|
|
|
return Promise.all(promises)
|
|
}
|
|
|
|
async function setVideoTags (options: {
|
|
video: MVideoTag
|
|
tags: string[]
|
|
transaction?: Transaction
|
|
}) {
|
|
const { video, tags, transaction } = options
|
|
|
|
const internalTags = tags || []
|
|
const tagInstances = await TagModel.findOrCreateTags(internalTags, transaction)
|
|
|
|
await video.$set('Tags', tagInstances, { transaction })
|
|
video.Tags = tagInstances
|
|
}
|
|
|
|
async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
|
|
const result = await sequelizeTypescript.transaction(async t => {
|
|
// Maybe the video changed in database, refresh it
|
|
const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
|
|
// Video does not exist anymore
|
|
if (!videoDatabase) return undefined
|
|
|
|
// We transcoded the video file in another format, now we can publish it
|
|
const videoPublished = await videoDatabase.publishIfNeededAndSave(t)
|
|
|
|
// If the video was not published, we consider it is a new one for other instances
|
|
// Live videos are always federated, so it's not a new video
|
|
await federateVideoIfNeeded(videoDatabase, !wasLive && videoPublished, t)
|
|
|
|
return { videoDatabase, videoPublished }
|
|
})
|
|
|
|
if (result?.videoPublished) {
|
|
Notifier.Instance.notifyOnNewVideoIfNeeded(result.videoDatabase)
|
|
Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(result.videoDatabase)
|
|
}
|
|
}
|
|
|
|
async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) {
|
|
let dataInput: VideoTranscodingPayload
|
|
|
|
if (videoFile.isAudio()) {
|
|
dataInput = {
|
|
type: 'merge-audio-to-webtorrent',
|
|
resolution: DEFAULT_AUDIO_RESOLUTION,
|
|
videoUUID: video.uuid,
|
|
isNewVideo: true
|
|
}
|
|
} else {
|
|
dataInput = {
|
|
type: 'optimize-to-webtorrent',
|
|
videoUUID: video.uuid,
|
|
isNewVideo: true
|
|
}
|
|
}
|
|
|
|
const jobOptions = {
|
|
priority: await getTranscodingJobPriority(user)
|
|
}
|
|
|
|
return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions)
|
|
}
|
|
|
|
async function getTranscodingJobPriority (user: MUserId) {
|
|
const now = new Date()
|
|
const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
|
|
|
|
const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
|
|
|
|
return JOB_PRIORITY.TRANSCODING + videoUploadedByUser
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export {
|
|
buildLocalVideoFromReq,
|
|
publishAndFederateIfNeeded,
|
|
buildVideoThumbnailsFromReq,
|
|
setVideoTags,
|
|
addOptimizeOrMergeAudioJob,
|
|
getTranscodingJobPriority
|
|
}
|