From 12dc3a942a13c7f1489822dae052da197ef15905 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Jul 2023 16:02:49 +0200 Subject: [PATCH] Implement replace file in server side --- config/default.yaml | 5 + config/production.yaml.example | 5 + server/controllers/api/config.ts | 5 + server/controllers/api/videos/index.ts | 16 +- server/controllers/api/videos/source.ts | 206 ++++++++ server/controllers/api/videos/update.ts | 1 + server/controllers/api/videos/upload.ts | 11 +- .../custom-validators/activitypub/videos.ts | 1 + server/helpers/image-utils.ts | 2 + server/initializers/checker-before-init.ts | 1 + server/initializers/config.ts | 5 + server/initializers/constants.ts | 2 +- .../migrations/0800-video-replace-file.ts | 38 ++ server/lib/activitypub/context.ts | 3 + .../lib/activitypub/videos/shared/creator.ts | 1 + .../shared/object-to-model-attributes.ts | 4 + server/lib/activitypub/videos/updater.ts | 4 + .../job-queue/handlers/video-live-ending.ts | 20 +- server/lib/moderation.ts | 4 +- server/lib/server-config-manager.ts | 5 + server/lib/thumbnail.ts | 25 +- server/lib/video-blacklist.ts | 10 +- server/lib/video-pre-import.ts | 1 + server/middlewares/validators/config.ts | 2 + server/middlewares/validators/videos/index.ts | 6 +- .../validators/videos/shared/index.ts | 2 + .../validators/videos/shared/upload.ts | 39 ++ .../videos/shared/video-validators.ts | 104 ++++ .../validators/videos/video-source.ts | 110 ++++- .../validators/videos/video-studio.ts | 12 +- .../middlewares/validators/videos/videos.ts | 112 +---- .../formatter/video-activity-pub-format.ts | 2 + .../video/formatter/video-api-format.ts | 1 + .../video/shared/video-table-attributes.ts | 1 + server/models/video/video-source.ts | 45 +- server/models/video/video.ts | 8 +- server/tests/api/check-params/config.ts | 5 + server/tests/api/check-params/video-source.ts | 148 +++++- server/tests/api/server/config.ts | 9 + server/tests/api/videos/index.ts | 2 +- server/tests/api/videos/resumable-upload.ts | 8 +- server/tests/api/videos/video-source.ts | 447 +++++++++++++++++- server/tests/cli/prune-storage.ts | 22 +- server/tests/shared/videos.ts | 2 +- server/types/express.d.ts | 7 +- .../activitypub/objects/video-object.ts | 2 + .../plugins/server/server-hook.model.ts | 4 + shared/models/server/custom-config.model.ts | 6 + shared/models/server/server-config.model.ts | 6 + shared/models/videos/video-source.ts | 1 + shared/models/videos/video.model.ts | 2 + .../server-commands/server/config-command.ts | 27 ++ .../server-commands/server/servers-command.ts | 8 +- .../server-commands/videos/videos-command.ts | 71 ++- support/doc/api/openapi.yaml | 276 ++++++++--- 55 files changed, 1547 insertions(+), 325 deletions(-) create mode 100644 server/controllers/api/videos/source.ts create mode 100644 server/initializers/migrations/0800-video-replace-file.ts create mode 100644 server/middlewares/validators/videos/shared/index.ts create mode 100644 server/middlewares/validators/videos/shared/upload.ts create mode 100644 server/middlewares/validators/videos/shared/video-validators.ts diff --git a/config/default.yaml b/config/default.yaml index e590ab300..10d3f79e7 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -595,6 +595,11 @@ video_studio: remote_runners: enabled: false +video_file: + update: + # Add ability for users to replace the video file of an existing video + enabled: false + import: # Add ability for your users to import remote videos (from YouTube, torrent...) videos: diff --git a/config/production.yaml.example b/config/production.yaml.example index 884300ddb..a829b46f9 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -605,6 +605,11 @@ video_studio: remote_runners: enabled: false +video_file: + update: + # Add ability for users to replace the video file of an existing video + enabled: false + import: # Add ability for your users to import remote videos (from YouTube, torrent...) videos: diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 0980ec10a..c5c4c8a74 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -284,6 +284,11 @@ function customConfig (): CustomConfig { enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED } }, + videoFile: { + update: { + enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED + } + }, import: { videos: { concurrency: CONFIG.IMPORT.VIDEOS.CONCURRENCY, diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 520d8cbbb..3cdd42289 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -26,7 +26,6 @@ import { setDefaultVideosSort, videosCustomGetValidator, videosGetValidator, - videoSourceGetValidator, videosRemoveValidator, videosSortValidator } from '../../../middlewares' @@ -39,7 +38,9 @@ import { filesRouter } from './files' import { videoImportsRouter } from './import' import { liveRouter } from './live' import { ownershipVideoRouter } from './ownership' +import { videoPasswordRouter } from './passwords' import { rateVideoRouter } from './rate' +import { videoSourceRouter } from './source' import { statsRouter } from './stats' import { storyboardRouter } from './storyboard' import { studioRouter } from './studio' @@ -48,7 +49,6 @@ import { transcodingRouter } from './transcoding' import { updateRouter } from './update' import { uploadRouter } from './upload' import { viewRouter } from './view' -import { videoPasswordRouter } from './passwords' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -72,6 +72,7 @@ videosRouter.use('/', transcodingRouter) videosRouter.use('/', tokenRouter) videosRouter.use('/', videoPasswordRouter) videosRouter.use('/', storyboardRouter) +videosRouter.use('/', videoSourceRouter) videosRouter.get('/categories', openapiOperationDoc({ operationId: 'getCategories' }), @@ -108,13 +109,6 @@ videosRouter.get('/:id/description', asyncMiddleware(getVideoDescription) ) -videosRouter.get('/:id/source', - openapiOperationDoc({ operationId: 'getVideoSource' }), - authenticate, - asyncMiddleware(videoSourceGetValidator), - getVideoSource -) - videosRouter.get('/:id', openapiOperationDoc({ operationId: 'getVideo' }), optionalAuthenticate, @@ -177,10 +171,6 @@ async function getVideoDescription (req: express.Request, res: express.Response) return res.json({ description }) } -function getVideoSource (req: express.Request, res: express.Response) { - return res.json(res.locals.videoSource.toFormattedJSON()) -} - async function listVideos (req: express.Request, res: express.Response) { const serverActor = await getServerActor() diff --git a/server/controllers/api/videos/source.ts b/server/controllers/api/videos/source.ts new file mode 100644 index 000000000..b20c4af0e --- /dev/null +++ b/server/controllers/api/videos/source.ts @@ -0,0 +1,206 @@ +import express from 'express' +import { move } from 'fs-extra' +import { sequelizeTypescript } from '@server/initializers/database' +import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue' +import { Hooks } from '@server/lib/plugins/hooks' +import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail' +import { uploadx } from '@server/lib/uploadx' +import { buildMoveToObjectStorageJob } from '@server/lib/video' +import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' +import { buildNewFile } from '@server/lib/video-file' +import { VideoPathManager } from '@server/lib/video-path-manager' +import { buildNextVideoState } from '@server/lib/video-state' +import { openapiOperationDoc } from '@server/middlewares/doc' +import { VideoModel } from '@server/models/video/video' +import { VideoSourceModel } from '@server/models/video/video-source' +import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { HttpStatusCode, VideoState } from '@shared/models' +import { logger, loggerTagsFactory } from '../../../helpers/logger' +import { + asyncMiddleware, + authenticate, + replaceVideoSourceResumableInitValidator, + replaceVideoSourceResumableValidator, + videoSourceGetLatestValidator +} from '../../../middlewares' + +const lTags = loggerTagsFactory('api', 'video') + +const videoSourceRouter = express.Router() + +videoSourceRouter.get('/:id/source', + openapiOperationDoc({ operationId: 'getVideoSource' }), + authenticate, + asyncMiddleware(videoSourceGetLatestValidator), + getVideoLatestSource +) + +videoSourceRouter.post('/:id/source/replace-resumable', + authenticate, + asyncMiddleware(replaceVideoSourceResumableInitValidator), + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +videoSourceRouter.delete('/:id/source/replace-resumable', + authenticate, + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +videoSourceRouter.put('/:id/source/replace-resumable', + authenticate, + uploadx.upload, // uploadx doesn't next() before the file upload completes + asyncMiddleware(replaceVideoSourceResumableValidator), + asyncMiddleware(replaceVideoSourceResumable) +) + +// --------------------------------------------------------------------------- + +export { + videoSourceRouter +} + +// --------------------------------------------------------------------------- + +function getVideoLatestSource (req: express.Request, res: express.Response) { + return res.json(res.locals.videoSource.toFormattedJSON()) +} + +async function replaceVideoSourceResumable (req: express.Request, res: express.Response) { + const videoPhysicalFile = res.locals.updateVideoFileResumable + const user = res.locals.oauth.token.User + + const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) + const originalFilename = videoPhysicalFile.originalname + + const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid) + + try { + const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(res.locals.videoAll, videoFile) + await move(videoPhysicalFile.path, destination) + + let oldWebVideoFiles: MVideoFile[] = [] + let oldStreamingPlaylists: MStreamingPlaylistFiles[] = [] + + const inputFileUpdatedAt = new Date() + + const video = await sequelizeTypescript.transaction(async transaction => { + const video = await VideoModel.loadFull(res.locals.videoAll.id, transaction) + + oldWebVideoFiles = video.VideoFiles + oldStreamingPlaylists = video.VideoStreamingPlaylists + + for (const file of video.VideoFiles) { + await file.destroy({ transaction }) + } + for (const playlist of oldStreamingPlaylists) { + await playlist.destroy({ transaction }) + } + + videoFile.videoId = video.id + await videoFile.save({ transaction }) + + video.VideoFiles = [ videoFile ] + video.VideoStreamingPlaylists = [] + + video.state = buildNextVideoState() + video.duration = videoPhysicalFile.duration + video.inputFileUpdatedAt = inputFileUpdatedAt + await video.save({ transaction }) + + await autoBlacklistVideoIfNeeded({ + video, + user, + isRemote: false, + isNew: false, + isNewFile: true, + transaction + }) + + return video + }) + + await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists }) + + await VideoSourceModel.create({ + filename: originalFilename, + videoId: video.id, + createdAt: inputFileUpdatedAt + }) + + await regenerateMiniaturesIfNeeded(video) + await video.VideoChannel.setAsUpdated() + await addVideoJobsAfterUpload(video, video.getMaxQualityFile()) + + logger.info('Replaced video file of video %s with uuid %s.', video.name, video.uuid, lTags(video.uuid)) + + Hooks.runAction('action:api.video.file-updated', { video, req, res }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) + } finally { + videoFileMutexReleaser() + } +} + +async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { + const jobs: (CreateJobArgument & CreateJobOptions)[] = [ + { + type: 'manage-video-torrent' as 'manage-video-torrent', + payload: { + videoId: video.id, + videoFileId: videoFile.id, + action: 'create' + } + }, + + { + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + // No need to federate, we process these jobs sequentially + federate: false + } + }, + + { + type: 'federate-video' as 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideo: false + } + } + ] + + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + jobs.push(await buildMoveToObjectStorageJob({ video, isNewVideo: false, previousVideoState: undefined })) + } + + if (video.state === VideoState.TO_TRANSCODE) { + jobs.push({ + type: 'transcoding-job-builder' as 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + optimizeJob: { + isNewVideo: false + } + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) +} + +async function removeOldFiles (options: { + video: MVideo + files: MVideoFile[] + playlists: MStreamingPlaylistFiles[] +}) { + const { video, files, playlists } = options + + for (const file of files) { + await video.removeWebVideoFile(file) + } + + for (const playlist of playlists) { + await video.removeStreamingPlaylistFiles(playlist) + } +} diff --git a/server/controllers/api/videos/update.ts b/server/controllers/api/videos/update.ts index 28ec2cf37..1edc509dc 100644 --- a/server/controllers/api/videos/update.ts +++ b/server/controllers/api/videos/update.ts @@ -130,6 +130,7 @@ async function updateVideo (req: express.Request, res: express.Response) { user: res.locals.oauth.token.User, isRemote: false, isNew: false, + isNewFile: false, transaction: t }) diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 27fef0b1a..e520bf4b5 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -11,8 +11,9 @@ import { buildNewFile } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' import { buildNextVideoState } from '@server/lib/video-state' import { openapiOperationDoc } from '@server/middlewares/doc' +import { VideoPasswordModel } from '@server/models/video/video-password' import { VideoSourceModel } from '@server/models/video/video-source' -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' +import { MVideoFile, MVideoFullLight } from '@server/types/models' import { uuidToShort } from '@shared/extra-utils' import { HttpStatusCode, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' @@ -33,7 +34,6 @@ import { } from '../../../middlewares' import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' import { VideoModel } from '../../../models/video/video' -import { VideoPasswordModel } from '@server/models/video/video-password' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') @@ -109,7 +109,7 @@ async function addVideoLegacy (req: express.Request, res: express.Response) { } async function addVideoResumable (req: express.Request, res: express.Response) { - const videoPhysicalFile = res.locals.videoFileResumable + const videoPhysicalFile = res.locals.uploadVideoFileResumable const videoInfo = videoPhysicalFile.metadata const files = { previewfile: videoInfo.previewfile, thumbnailfile: videoInfo.thumbnailfile } @@ -193,6 +193,7 @@ async function addVideo (options: { user, isRemote: false, isNew: true, + isNewFile: true, transaction: t }) @@ -209,7 +210,7 @@ async function addVideo (options: { // Channel has a new content, set as updated await videoCreated.VideoChannel.setAsUpdated() - addVideoJobsAfterUpload(videoCreated, videoFile, user) + addVideoJobsAfterUpload(videoCreated, videoFile) .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res }) @@ -223,7 +224,7 @@ async function addVideo (options: { } } -async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) { +async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { const jobs: (CreateJobArgument & CreateJobOptions)[] = [ { type: 'manage-video-torrent' as 'manage-video-torrent', diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 573a29754..07e25b8ba 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -76,6 +76,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { isDateValid(video.published) && isDateValid(video.updated) && (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && + (!video.uploadDate || isDateValid(video.uploadDate)) && (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && video.attributedTo.length !== 0 } diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index 7b77e694a..2a8bb6e6e 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -63,6 +63,8 @@ async function generateImageFromVideoFile (options: { } catch (err) { logger.debug('Cannot remove pending image path after generation error.', { err, ...lTags() }) } + + throw err } } diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index a872fcba3..f77b0defb 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -40,6 +40,7 @@ function checkMissedConfig () { 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', 'video_studio.enabled', 'video_studio.remote_runners.enabled', + 'video_file.update.enabled', 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 37cd852f1..f12d9b85a 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -435,6 +435,11 @@ const CONFIG = { get ENABLED () { return config.get('video_studio.remote_runners.enabled') } } }, + VIDEO_FILE: { + UPDATE: { + get ENABLED () { return config.get('video_file.update.enabled') } + } + }, IMPORT: { VIDEOS: { get CONCURRENCY () { return config.get('import.videos.concurrency') }, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index e09f0e3c6..9e5a02854 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -27,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 795 +const LAST_MIGRATION_VERSION = 800 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0800-video-replace-file.ts b/server/initializers/migrations/0800-video-replace-file.ts new file mode 100644 index 000000000..f924a4d92 --- /dev/null +++ b/server/initializers/migrations/0800-video-replace-file.ts @@ -0,0 +1,38 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + { + const query = 'DELETE FROM "videoSource" WHERE "videoId" IS NULL' + await utils.sequelize.query(query, { transaction }) + } + + { + const query = 'ALTER TABLE "videoSource" ALTER COLUMN "videoId" SET NOT NULL' + await utils.sequelize.query(query, { transaction }) + } + + { + const data = { + type: Sequelize.DATE, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.addColumn('video', 'inputFileUpdatedAt', data, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts index 750276a11..eba6d636d 100644 --- a/server/lib/activitypub/context.ts +++ b/server/lib/activitypub/context.ts @@ -60,6 +60,9 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string }, originallyPublishedAt: 'sc:datePublished', + + uploadDate: 'sc:uploadDate', + views: { '@type': 'sc:Number', '@id': 'pt:views' diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts index bc139e4fa..512d14d82 100644 --- a/server/lib/activitypub/videos/shared/creator.ts +++ b/server/lib/activitypub/videos/shared/creator.ts @@ -49,6 +49,7 @@ export class APVideoCreator extends APVideoAbstractBuilder { user: undefined, isRemote: true, isNew: true, + isNewFile: true, transaction: t }) diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index a9e0bed97..6cbe72e27 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -231,6 +231,10 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi ? new Date(videoObject.originallyPublishedAt) : null, + inputFileUpdatedAt: videoObject.uploadDate + ? new Date(videoObject.uploadDate) + : null, + updatedAt: new Date(videoObject.updated), views: videoObject.views, remote: true, diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 522d7b043..acb087895 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts @@ -38,6 +38,8 @@ export class APVideoUpdater extends APVideoAbstractBuilder { { videoObject: this.videoObject, ...this.lTags() } ) + const oldInputFileUpdatedAt = this.video.inputFileUpdatedAt + try { const channelActor = await this.getOrCreateVideoChannelFromVideoObject() @@ -74,6 +76,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { user: undefined, isRemote: true, isNew: false, + isNewFile: oldInputFileUpdatedAt !== videoUpdated.inputFileUpdatedAt, transaction: undefined }) @@ -129,6 +132,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { this.video.createdAt = videoData.createdAt this.video.publishedAt = videoData.publishedAt this.video.originallyPublishedAt = videoData.originallyPublishedAt + this.video.inputFileUpdatedAt = videoData.inputFileUpdatedAt this.video.privacy = videoData.privacy this.video.channelId = videoData.channelId this.video.views = videoData.views diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index ae886de35..982280b55 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -7,7 +7,7 @@ import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' -import { generateLocalVideoMiniature } from '@server/lib/thumbnail' +import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail' import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' import { VideoPathManager } from '@server/lib/video-path-manager' import { moveToNextState } from '@server/lib/video-state' @@ -197,23 +197,7 @@ async function replaceLiveByReplay (options: { } // Regenerate the thumbnail & preview? - if (videoWithFiles.getMiniature().automaticallyGenerated === true) { - const miniature = await generateLocalVideoMiniature({ - video: videoWithFiles, - videoFile: videoWithFiles.getMaxQualityFile(), - type: ThumbnailType.MINIATURE - }) - await videoWithFiles.addAndSaveThumbnail(miniature) - } - - if (videoWithFiles.getPreview().automaticallyGenerated === true) { - const preview = await generateLocalVideoMiniature({ - video: videoWithFiles, - videoFile: videoWithFiles.getMaxQualityFile(), - type: ThumbnailType.PREVIEW - }) - await videoWithFiles.addAndSaveThumbnail(preview) - } + await regenerateMiniaturesIfNeeded(videoWithFiles) // We consider this is a new video await moveToNextState({ video: videoWithFiles, isNewVideo: true }) diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts index dc5d8c83c..db8284872 100644 --- a/server/lib/moderation.ts +++ b/server/lib/moderation.ts @@ -36,7 +36,7 @@ export type AcceptResult = { // --------------------------------------------------------------------------- // Stub function that can be filtered by plugins -function isLocalVideoAccepted (object: { +function isLocalVideoFileAccepted (object: { videoBody: VideoCreate videoFile: VideoUploadFile user: UserModel @@ -201,7 +201,7 @@ function createAccountAbuse (options: { export { isLocalLiveVideoAccepted, - isLocalVideoAccepted, + isLocalVideoFileAccepted, isLocalVideoThreadAccepted, isRemoteVideoCommentAccepted, isLocalVideoCommentReplyAccepted, diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index 5ce89b16d..beb5d4d82 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts @@ -171,6 +171,11 @@ class ServerConfigManager { enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED } }, + videoFile: { + update: { + enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED + } + }, import: { videos: { http: { diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index d95442795..0b98da14f 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts @@ -4,7 +4,7 @@ import { generateImageFilename, generateImageFromVideoFile } from '../helpers/im import { CONFIG } from '../initializers/config' import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' import { ThumbnailModel } from '../models/video/thumbnail' -import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' +import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models' import { MThumbnail } from '../types/models/video/thumbnail' import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' import { VideoPathManager } from './video-path-manager' @@ -187,8 +187,31 @@ function updateRemoteVideoThumbnail (options: { // --------------------------------------------------------------------------- +async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { + if (video.getMiniature().automaticallyGenerated === true) { + const miniature = await generateLocalVideoMiniature({ + video, + videoFile: video.getMaxQualityFile(), + type: ThumbnailType.MINIATURE + }) + await video.addAndSaveThumbnail(miniature) + } + + if (video.getPreview().automaticallyGenerated === true) { + const preview = await generateLocalVideoMiniature({ + video, + videoFile: video.getMaxQualityFile(), + type: ThumbnailType.PREVIEW + }) + await video.addAndSaveThumbnail(preview) + } +} + +// --------------------------------------------------------------------------- + export { generateLocalVideoMiniature, + regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromUrl, updateLocalVideoMiniatureFromExisting, updateRemoteVideoThumbnail, diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts index cb1ea834c..d5664a1b9 100644 --- a/server/lib/video-blacklist.ts +++ b/server/lib/video-blacklist.ts @@ -27,13 +27,14 @@ async function autoBlacklistVideoIfNeeded (parameters: { user?: MUser isRemote: boolean isNew: boolean + isNewFile: boolean notify?: boolean transaction?: Transaction }) { - const { video, user, isRemote, isNew, notify = true, transaction } = parameters + const { video, user, isRemote, isNew, isNewFile, notify = true, transaction } = parameters const doAutoBlacklist = await Hooks.wrapFun( autoBlacklistNeeded, - { video, user, isRemote, isNew }, + { video, user, isRemote, isNew, isNewFile }, 'filter:video.auto-blacklist.result' ) @@ -128,14 +129,15 @@ function autoBlacklistNeeded (parameters: { video: MVideoWithBlacklistLight isRemote: boolean isNew: boolean + isNewFile: boolean user?: MUser }) { - const { user, video, isRemote, isNew } = parameters + const { user, video, isRemote, isNew, isNewFile } = parameters // Already blacklisted if (video.VideoBlacklist) return false if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED || !user) return false - if (isRemote || isNew === false) return false + if (isRemote || (isNew === false && isNewFile === false)) return false if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST)) return false diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts index 381f1f535..fcb9f77d7 100644 --- a/server/lib/video-pre-import.ts +++ b/server/lib/video-pre-import.ts @@ -89,6 +89,7 @@ async function insertFromImportIntoDB (parameters: { notify: false, isRemote: false, isNew: true, + isNewFile: true, transaction: t }) diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index a6dbba524..4c1aa26c1 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -65,6 +65,8 @@ const customConfigUpdateValidator = [ body('videoStudio.enabled').isBoolean(), body('videoStudio.remoteRunners.enabled').isBoolean(), + body('videoFile.update.enabled').isBoolean(), + body('import.videos.concurrency').isInt({ min: 0 }), body('import.videos.http.enabled').isBoolean(), body('import.videos.torrent.enabled').isBoolean(), diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts index 0c824c314..8c6fc49b1 100644 --- a/server/middlewares/validators/videos/index.ts +++ b/server/middlewares/validators/videos/index.ts @@ -1,12 +1,13 @@ export * from './video-blacklist' export * from './video-captions' +export * from './video-channel-sync' export * from './video-channels' export * from './video-comments' export * from './video-files' export * from './video-imports' export * from './video-live' export * from './video-ownership-changes' -export * from './video-view' +export * from './video-passwords' export * from './video-rates' export * from './video-shares' export * from './video-source' @@ -14,6 +15,5 @@ export * from './video-stats' export * from './video-studio' export * from './video-token' export * from './video-transcoding' +export * from './video-view' export * from './videos' -export * from './video-channel-sync' -export * from './video-passwords' diff --git a/server/middlewares/validators/videos/shared/index.ts b/server/middlewares/validators/videos/shared/index.ts new file mode 100644 index 000000000..eb11dcc6a --- /dev/null +++ b/server/middlewares/validators/videos/shared/index.ts @@ -0,0 +1,2 @@ +export * from './upload' +export * from './video-validators' diff --git a/server/middlewares/validators/videos/shared/upload.ts b/server/middlewares/validators/videos/shared/upload.ts new file mode 100644 index 000000000..ea0dddc3c --- /dev/null +++ b/server/middlewares/validators/videos/shared/upload.ts @@ -0,0 +1,39 @@ +import express from 'express' +import { logger } from '@server/helpers/logger' +import { getVideoStreamDuration } from '@shared/ffmpeg' +import { HttpStatusCode } from '@shared/models' + +export async function addDurationToVideoFileIfNeeded (options: { + res: express.Response + videoFile: { path: string, duration?: number } + middlewareName: string +}) { + const { res, middlewareName, videoFile } = options + + try { + if (!videoFile.duration) await addDurationToVideo(videoFile) + } catch (err) { + logger.error('Invalid input file in ' + middlewareName, { err }) + + res.fail({ + status: HttpStatusCode.UNPROCESSABLE_ENTITY_422, + message: 'Video file unreadable.' + }) + return false + } + + return true +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function addDurationToVideo (videoFile: { path: string, duration?: number }) { + const duration = await getVideoStreamDuration(videoFile.path) + + // FFmpeg may not be able to guess video duration + // For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2 + if (isNaN(duration)) videoFile.duration = 0 + else videoFile.duration = duration +} diff --git a/server/middlewares/validators/videos/shared/video-validators.ts b/server/middlewares/validators/videos/shared/video-validators.ts new file mode 100644 index 000000000..72536011d --- /dev/null +++ b/server/middlewares/validators/videos/shared/video-validators.ts @@ -0,0 +1,104 @@ +import express from 'express' +import { isVideoFileMimeTypeValid, isVideoFileSizeValid } from '@server/helpers/custom-validators/videos' +import { logger } from '@server/helpers/logger' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' +import { isLocalVideoFileAccepted } from '@server/lib/moderation' +import { Hooks } from '@server/lib/plugins/hooks' +import { MUserAccountId, MVideo } from '@server/types/models' +import { HttpStatusCode, ServerErrorCode, ServerFilterHookName, VideoState } from '@shared/models' +import { checkUserQuota } from '../../shared' + +export async function commonVideoFileChecks (options: { + res: express.Response + user: MUserAccountId + videoFileSize: number + files: express.UploadFilesForCheck +}): Promise { + const { res, user, videoFileSize, files } = options + + if (!isVideoFileMimeTypeValid(files)) { + res.fail({ + status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, + message: 'This file is not supported. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') + }) + return false + } + + if (!isVideoFileSizeValid(videoFileSize.toString())) { + res.fail({ + status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, + message: 'This file is too large. It exceeds the maximum file size authorized.', + type: ServerErrorCode.MAX_FILE_SIZE_REACHED + }) + return false + } + + if (await checkUserQuota(user, videoFileSize, res) === false) return false + + return true +} + +export async function isVideoFileAccepted (options: { + req: express.Request + res: express.Response + videoFile: express.VideoUploadFile + hook: Extract +}) { + const { req, res, videoFile } = options + + // Check we accept this video + const acceptParameters = { + videoBody: req.body, + videoFile, + user: res.locals.oauth.token.User + } + const acceptedResult = await Hooks.wrapFun( + isLocalVideoFileAccepted, + acceptParameters, + 'filter:api.video.upload.accept.result' + ) + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused local video file.', { acceptedResult, acceptParameters }) + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: acceptedResult.errorMessage || 'Refused local video file' + }) + return false + } + + return true +} + +export function checkVideoFileCanBeEdited (video: MVideo, res: express.Response) { + if (video.isLive) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot edit a live video' + }) + + return false + } + + if (video.state === VideoState.TO_TRANSCODE || video.state === VideoState.TO_EDIT) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot edit video that is already waiting for transcoding/edition' + }) + + return false + } + + const validStates = new Set([ VideoState.PUBLISHED, VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, VideoState.TRANSCODING_FAILED ]) + if (!validStates.has(video.state)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Video state is not compatible with edition' + }) + + return false + } + + return true +} diff --git a/server/middlewares/validators/videos/video-source.ts b/server/middlewares/validators/videos/video-source.ts index c6d8f1a81..bbccb58b0 100644 --- a/server/middlewares/validators/videos/video-source.ts +++ b/server/middlewares/validators/videos/video-source.ts @@ -1,20 +1,31 @@ import express from 'express' +import { body, header } from 'express-validator' +import { getResumableUploadPath } from '@server/helpers/upload' import { getVideoWithAttributes } from '@server/helpers/video' +import { CONFIG } from '@server/initializers/config' +import { uploadx } from '@server/lib/uploadx' import { VideoSourceModel } from '@server/models/video/video-source' import { MVideoFullLight } from '@server/types/models' import { HttpStatusCode, UserRight } from '@shared/models' +import { Metadata as UploadXMetadata } from '@uploadx/core' +import { logger } from '../../../helpers/logger' import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared' +import { addDurationToVideoFileIfNeeded, checkVideoFileCanBeEdited, commonVideoFileChecks, isVideoFileAccepted } from './shared' -const videoSourceGetValidator = [ +export const videoSourceGetLatestValidator = [ isValidVideoIdParam('id'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res, 'for-api')) return + if (!await doesVideoExist(req.params.id, res, 'all')) return const video = getVideoWithAttributes(res) as MVideoFullLight - res.locals.videoSource = await VideoSourceModel.loadByVideoId(video.id) + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return + + res.locals.videoSource = await VideoSourceModel.loadLatest(video.id) + if (!res.locals.videoSource) { return res.fail({ status: HttpStatusCode.NOT_FOUND_404, @@ -22,13 +33,98 @@ const videoSourceGetValidator = [ }) } - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return + return next() + } +] + +export const replaceVideoSourceResumableValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body: express.CustomUploadXFile = req.body + const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename } + const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err })) + + if (!await checkCanUpdateVideoFile({ req, res })) { + return cleanup() + } + + if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'updateVideoFileResumableValidator' })) { + return cleanup() + } + + if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.update-file.accept.result' })) { + return cleanup() + } + + res.locals.updateVideoFileResumable = { ...file, originalname: file.filename } return next() } ] -export { - videoSourceGetValidator +export const replaceVideoSourceResumableInitValidator = [ + body('filename') + .exists(), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + logger.debug('Checking updateVideoFileResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers + }) + + if (areValidationErrors(req, res, { omitLog: true })) return + + if (!await checkCanUpdateVideoFile({ req, res })) return + + const videoFileMetadata = { + mimetype: req.headers['x-upload-content-type'] as string, + size: +req.headers['x-upload-content-length'], + originalname: req.body.filename + } + + const files = { videofile: [ videoFileMetadata ] } + if (await commonVideoFileChecks({ res, user, videoFileSize: videoFileMetadata.size, files }) === false) return + + return next() + } +] + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function checkCanUpdateVideoFile (options: { + req: express.Request + res: express.Response +}) { + const { req, res } = options + + if (!CONFIG.VIDEO_FILE.UPDATE.ENABLED) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Updating the file of an existing video is not allowed on this instance' + }) + return false + } + + if (!await doesVideoExist(req.params.id, res)) return false + + const user = res.locals.oauth.token.User + const video = res.locals.videoAll + + if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return false + + if (!checkVideoFileCanBeEdited(video, res)) return false + + return true } diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts index 7a68f88e5..a375af60a 100644 --- a/server/middlewares/validators/videos/video-studio.ts +++ b/server/middlewares/validators/videos/video-studio.ts @@ -11,8 +11,9 @@ import { cleanUpReqFiles } from '@server/helpers/express-utils' import { CONFIG } from '@server/initializers/config' import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio' import { isAudioFile } from '@shared/ffmpeg' -import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' +import { HttpStatusCode, UserRight, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' +import { checkVideoFileCanBeEdited } from './shared' const videoStudioAddEditionValidator = [ param('videoId') @@ -66,14 +67,7 @@ const videoStudioAddEditionValidator = [ if (!await doesVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req) const video = res.locals.videoAll - if (video.state === VideoState.TO_TRANSCODE || video.state === VideoState.TO_EDIT) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot edit video that is already waiting for transcoding/edition' - }) - - return cleanUpReqFiles(req) - } + if (!checkVideoFileCanBeEdited(video, res)) return cleanUpReqFiles(req) const user = res.locals.oauth.token.User if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index b39d13a23..aea3453b5 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -2,13 +2,12 @@ import express from 'express' import { body, header, param, query, ValidationChain } from 'express-validator' import { isTestInstance } from '@server/helpers/core-utils' import { getResumableUploadPath } from '@server/helpers/upload' -import { uploadx } from '@server/lib/uploadx' import { Redis } from '@server/lib/redis' +import { uploadx } from '@server/lib/uploadx' import { getServerActor } from '@server/models/application/application' import { ExpressPromiseHandler } from '@server/types/express-handler' import { MUserAccountId, MVideoFullLight } from '@server/types/models' import { arrayify, getAllPrivacies } from '@shared/core-utils' -import { getVideoStreamDuration } from '@shared/ffmpeg' import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models' import { exists, @@ -27,8 +26,6 @@ import { isValidPasswordProtectedPrivacy, isVideoCategoryValid, isVideoDescriptionValid, - isVideoFileMimeTypeValid, - isVideoFileSizeValid, isVideoFilterValid, isVideoImageValid, isVideoIncludeValid, @@ -44,21 +41,19 @@ import { logger } from '../../../helpers/logger' import { getVideoWithAttributes } from '../../../helpers/video' import { CONFIG } from '../../../initializers/config' import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants' -import { isLocalVideoAccepted } from '../../../lib/moderation' -import { Hooks } from '../../../lib/plugins/hooks' import { VideoModel } from '../../../models/video/video' import { areValidationErrors, checkCanAccessVideoStaticFiles, checkCanSeeVideo, checkUserCanManageVideo, - checkUserQuota, doesVideoChannelOfAccountExist, doesVideoExist, doesVideoFileOfVideoExist, isValidVideoIdParam, isValidVideoPasswordHeader } from '../shared' +import { addDurationToVideoFileIfNeeded, commonVideoFileChecks, isVideoFileAccepted } from './shared' const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ body('videofile') @@ -83,26 +78,15 @@ const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ const videoFile: express.VideoUploadFile = req.files['videofile'][0] const user = res.locals.oauth.token.User - if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) { + if ( + !await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files }) || + !isValidPasswordProtectedPrivacy(req, res) || + !await addDurationToVideoFileIfNeeded({ videoFile, res, middlewareName: 'videosAddvideosAddLegacyValidatorResumableValidator' }) || + !await isVideoFileAccepted({ req, res, videoFile, hook: 'filter:api.video.upload.accept.result' }) + ) { return cleanUpReqFiles(req) } - if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) - - try { - if (!videoFile.duration) await addDurationToVideo(videoFile) - } catch (err) { - logger.error('Invalid input file in videosAddLegacyValidator.', { err }) - - res.fail({ - status: HttpStatusCode.UNPROCESSABLE_ENTITY_422, - message: 'Video file unreadable.' - }) - return cleanUpReqFiles(req) - } - - if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req) - return next() } ]) @@ -146,22 +130,10 @@ const videosAddResumableValidator = [ await Redis.Instance.setUploadSession(uploadId) if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() + if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'videosAddResumableValidator' })) return cleanup() + if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.upload.accept.result' })) return cleanup() - try { - if (!file.duration) await addDurationToVideo(file) - } catch (err) { - logger.error('Invalid input file in videosAddResumableValidator.', { err }) - - res.fail({ - status: HttpStatusCode.UNPROCESSABLE_ENTITY_422, - message: 'Video file unreadable.' - }) - return cleanup() - } - - if (!await isVideoAccepted(req, res, file)) return cleanup() - - res.locals.videoFileResumable = { ...file, originalname: file.filename } + res.locals.uploadVideoFileResumable = { ...file, originalname: file.filename } return next() } @@ -604,76 +576,20 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) return false } -async function commonVideoChecksPass (parameters: { +async function commonVideoChecksPass (options: { req: express.Request res: express.Response user: MUserAccountId videoFileSize: number files: express.UploadFilesForCheck }): Promise { - const { req, res, user, videoFileSize, files } = parameters + const { req, res, user } = options if (areErrorsInScheduleUpdate(req, res)) return false if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false - if (!isVideoFileMimeTypeValid(files)) { - res.fail({ - status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, - message: 'This file is not supported. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') - }) - return false - } - - if (!isVideoFileSizeValid(videoFileSize.toString())) { - res.fail({ - status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, - message: 'This file is too large. It exceeds the maximum file size authorized.', - type: ServerErrorCode.MAX_FILE_SIZE_REACHED - }) - return false - } - - if (await checkUserQuota(user, videoFileSize, res) === false) return false + if (!await commonVideoFileChecks(options)) return false return true } - -export async function isVideoAccepted ( - req: express.Request, - res: express.Response, - videoFile: express.VideoUploadFile -) { - // Check we accept this video - const acceptParameters = { - videoBody: req.body, - videoFile, - user: res.locals.oauth.token.User - } - const acceptedResult = await Hooks.wrapFun( - isLocalVideoAccepted, - acceptParameters, - 'filter:api.video.upload.accept.result' - ) - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused local video.', { acceptedResult, acceptParameters }) - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: acceptedResult.errorMessage || 'Refused local video' - }) - return false - } - - return true -} - -async function addDurationToVideo (videoFile: { path: string, duration?: number }) { - const duration = await getVideoStreamDuration(videoFile.path) - - // FFmpeg may not be able to guess video duration - // For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2 - if (isNaN(duration)) videoFile.duration = 0 - else videoFile.duration = duration -} diff --git a/server/models/video/formatter/video-activity-pub-format.ts b/server/models/video/formatter/video-activity-pub-format.ts index c0d3d5f3e..a5b3e9ca6 100644 --- a/server/models/video/formatter/video-activity-pub-format.ts +++ b/server/models/video/formatter/video-activity-pub-format.ts @@ -76,6 +76,8 @@ export function videoModelToActivityPubObject (video: MVideoAP): VideoObject { updated: video.updatedAt.toISOString(), + uploadDate: video.inputFileUpdatedAt?.toISOString(), + tag: buildTags(video), mediaType: 'text/markdown', diff --git a/server/models/video/formatter/video-api-format.ts b/server/models/video/formatter/video-api-format.ts index 1af51d132..7a58f5d3c 100644 --- a/server/models/video/formatter/video-api-format.ts +++ b/server/models/video/formatter/video-api-format.ts @@ -149,6 +149,7 @@ export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetail commentsEnabled: video.commentsEnabled, downloadEnabled: video.downloadEnabled, waitTranscoding: video.waitTranscoding, + inputFileUpdatedAt: video.inputFileUpdatedAt, state: { id: video.state, label: getStateLabel(video.state) diff --git a/server/models/video/sql/video/shared/video-table-attributes.ts b/server/models/video/sql/video/shared/video-table-attributes.ts index e0fa9d7c1..ef625c57b 100644 --- a/server/models/video/sql/video/shared/video-table-attributes.ts +++ b/server/models/video/sql/video/shared/video-table-attributes.ts @@ -263,6 +263,7 @@ export class VideoTableAttributes { 'state', 'publishedAt', 'originallyPublishedAt', + 'inputFileUpdatedAt', 'channelId', 'createdAt', 'updatedAt', diff --git a/server/models/video/video-source.ts b/server/models/video/video-source.ts index e306b160d..1b6868b85 100644 --- a/server/models/video/video-source.ts +++ b/server/models/video/video-source.ts @@ -1,27 +1,18 @@ -import { Op } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - ForeignKey, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' +import { Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoSource } from '@shared/models/videos/video-source' import { AttributesOnly } from '@shared/typescript-utils' +import { getSort } from '../shared' import { VideoModel } from './video' @Table({ tableName: 'videoSource', indexes: [ { - fields: [ 'videoId' ], - where: { - videoId: { - [Op.ne]: null - } - } + fields: [ 'videoId' ] + }, + { + fields: [ { name: 'createdAt', order: 'DESC' } ] } ] }) @@ -40,16 +31,26 @@ export class VideoSourceModel extends Model VideoModel) + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) Video: VideoModel - static loadByVideoId (videoId) { - return VideoSourceModel.findOne({ where: { videoId } }) + static loadLatest (videoId: number, transaction?: Transaction) { + return VideoSourceModel.findOne({ + where: { videoId }, + order: getSort('-createdAt'), + transaction + }) } - toFormattedJSON () { + toFormattedJSON (): VideoSource { return { - filename: this.filename + filename: this.filename, + createdAt: this.createdAt.toISOString() } } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 4c6297243..2fe701436 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -546,6 +546,12 @@ export class VideoModel extends Model>> { @Column state: VideoState + // We already have the information in videoSource table for local videos, but we prefer to normalize it for performance + // And also to store the info from remote instances + @AllowNull(true) + @Column + inputFileUpdatedAt: Date + @CreatedAt createdAt: Date @@ -610,7 +616,7 @@ export class VideoModel extends Model>> { @HasOne(() => VideoSourceModel, { foreignKey: { name: 'videoId', - allowNull: true + allowNull: false }, onDelete: 'CASCADE' }) diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 80b616ccf..2f523d4ce 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -170,6 +170,11 @@ describe('Test config API validators', function () { enabled: true } }, + videoFile: { + update: { + enabled: true + } + }, import: { videos: { concurrency: 1, diff --git a/server/tests/api/check-params/video-source.ts b/server/tests/api/check-params/video-source.ts index ca324bb9d..3c641ccd3 100644 --- a/server/tests/api/check-params/video-source.ts +++ b/server/tests/api/check-params/video-source.ts @@ -1,5 +1,12 @@ import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@shared/server-commands' describe('Test video sources API validator', function () { let server: PeerTubeServer = null @@ -7,35 +14,138 @@ describe('Test video sources API validator', function () { let userToken: string before(async function () { - this.timeout(30000) + this.timeout(120000) server = await createSingleServer(1) await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) - const created = await server.videos.quickUpload({ name: 'video' }) - uuid = created.uuid - - userToken = await server.users.generateUserAndToken('user') + userToken = await server.users.generateUserAndToken('user1') }) - it('Should fail without a valid uuid', async function () { - await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + describe('When getting latest source', function () { + + before(async function () { + const created = await server.videos.quickUpload({ name: 'video' }) + uuid = created.uuid + }) + + it('Should fail without a valid uuid', async function () { + await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should receive 404 when passing a non existing video id', async function () { + await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not get the source as unauthenticated', async function () { + await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) + }) + + it('Should not get the source with another user', async function () { + await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: userToken }) + }) + + it('Should succeed with the correct parameters get the source as another user', async function () { + await server.videos.getSource({ id: uuid }) + }) }) - it('Should receive 404 when passing a non existing video id', async function () { - await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) + describe('When updating source video file', function () { + let userAccessToken: string + let userId: number - it('Should not get the source as unauthenticated', async function () { - await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) - }) + let videoId: string + let userVideoId: string - it('Should not get the source with another user', async function () { - await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: userToken }) - }) + before(async function () { + const res = await server.users.generate('user2') + userAccessToken = res.token + userId = res.userId - it('Should succeed with the correct parameters get the source as another user', async function () { - await server.videos.getSource({ id: uuid }) + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoId = uuid + + await waitJobs([ server ]) + }) + + it('Should fail if not enabled on the instance', async function () { + await server.config.disableFileUpdate() + + await server.videos.replaceSourceFile({ videoId, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail on an unknown video', async function () { + await server.config.enableFileUpdate() + + await server.videos.replaceSourceFile({ videoId: 404, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an invalid video', async function () { + await server.config.enableLive({ allowReplay: false }) + + const { video } = await server.live.quickCreate({ saveReplay: false, permanentLive: true }) + await server.videos.replaceSourceFile({ + videoId: video.uuid, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail without token', async function () { + await server.videos.replaceSourceFile({ + token: null, + videoId, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another user', async function () { + await server.videos.replaceSourceFile({ + token: userAccessToken, + videoId, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an incorrect input file', async function () { + await server.videos.replaceSourceFile({ + fixture: 'video_short_fake.webm', + videoId, + expectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + }) + + await server.videos.replaceSourceFile({ + fixture: 'video_short.mkv', + videoId, + expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 + }) + }) + + it('Should fail if quota is exceeded', async function () { + this.timeout(60000) + + const { uuid } = await server.videos.quickUpload({ name: 'user video' }) + userVideoId = uuid + await waitJobs([ server ]) + + await server.users.update({ userId, videoQuota: 1 }) + await server.videos.replaceSourceFile({ + token: userAccessToken, + videoId: uuid, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + this.timeout(60000) + + await server.users.update({ userId, videoQuota: 1000 * 1000 * 1000 }) + await server.videos.replaceSourceFile({ videoId: userVideoId, fixture: 'video_short.mp4' }) + }) }) after(async function () { diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 0e700eddb..a614d92d2 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -105,6 +105,8 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { expect(data.videoStudio.enabled).to.be.false expect(data.videoStudio.remoteRunners.enabled).to.be.false + expect(data.videoFile.update.enabled).to.be.false + expect(data.import.videos.concurrency).to.equal(2) expect(data.import.videos.http.enabled).to.be.true expect(data.import.videos.torrent.enabled).to.be.true @@ -216,6 +218,8 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.videoStudio.enabled).to.be.true expect(data.videoStudio.remoteRunners.enabled).to.be.true + expect(data.videoFile.update.enabled).to.be.true + expect(data.import.videos.concurrency).to.equal(4) expect(data.import.videos.http.enabled).to.be.false expect(data.import.videos.torrent.enabled).to.be.false @@ -386,6 +390,11 @@ const newCustomConfig: CustomConfig = { enabled: true } }, + videoFile: { + update: { + enabled: true + } + }, import: { videos: { concurrency: 4, diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 9c79b3aa6..01d0c5852 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -13,11 +13,11 @@ import './video-imports' import './video-nsfw' import './video-playlists' import './video-playlist-thumbnails' +import './video-source' import './video-privacy' import './video-schedule-update' import './videos-common-filters' import './videos-history' import './videos-overview' -import './video-source' import './video-static-file-privacy' import './video-storyboard' diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts index 91eb61833..cac1201e9 100644 --- a/server/tests/api/videos/resumable-upload.ts +++ b/server/tests/api/videos/resumable-upload.ts @@ -11,6 +11,7 @@ import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServ // Most classic resumable upload tests are done in other test suites describe('Test resumable upload', function () { + const path = '/api/v1/videos/upload-resumable' const defaultFixture = 'video_short.mp4' let server: PeerTubeServer let rootId: number @@ -44,7 +45,7 @@ describe('Test resumable upload', function () { const mimetype = 'video/mp4' - const res = await server.videos.prepareResumableUpload({ token, attributes, size, mimetype, originalName, lastModified }) + const res = await server.videos.prepareResumableUpload({ path, token, attributes, size, mimetype, originalName, lastModified }) return res.header['location'].split('?')[1] } @@ -66,6 +67,7 @@ describe('Test resumable upload', function () { return server.videos.sendResumableChunks({ token, + path, pathUploadId, videoFilePath: absoluteFilePath, size, @@ -125,7 +127,7 @@ describe('Test resumable upload', function () { it('Should correctly delete files after an upload', async function () { const uploadId = await prepareUpload() await sendChunks({ pathUploadId: uploadId }) - await server.videos.endResumableUpload({ pathUploadId: uploadId }) + await server.videos.endResumableUpload({ path, pathUploadId: uploadId }) expect(await countResumableUploads()).to.equal(0) }) @@ -251,7 +253,7 @@ describe('Test resumable upload', function () { const uploadId1 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) await sendChunks({ pathUploadId: uploadId1 }) - await server.videos.endResumableUpload({ pathUploadId: uploadId1 }) + await server.videos.endResumableUpload({ path, pathUploadId: uploadId1 }) const uploadId2 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) expect(uploadId1).to.equal(uploadId2) diff --git a/server/tests/api/videos/video-source.ts b/server/tests/api/videos/video-source.ts index 5ecf8316f..8669f342e 100644 --- a/server/tests/api/videos/video-source.ts +++ b/server/tests/api/videos/video-source.ts @@ -1,36 +1,447 @@ import { expect } from 'chai' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' +import { expectStartWith } from '@server/tests/shared' +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' +import { HttpStatusCode } from '@shared/models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@shared/server-commands' -describe('Test video source', () => { - let server: PeerTubeServer = null - const fixture = 'video_short.webm' +describe('Test a video file replacement', function () { + let servers: PeerTubeServer[] = [] + + let replaceDate: Date + let userToken: string + let uuid: string before(async function () { - this.timeout(30000) + this.timeout(50000) - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + await servers[0].config.enableFileUpdate() + + userToken = await servers[0].users.generateUserAndToken('user1') + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) }) - it('Should get the source filename with legacy upload', async function () { - this.timeout(30000) + describe('Getting latest video source', () => { + const fixture = 'video_short.webm' + const uuids: string[] = [] - const { uuid } = await server.videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' }) + it('Should get the source filename with legacy upload', async function () { + this.timeout(30000) - const source = await server.videos.getSource({ id: uuid }) - expect(source.filename).to.equal(fixture) + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' }) + uuids.push(uuid) + + const source = await servers[0].videos.getSource({ id: uuid }) + expect(source.filename).to.equal(fixture) + }) + + it('Should get the source filename with resumable upload', async function () { + this.timeout(30000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' }) + uuids.push(uuid) + + const source = await servers[0].videos.getSource({ id: uuid }) + expect(source.filename).to.equal(fixture) + }) + + after(async function () { + this.timeout(60000) + + for (const uuid of uuids) { + await servers[0].videos.remove({ id: uuid }) + } + + await waitJobs(servers) + }) }) - it('Should get the source filename with resumable upload', async function () { - this.timeout(30000) + describe('Updating video source', function () { - const { uuid } = await server.videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' }) + describe('Filesystem', function () { - const source = await server.videos.getSource({ id: uuid }) - expect(source.filename).to.equal(fixture) + it('Should replace a video file with transcoding disabled', async function () { + this.timeout(120000) + + await servers[0].config.disableTranscoding() + + const { uuid } = await servers[0].videos.quickUpload({ name: 'fs without transcoding', fixture: 'video_short_720p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(720) + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(360) + } + }) + + it('Should replace a video file with transcoding enabled', async function () { + this.timeout(120000) + + const previousPaths: string[] = [] + + await servers[0].config.enableTranscoding(true, true, true) + + const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: 'video_short_720p.mp4' }) + uuid = videoUUID + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + expect(video.inputFileUpdatedAt).to.be.null + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(6 * 2) + + // Grab old paths to ensure we'll regenerate + + previousPaths.push(video.previewPath) + previousPaths.push(video.thumbnailPath) + + for (const file of files) { + previousPaths.push(file.fileUrl) + previousPaths.push(file.torrentUrl) + previousPaths.push(file.metadataUrl) + + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + previousPaths.push(JSON.stringify(metadata)) + } + + const { storyboards } = await server.storyboard.list({ id: uuid }) + for (const s of storyboards) { + previousPaths.push(s.storyboardPath) + } + } + + replaceDate = new Date() + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + expect(video.inputFileUpdatedAt).to.not.be.null + expect(new Date(video.inputFileUpdatedAt)).to.be.above(replaceDate) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(4 * 2) + + expect(previousPaths).to.not.include(video.previewPath) + expect(previousPaths).to.not.include(video.thumbnailPath) + + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + + for (const file of files) { + expect(previousPaths).to.not.include(file.fileUrl) + expect(previousPaths).to.not.include(file.torrentUrl) + expect(previousPaths).to.not.include(file.metadataUrl) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) + + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + expect(previousPaths).to.not.include(JSON.stringify(metadata)) + } + + const { storyboards } = await server.storyboard.list({ id: uuid }) + for (const s of storyboards) { + expect(previousPaths).to.not.include(s.storyboardPath) + + await makeGetRequest({ url: server.url, path: s.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + await servers[0].config.enableMinimumTranscoding() + }) + + it('Should have cleaned up old files', async function () { + { + const count = await servers[0].servers.countFiles('storyboards') + expect(count).to.equal(2) + } + + { + const count = await servers[0].servers.countFiles('web-videos') + expect(count).to.equal(5 + 1) // +1 for private directory + } + + { + const count = await servers[0].servers.countFiles('streaming-playlists/hls') + expect(count).to.equal(1 + 1) // +1 for private directory + } + + { + const count = await servers[0].servers.countFiles('torrents') + expect(count).to.equal(9) + } + }) + + it('Should have the correct source input', async function () { + const source = await servers[0].videos.getSource({ id: uuid }) + + expect(source.filename).to.equal('video_short_360p.mp4') + expect(new Date(source.createdAt)).to.be.above(replaceDate) + }) + + it('Should not have regenerated miniatures that were previously uploaded', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.upload({ + attributes: { + name: 'custom miniatures', + thumbnailfile: 'custom-thumbnail.jpg', + previewfile: 'custom-preview.jpg' + } + }) + + await waitJobs(servers) + + const previousPaths: string[] = [] + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + previousPaths.push(video.previewPath) + previousPaths.push(video.thumbnailPath) + + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + expect(previousPaths).to.include(video.previewPath) + expect(previousPaths).to.include(video.thumbnailPath) + + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + }) + + describe('Autoblacklist', function () { + + function updateAutoBlacklist (enabled: boolean) { + return servers[0].config.updateExistingSubConfig({ + newConfig: { + autoBlacklist: { + videos: { + ofUsers: { + enabled + } + } + } + } + }) + } + + async function expectBlacklist (uuid: string, value: boolean) { + const video = await servers[0].videos.getWithToken({ id: uuid }) + + expect(video.blacklisted).to.equal(value) + } + + before(async function () { + await updateAutoBlacklist(true) + }) + + it('Should auto blacklist an unblacklisted video after file replacement', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) + await waitJobs(servers) + await expectBlacklist(uuid, true) + + await servers[0].blacklist.remove({ videoId: uuid }) + await expectBlacklist(uuid, false) + + await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + await expectBlacklist(uuid, true) + }) + + it('Should auto blacklist an already blacklisted video after file replacement', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) + await waitJobs(servers) + await expectBlacklist(uuid, true) + + await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + await expectBlacklist(uuid, true) + }) + + it('Should not auto blacklist if auto blacklist has been disabled between the upload and the replacement', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) + await waitJobs(servers) + await expectBlacklist(uuid, true) + + await servers[0].blacklist.remove({ videoId: uuid }) + await expectBlacklist(uuid, false) + + await updateAutoBlacklist(false) + + await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short1.webm' }) + await waitJobs(servers) + + await expectBlacklist(uuid, false) + }) + }) + + describe('With object storage enabled', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + await servers[0].run(configOverride) + }) + + it('Should replace a video file with transcoding disabled', async function () { + this.timeout(120000) + + await servers[0].config.disableTranscoding() + + const { uuid } = await servers[0].videos.quickUpload({ + name: 'object storage without transcoding', + fixture: 'video_short_720p.mp4' + }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(720) + expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(360) + expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + }) + + it('Should replace a video file with transcoding enabled', async function () { + this.timeout(120000) + + const previousPaths: string[] = [] + + await servers[0].config.enableTranscoding(true, true, true) + + const { uuid: videoUUID } = await servers[0].videos.quickUpload({ + name: 'object storage with transcoding', + fixture: 'video_short_360p.mp4' + }) + uuid = videoUUID + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(4 * 2) + + for (const file of files) { + previousPaths.push(file.fileUrl) + } + + for (const file of video.files) { + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + for (const file of video.streamingPlaylists[0].files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_240p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(3 * 2) + + for (const file of files) { + expect(previousPaths).to.not.include(file.fileUrl) + } + + for (const file of video.files) { + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + for (const file of video.streamingPlaylists[0].files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + } + }) + }) }) after(async function () { - await cleanupTests([ server ]) + await cleanupTests(servers) }) }) diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 00f63570f..72a4b1332 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts @@ -19,12 +19,6 @@ import { waitJobs } from '@shared/server-commands' -async function countFiles (server: PeerTubeServer, directory: string) { - const files = await readdir(server.servers.buildDirectory(directory)) - - return files.length -} - async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) { const files = await readdir(server.servers.buildDirectory(directory)) @@ -35,28 +29,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst async function assertCountAreOkay (servers: PeerTubeServer[]) { for (const server of servers) { - const videosCount = await countFiles(server, 'web-videos') + const videosCount = await server.servers.countFiles('web-videos') expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory - const privateVideosCount = await countFiles(server, 'web-videos/private') + const privateVideosCount = await server.servers.countFiles('web-videos/private') expect(privateVideosCount).to.equal(4) - const torrentsCount = await countFiles(server, 'torrents') + const torrentsCount = await server.servers.countFiles('torrents') expect(torrentsCount).to.equal(24) - const previewsCount = await countFiles(server, 'previews') + const previewsCount = await server.servers.countFiles('previews') expect(previewsCount).to.equal(3) - const thumbnailsCount = await countFiles(server, 'thumbnails') + const thumbnailsCount = await server.servers.countFiles('thumbnails') expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist - const avatarsCount = await countFiles(server, 'avatars') + const avatarsCount = await server.servers.countFiles('avatars') expect(avatarsCount).to.equal(4) - const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls')) + const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls')) expect(hlsRootCount).to.equal(3) // 2 videos + private directory - const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private')) + const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private')) expect(hlsPrivateRootCount).to.equal(1) } } diff --git a/server/tests/shared/videos.ts b/server/tests/shared/videos.ts index e09bd60b5..3f59c329f 100644 --- a/server/tests/shared/videos.ts +++ b/server/tests/shared/videos.ts @@ -277,7 +277,7 @@ function checkUploadVideoParam ( ) { return mode === 'legacy' ? server.videos.buildLegacyUpload({ token, attributes, expectedStatus }) - : server.videos.buildResumeUpload({ token, attributes, expectedStatus }) + : server.videos.buildResumeUpload({ token, attributes, expectedStatus, path: '/api/v1/videos/upload-resumable' }) } // serverNumber starts from 1 diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 9c1be9bd1..4729c4534 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts @@ -86,13 +86,15 @@ declare module 'express' { // Our custom UploadXFile object using our custom metadata export type CustomUploadXFile = UploadXFile & { metadata: T } - export type EnhancedUploadXFile = CustomUploadXFile & { + export type EnhancedUploadXFile = CustomUploadXFile & { duration: number path: string filename: string originalname: string } + export type UploadNewVideoUploadXFile = EnhancedUploadXFile & CustomUploadXFile + // Extends Response with added functions and potential variables passed by middlewares interface Response { fail: (options: { @@ -139,7 +141,8 @@ declare module 'express' { videoFile?: MVideoFile - videoFileResumable?: EnhancedUploadXFile + uploadVideoFileResumable?: UploadNewVideoUploadXFile + updateVideoFileResumable?: EnhancedUploadXFile videoImport?: MVideoImportDefault diff --git a/shared/models/activitypub/objects/video-object.ts b/shared/models/activitypub/objects/video-object.ts index a252a2df0..409504f0f 100644 --- a/shared/models/activitypub/objects/video-object.ts +++ b/shared/models/activitypub/objects/video-object.ts @@ -31,9 +31,11 @@ export interface VideoObject { downloadEnabled: boolean waitTranscoding: boolean state: VideoState + published: string originallyPublishedAt: string updated: string + uploadDate: string mediaType: 'text/markdown' content: string diff --git a/shared/models/plugins/server/server-hook.model.ts b/shared/models/plugins/server/server-hook.model.ts index 0ec62222d..cf387ffd7 100644 --- a/shared/models/plugins/server/server-hook.model.ts +++ b/shared/models/plugins/server/server-hook.model.ts @@ -64,6 +64,7 @@ export const serverFilterHookObject = { 'filter:api.video.pre-import-torrent.accept.result': true, 'filter:api.video.post-import-url.accept.result': true, 'filter:api.video.post-import-torrent.accept.result': true, + 'filter:api.video.update-file.accept.result': true, // Filter the result of the accept comment (thread or reply) functions // If the functions return false then the user cannot post its comment 'filter:api.video-thread.create.accept.result': true, @@ -155,6 +156,9 @@ export const serverActionHookObject = { // Fired when a local video is viewed 'action:api.video.viewed': true, + // Fired when a local video file has been replaced by a new one + 'action:api.video.file-updated': true, + // Fired when a video channel is created 'action:api.video-channel.created': true, // Fired when a video channel is updated diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 9aa66f2b8..0dbb46fa8 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -175,6 +175,12 @@ export interface CustomConfig { } } + videoFile: { + update: { + enabled: boolean + } + } + import: { videos: { concurrency: number diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 288cf84cd..3f61e93b5 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -192,6 +192,12 @@ export interface ServerConfig { } } + videoFile: { + update: { + enabled: boolean + } + } + import: { videos: { http: { diff --git a/shared/models/videos/video-source.ts b/shared/models/videos/video-source.ts index 57e54fc7f..bf4ad2453 100644 --- a/shared/models/videos/video-source.ts +++ b/shared/models/videos/video-source.ts @@ -1,3 +1,4 @@ export interface VideoSource { filename: string + createdAt: string | Date } diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index 9004efb35..7e5930067 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -94,4 +94,6 @@ export interface VideoDetails extends Video { files: VideoFile[] streamingPlaylists: VideoStreamingPlaylist[] + + inputFileUpdatedAt: string | Date } diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts index 7f1e9d977..3521b2d69 100644 --- a/shared/server-commands/server/config-command.ts +++ b/shared/server-commands/server/config-command.ts @@ -74,6 +74,28 @@ export class ConfigCommand extends AbstractCommand { // --------------------------------------------------------------------------- + disableFileUpdate () { + return this.setFileUpdateEnabled(false) + } + + enableFileUpdate () { + return this.setFileUpdateEnabled(true) + } + + private setFileUpdateEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + videoFile: { + update: { + enabled + } + } + } + }) + } + + // --------------------------------------------------------------------------- + enableChannelSync () { return this.setChannelSyncEnabled(true) } @@ -466,6 +488,11 @@ export class ConfigCommand extends AbstractCommand { enabled: false } }, + videoFile: { + update: { + enabled: false + } + }, import: { videos: { concurrency: 3, diff --git a/shared/server-commands/server/servers-command.ts b/shared/server-commands/server/servers-command.ts index c91c2b008..54e586a18 100644 --- a/shared/server-commands/server/servers-command.ts +++ b/shared/server-commands/server/servers-command.ts @@ -1,5 +1,5 @@ import { exec } from 'child_process' -import { copy, ensureDir, readFile, remove } from 'fs-extra' +import { copy, ensureDir, readFile, readdir, remove } from 'fs-extra' import { basename, join } from 'path' import { isGithubCI, root, wait } from '@shared/core-utils' import { getFileSize } from '@shared/extra-utils' @@ -77,6 +77,12 @@ export class ServersCommand extends AbstractCommand { return join(root(), 'test' + this.server.internalServerNumber, directory) } + async countFiles (directory: string) { + const files = await readdir(this.buildDirectory(directory)) + + return files.length + } + buildWebVideoFilePath (fileUrl: string) { return this.buildDirectory(join('web-videos', basename(fileUrl))) } diff --git a/shared/server-commands/videos/videos-command.ts b/shared/server-commands/videos/videos-command.ts index 9602fa7da..6c38fa7ef 100644 --- a/shared/server-commands/videos/videos-command.ts +++ b/shared/server-commands/videos/videos-command.ts @@ -32,6 +32,7 @@ export type VideoEdit = Partial { - const { attributes, expectedStatus } = options + const { path, attributes, expectedStatus } = options let size = 0 let videoFilePath: string @@ -478,7 +480,15 @@ export class VideosCommand extends AbstractCommand { } // Do not check status automatically, we'll check it manually - const initializeSessionRes = await this.prepareResumableUpload({ ...options, expectedStatus: null, attributes, size, mimetype }) + const initializeSessionRes = await this.prepareResumableUpload({ + ...options, + + path, + expectedStatus: null, + attributes, + size, + mimetype + }) const initStatus = initializeSessionRes.status if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) { @@ -487,10 +497,23 @@ export class VideosCommand extends AbstractCommand { const pathUploadId = locationHeader.split('?')[1] - const result = await this.sendResumableChunks({ ...options, pathUploadId, videoFilePath, size }) + const result = await this.sendResumableChunks({ + ...options, + + path, + pathUploadId, + videoFilePath, + size + }) if (result.statusCode === HttpStatusCode.OK_200) { - await this.endResumableUpload({ ...options, expectedStatus: HttpStatusCode.NO_CONTENT_204, pathUploadId }) + await this.endResumableUpload({ + ...options, + + expectedStatus: HttpStatusCode.NO_CONTENT_204, + path, + pathUploadId + }) } return result.body?.video || result.body as any @@ -506,18 +529,19 @@ export class VideosCommand extends AbstractCommand { } async prepareResumableUpload (options: OverrideCommandOptions & { - attributes: VideoEdit + path: string + attributes: { fixture?: string } & { [id: string]: any } size: number mimetype: string originalName?: string lastModified?: number }) { - const { attributes, originalName, lastModified, size, mimetype } = options + const { path, attributes, originalName, lastModified, size, mimetype } = options - const path = '/api/v1/videos/upload-resumable' + const attaches = this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])) - return this.postUploadRequest({ + const uploadOptions = { ...options, path, @@ -538,11 +562,16 @@ export class VideosCommand extends AbstractCommand { implicitToken: true, defaultExpectedStatus: null - }) + } + + if (Object.keys(attaches).length === 0) return this.postBodyRequest(uploadOptions) + + return this.postUploadRequest(uploadOptions) } sendResumableChunks (options: OverrideCommandOptions & { pathUploadId: string + path: string videoFilePath: string size: number contentLength?: number @@ -550,6 +579,7 @@ export class VideosCommand extends AbstractCommand { digestBuilder?: (chunk: any) => string }) { const { + path, pathUploadId, videoFilePath, size, @@ -559,7 +589,6 @@ export class VideosCommand extends AbstractCommand { expectedStatus = HttpStatusCode.OK_200 } = options - const path = '/api/v1/videos/upload-resumable' let start = 0 const token = this.buildCommonRequestToken({ ...options, implicitToken: true }) @@ -610,12 +639,13 @@ export class VideosCommand extends AbstractCommand { } endResumableUpload (options: OverrideCommandOptions & { + path: string pathUploadId: string }) { return this.deleteRequest({ ...options, - path: '/api/v1/videos/upload-resumable', + path: options.path, rawQuery: options.pathUploadId, implicitToken: true, defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 @@ -657,6 +687,21 @@ export class VideosCommand extends AbstractCommand { // --------------------------------------------------------------------------- + replaceSourceFile (options: OverrideCommandOptions & { + videoId: number | string + fixture: string + }) { + return this.buildResumeUpload({ + ...options, + + path: '/api/v1/videos/' + options.videoId + '/source/replace-resumable', + attributes: { fixture: options.fixture }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + removeHLSPlaylist (options: OverrideCommandOptions & { videoId: number | string }) { diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 90aaebd26..654bd7461 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -2641,22 +2641,6 @@ paths: example: | **[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)** - '/api/v1/videos/{id}/source': - post: - summary: Get video source file metadata - operationId: getVideoSource - tags: - - Video - parameters: - - $ref: '#/components/parameters/idOrUUID' - responses: - '200': - description: successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/VideoSource' - '/api/v1/videos/{id}/views': post: summary: Notify user is watching a video @@ -2871,21 +2855,8 @@ paths: - Video - Video Upload parameters: - - name: X-Upload-Content-Length - in: header - schema: - type: number - example: 2469036 - required: true - description: Number of bytes that will be uploaded in subsequent requests. Set this value to the size of the file you are uploading. - - name: X-Upload-Content-Type - in: header - schema: - type: string - format: mimetype - example: video/mp4 - required: true - description: MIME type of the file that you are uploading. Depending on your instance settings, acceptable values might vary. + - $ref: '#/components/parameters/resumableUploadInitContentLengthHeader' + - $ref: '#/components/parameters/resumableUploadInitContentTypeHeader' requestBody: content: application/json: @@ -2924,36 +2895,9 @@ paths: - Video - Video Upload parameters: - - name: upload_id - in: query - required: true - description: | - Created session id to proceed with. If you didn't send chunks in the last hour, it is - not valid anymore and you need to initialize a new upload. - schema: - type: string - - name: Content-Range - in: header - schema: - type: string - example: bytes 0-262143/2469036 - required: true - description: | - Specifies the bytes in the file that the request is uploading. - - For example, a value of `bytes 0-262143/1000000` shows that the request is sending the first - 262144 bytes (256 x 1024) in a 2,469,036 byte file. - - name: Content-Length - in: header - schema: - type: number - example: 262144 - required: true - description: | - Size of the chunk that the request is sending. - - Remember that larger chunks are more efficient. PeerTube's web client uses chunks varying from - 1048576 bytes (~1MB) and increases or reduces size depending on connection health. + - $ref: '#/components/parameters/resumableUploadId' + - $ref: '#/components/parameters/resumableUploadChunkContentRangeHeader' + - $ref: '#/components/parameters/resumableUploadChunkContentLengthHeader' requestBody: content: application/octet-stream: @@ -3009,14 +2953,7 @@ paths: - Video - Video Upload parameters: - - name: upload_id - in: query - required: true - description: | - Created session id to proceed with. If you didn't send chunks in the last 12 hours, it is - not valid anymore and the upload session has already been deleted with its data ;-) - schema: - type: string + - $ref: '#/components/parameters/resumableUploadId' - name: Content-Length in: header required: true @@ -3286,6 +3223,140 @@ paths: schema: $ref: '#/components/schemas/LiveVideoSessionResponse' + '/api/v1/videos/{id}/source': + get: + summary: Get video source file metadata + operationId: getVideoSource + tags: + - Video + parameters: + - $ref: '#/components/parameters/idOrUUID' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/VideoSource' + + '/api/v1/videos/{id}/source/replace-resumable': + post: + summary: Initialize the resumable replacement of a video + description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to initialize the replacement of a video + operationId: replaceVideoSourceResumableInit + security: + - OAuth2: [] + tags: + - Video + - Video Upload + parameters: + - $ref: '#/components/parameters/resumableUploadInitContentLengthHeader' + - $ref: '#/components/parameters/resumableUploadInitContentTypeHeader' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/VideoReplaceSourceRequestResumable' + responses: + '200': + description: file already exists, send a [`resume`](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) request instead + '201': + description: created + headers: + Location: + schema: + type: string + format: url + Content-Length: + schema: + type: number + example: 0 + '413': + x-summary: video file too large, due to quota, absolute max file size or concurrent partial upload limit + description: | + Disambiguate via `type`: + - `max_file_size_reached` for the absolute file size limit + - `quota_reached` for quota limits whether daily or global + '415': + description: video type unsupported + put: + summary: Send chunk for the resumable replacement of a video + description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to continue, pause or resume the replacement of a video + operationId: replaceVideoSourceResumable + security: + - OAuth2: [] + tags: + - Video + - Video Upload + parameters: + - $ref: '#/components/parameters/resumableUploadId' + - $ref: '#/components/parameters/resumableUploadChunkContentRangeHeader' + - $ref: '#/components/parameters/resumableUploadChunkContentLengthHeader' + requestBody: + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + '204': + description: 'last chunk received: successful operation' + '308': + description: resume incomplete + headers: + Range: + schema: + type: string + example: bytes=0-262143 + Content-Length: + schema: + type: number + example: 0 + '403': + description: video didn't pass file replacement filter + '404': + description: replace upload not found + '409': + description: chunk doesn't match range + '422': + description: video unreadable + '429': + description: too many concurrent requests + '503': + description: upload is already being processed + headers: + 'Retry-After': + schema: + type: number + example: 300 + delete: + summary: Cancel the resumable replacement of a video + description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to cancel the replacement of a video + operationId: replaceVideoSourceResumableCancel + security: + - OAuth2: [] + tags: + - Video + - Video Upload + parameters: + - $ref: '#/components/parameters/resumableUploadId' + - name: Content-Length + in: header + required: true + schema: + type: number + example: 0 + responses: + '204': + description: source file replacement cancelled + headers: + Content-Length: + schema: + type: number + example: 0 + '404': + description: source file replacement not found + /api/v1/users/me/abuses: get: summary: List my abuses @@ -6640,6 +6711,58 @@ components: required: false schema: type: string + resumableUploadInitContentLengthHeader: + name: X-Upload-Content-Length + in: header + schema: + type: number + example: 2469036 + required: true + description: Number of bytes that will be uploaded in subsequent requests. Set this value to the size of the file you are uploading. + resumableUploadInitContentTypeHeader: + name: X-Upload-Content-Type + in: header + schema: + type: string + format: mimetype + example: video/mp4 + required: true + description: MIME type of the file that you are uploading. Depending on your instance settings, acceptable values might vary. + resumableUploadChunkContentRangeHeader: + name: Content-Range + in: header + schema: + type: string + example: bytes 0-262143/2469036 + required: true + description: | + Specifies the bytes in the file that the request is uploading. + + For example, a value of `bytes 0-262143/1000000` shows that the request is sending the first + 262144 bytes (256 x 1024) in a 2,469,036 byte file. + resumableUploadChunkContentLengthHeader: + name: Content-Length + in: header + schema: + type: number + example: 262144 + required: true + description: | + Size of the chunk that the request is sending. + + Remember that larger chunks are more efficient. PeerTube's web client uses chunks varying from + 1048576 bytes (~1MB) and increases or reduces size depending on connection health. + resumableUploadId: + name: upload_id + in: query + required: true + description: | + Created session id to proceed with. If you didn't send chunks in the last hour, it is + not valid anymore and you need to initialize a new upload. + schema: + type: string + + securitySchemes: OAuth2: description: | @@ -7209,6 +7332,11 @@ components: type: boolean downloadEnabled: type: boolean + inputFileUpdatedAt: + type: string + format: date-time + nullable: true + description: Latest input file update. Null if the file has never been replaced since the original upload trackerUrls: type: array items: @@ -7554,6 +7682,9 @@ components: properties: filename: type: string + createdAt: + type: string + format: date-time ActorImage: properties: path: @@ -8403,6 +8534,13 @@ components: $ref: '#/components/schemas/Video/properties/uuid' shortUUID: $ref: '#/components/schemas/Video/properties/shortUUID' + VideoReplaceSourceRequestResumable: + properties: + filename: + description: Video filename including extension + type: string + format: filename + example: what_is_peertube.mp4 CommentThreadResponse: properties: total: