From 6a4905602636afd6650c9e6f4d0fcc2105d91100 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 3 May 2023 15:17:11 +0200 Subject: [PATCH] Add TMP persistent directory To store files that must be preserved between peertube restarts --- config/default.yaml | 1 + config/production.yaml.example | 1 + config/test-1.yaml | 1 + config/test-2.yaml | 1 + config/test-3.yaml | 1 + config/test-4.yaml | 1 + config/test-5.yaml | 1 + config/test-6.yaml | 1 + server/controllers/api/videos/studio.ts | 40 +++++-- server/initializers/config.ts | 1 + .../handlers/video-studio-edition.ts | 100 ++++++++++-------- server/lib/video-studio.ts | 23 +++- .../validators/videos/video-studio.ts | 6 +- server/tests/api/transcoding/video-studio.ts | 25 ++++- server/tests/shared/directories.ts | 5 + shared/server-commands/server/server.ts | 1 + .../docker/production/config/production.yaml | 1 + 17 files changed, 148 insertions(+), 62 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 986b2e999..f3f29ecb9 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -120,6 +120,7 @@ defaults: # From the project root directory storage: tmp: 'storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... + tmp_persistent: 'storage/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts bin: 'storage/bin/' avatars: 'storage/avatars/' videos: 'storage/videos/' diff --git a/config/production.yaml.example b/config/production.yaml.example index bd01375cd..ea6d77306 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -118,6 +118,7 @@ defaults: # From the project root directory storage: tmp: '/var/www/peertube/storage/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... + tmp_persistent: '/var/www/peertube/storage/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts bin: '/var/www/peertube/storage/bin/' avatars: '/var/www/peertube/storage/avatars/' videos: '/var/www/peertube/storage/videos/' diff --git a/config/test-1.yaml b/config/test-1.yaml index 1d1020214..7b62e3d0c 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -10,6 +10,7 @@ database: # From the project root directory storage: tmp: 'test1/tmp/' + tmp_persistent: 'test1/tmp-persistent/' bin: 'test1/bin/' avatars: 'test1/avatars/' videos: 'test1/videos/' diff --git a/config/test-2.yaml b/config/test-2.yaml index d155b017d..ba36369a6 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -10,6 +10,7 @@ database: # From the project root directory storage: tmp: 'test2/tmp/' + tmp_persistent: 'test2/tmp-persistent/' bin: 'test2/bin/' avatars: 'test2/avatars/' videos: 'test2/videos/' diff --git a/config/test-3.yaml b/config/test-3.yaml index 50acf613a..6adec7953 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -10,6 +10,7 @@ database: # From the project root directory storage: tmp: 'test3/tmp/' + tmp_persistent: 'test3/tmp-persistent/' bin: 'test3/bin/' avatars: 'test3/avatars/' videos: 'test3/videos/' diff --git a/config/test-4.yaml b/config/test-4.yaml index 615e288b3..f042aee46 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -10,6 +10,7 @@ database: # From the project root directory storage: tmp: 'test4/tmp/' + tmp_persistent: 'test4/tmp-persistent/' bin: 'test4/bin/' avatars: 'test4/avatars/' videos: 'test4/videos/' diff --git a/config/test-5.yaml b/config/test-5.yaml index 447e3862a..ad90fec04 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -10,6 +10,7 @@ database: # From the project root directory storage: tmp: 'test5/tmp/' + tmp_persistent: 'test5/tmp-persistent/' bin: 'test5/bin/' avatars: 'test5/avatars/' videos: 'test5/videos/' diff --git a/config/test-6.yaml b/config/test-6.yaml index c077d7e38..a579f1f01 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -10,6 +10,7 @@ database: # From the project root directory storage: tmp: 'test6/tmp/' + tmp_persistent: 'test6/tmp-persistent/' bin: 'test6/bin/' avatars: 'test6/avatars/' videos: 'test6/videos/' diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts index 6667532bf..2ccb2fb89 100644 --- a/server/controllers/api/videos/studio.ts +++ b/server/controllers/api/videos/studio.ts @@ -1,8 +1,12 @@ +import Bluebird from 'bluebird' import express from 'express' +import { move } from 'fs-extra' +import { basename, join } from 'path' import { createAnyReqFiles } from '@server/helpers/express-utils' +import { CONFIG } from '@server/initializers/config' import { MIMETYPES } from '@server/initializers/constants' import { JobQueue } from '@server/lib/job-queue' -import { buildTaskFileFieldname, getTaskFile } from '@server/lib/video-studio' +import { buildTaskFileFieldname, getTaskFileFromReq } from '@server/lib/video-studio' import { HttpStatusCode, VideoState, @@ -68,7 +72,7 @@ async function createEditionTasks (req: express.Request, res: express.Response) const payload = { videoUUID: video.uuid, - tasks: body.tasks.map((t, i) => buildTaskPayload(t, i, files)) + tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) } JobQueue.Instance.createJobAsync({ type: 'video-studio-edition', payload }) @@ -77,7 +81,11 @@ async function createEditionTasks (req: express.Request, res: express.Response) } const taskPayloadBuilders: { - [id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => VideoStudioTaskPayload + [id in VideoStudioTask['name']]: ( + task: VideoStudioTask, + indice?: number, + files?: Express.Multer.File[] + ) => Promise } = { 'add-intro': buildIntroOutroTask, 'add-outro': buildIntroOutroTask, @@ -85,34 +93,46 @@ const taskPayloadBuilders: { 'add-watermark': buildWatermarkTask } -function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): VideoStudioTaskPayload { +function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise { return taskPayloadBuilders[task.name](task, indice, files) } -function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { +async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { + const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) + return { name: task.name, options: { - file: getTaskFile(files, indice).path + file: destination } } } function buildCutTask (task: VideoStudioTaskCut) { - return { + return Promise.resolve({ name: task.name, options: { start: task.options.start, end: task.options.end } - } + }) } -function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { +async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { + const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) + return { name: task.name, options: { - file: getTaskFile(files, indice).path + file: destination } } } + +async function moveStudioFileToPersistentTMP (file: string) { + const destination = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, basename(file)) + + await move(file, destination) + + return destination +} diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 699dd4704..f2d8f99b5 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -98,6 +98,7 @@ const CONFIG = { STORAGE: { TMP_DIR: buildPath(config.get('storage.tmp')), + TMP_PERSISTENT_DIR: buildPath(config.get('storage.tmp_persistent')), BIN_DIR: buildPath(config.get('storage.bin')), ACTOR_IMAGES: buildPath(config.get('storage.avatars')), LOG_DIR: buildPath(config.get('storage.logs')), diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index fbb55a388..5e8dd4f51 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts @@ -12,7 +12,7 @@ import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default import { isAbleToUploadVideo } from '@server/lib/user' import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' import { VideoPathManager } from '@server/lib/video-path-manager' -import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' +import { approximateIntroOutroAdditionalSize, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' import { UserModel } from '@server/models/user/user' import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' @@ -39,63 +39,73 @@ async function processVideoStudioEdition (job: Job) { logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) - const video = await VideoModel.loadFull(payload.videoUUID) + try { + const video = await VideoModel.loadFull(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) - return undefined - } + // No video, maybe deleted? + if (!video) { + logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) - await checkUserQuotaOrThrow(video, payload) - - const inputFile = video.getMaxQualityFile() - - const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { - let tmpInputFilePath: string - let outputPath: string - - for (const task of payload.tasks) { - const outputFilename = buildUUID() + inputFile.extname - outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) - - await processTask({ - inputPath: tmpInputFilePath ?? originalFilePath, - video, - outputPath, - task, - lTags - }) - - if (tmpInputFilePath) await remove(tmpInputFilePath) - - // For the next iteration - tmpInputFilePath = outputPath + await safeCleanupStudioTMPFiles(payload) + return undefined } - return outputPath - }) + await checkUserQuotaOrThrow(video, payload) - logger.info('Video edition ended for video %s.', video.uuid, lTags) + const inputFile = video.getMaxQualityFile() - const newFile = await buildNewFile(video, editionResultPath) + const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { + let tmpInputFilePath: string + let outputPath: string - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) - await move(editionResultPath, outputPath) + for (const task of payload.tasks) { + const outputFilename = buildUUID() + inputFile.extname + outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) - await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) - await removeAllFiles(video, newFile) + await processTask({ + inputPath: tmpInputFilePath ?? originalFilePath, + video, + outputPath, + task, + lTags + }) - await newFile.save() + if (tmpInputFilePath) await remove(tmpInputFilePath) - video.duration = await getVideoStreamDuration(outputPath) - await video.save() + // For the next iteration + tmpInputFilePath = outputPath + } - await federateVideoIfNeeded(video, false, undefined) + return outputPath + }) - const user = await UserModel.loadByVideoId(video.id) + logger.info('Video edition ended for video %s.', video.uuid, lTags) - await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) + const newFile = await buildNewFile(video, editionResultPath) + + const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) + await move(editionResultPath, outputPath) + + await safeCleanupStudioTMPFiles(payload) + + await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) + await removeAllFiles(video, newFile) + + await newFile.save() + + video.duration = await getVideoStreamDuration(outputPath) + await video.save() + + await federateVideoIfNeeded(video, false, undefined) + + const user = await UserModel.loadByVideoId(video.id) + + await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) + } catch (err) { + await safeCleanupStudioTMPFiles(payload) + + throw err + } } // --------------------------------------------------------------------------- diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index b392bdb00..beda326a0 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts @@ -1,15 +1,31 @@ +import { logger } from '@server/helpers/logger' import { MVideoFullLight } from '@server/types/models' import { getVideoStreamDuration } from '@shared/ffmpeg' -import { VideoStudioTask } from '@shared/models' +import { VideoStudioEditionPayload, VideoStudioTask } from '@shared/models' +import { remove } from 'fs-extra' function buildTaskFileFieldname (indice: number, fieldName = 'file') { return `tasks[${indice}][options][${fieldName}]` } -function getTaskFile (files: Express.Multer.File[], indice: number, fieldName = 'file') { +function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) } +async function safeCleanupStudioTMPFiles (payload: VideoStudioEditionPayload) { + for (const task of payload.tasks) { + try { + if (task.name === 'add-intro' || task.name === 'add-outro') { + await remove(task.options.file) + } else if (task.name === 'add-watermark') { + await remove(task.options.file) + } + } catch (err) { + logger.error('Cannot remove studio file', { err }) + } + } +} + async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, tasks: VideoStudioTask[], fileFinder: (i: number) => string) { let additionalDuration = 0 @@ -28,5 +44,6 @@ async function approximateIntroOutroAdditionalSize (video: MVideoFullLight, task export { approximateIntroOutroAdditionalSize, buildTaskFileFieldname, - getTaskFile + getTaskFileFromReq, + safeCleanupStudioTMPFiles } diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts index 4397e887e..7a68f88e5 100644 --- a/server/middlewares/validators/videos/video-studio.ts +++ b/server/middlewares/validators/videos/video-studio.ts @@ -9,7 +9,7 @@ import { } from '@server/helpers/custom-validators/video-studio' import { cleanUpReqFiles } from '@server/helpers/express-utils' import { CONFIG } from '@server/initializers/config' -import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' +import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio' import { isAudioFile } from '@shared/ffmpeg' import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' @@ -49,7 +49,7 @@ const videoStudioAddEditionValidator = [ } if (task.name === 'add-intro' || task.name === 'add-outro') { - const filePath = getTaskFile(files, i).path + const filePath = getTaskFileFromReq(files, i).path // Our concat filter needs a video stream if (await isAudioFile(filePath)) { @@ -79,7 +79,7 @@ const videoStudioAddEditionValidator = [ if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) // Try to make an approximation of bytes added by the intro/outro - const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFile(files, i).path) + const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path) if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) return next() diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index ab08e8fb6..30f72e6e9 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { expectStartWith } from '@server/tests/shared' +import { checkPersistentTmpIsEmpty, expectStartWith } from '@server/tests/shared' import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' import { VideoStudioTask } from '@shared/models' import { @@ -356,6 +356,29 @@ describe('Test video studio', function () { }) }) + describe('Server restart', function () { + + it('Should still be able to run video edition after a server restart', async function () { + this.timeout(240_000) + + await renewVideo() + await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() }) + + await servers[0].kill() + await servers[0].run() + + await waitJobs(servers) + + for (const server of servers) { + await checkDuration(server, 9) + } + }) + + it('Should have an empty persistent tmp directory', async function () { + await checkPersistentTmpIsEmpty(servers[0]) + }) + }) + after(async function () { await cleanupTests(servers) }) diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index 90d534a06..a614cef7c 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts @@ -12,6 +12,10 @@ async function checkTmpIsEmpty (server: PeerTubeServer) { } } +async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { + await checkDirectoryIsEmpty(server, 'tmp-persistent') +} + async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { const directoryPath = server.getDirectoryPath(directory) @@ -26,5 +30,6 @@ async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, export { checkTmpIsEmpty, + checkPersistentTmpIsEmpty, checkDirectoryIsEmpty } diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts index f68b81367..c8f6a0c5b 100644 --- a/shared/server-commands/server/server.ts +++ b/shared/server-commands/server/server.ts @@ -364,6 +364,7 @@ export class PeerTubeServer { }, storage: { tmp: this.getDirectoryPath('tmp') + '/', + tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/', bin: this.getDirectoryPath('bin') + '/', avatars: this.getDirectoryPath('avatars') + '/', videos: this.getDirectoryPath('videos') + '/', diff --git a/support/docker/production/config/production.yaml b/support/docker/production/config/production.yaml index f1d698580..e3f6247d8 100644 --- a/support/docker/production/config/production.yaml +++ b/support/docker/production/config/production.yaml @@ -44,6 +44,7 @@ redis: # From the project root directory storage: tmp: '../data/tmp/' # Use to download data (imports etc), store uploaded files before and during processing... + tmp_persistent: '../data/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts bin: '../data/bin/' avatars: '../data/avatars/' videos: '../data/videos/'