From 5e2afe4290103bf0d54ae7b3e62781f2a00487c9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 5 Aug 2022 15:05:20 +0200 Subject: [PATCH] Limit import depending on transcoding resolutions --- scripts/create-transcoding-job.ts | 2 +- server/controllers/api/config.ts | 2 + server/controllers/api/videos/import.ts | 6 +- server/controllers/api/videos/transcoding.ts | 2 +- server/helpers/ffmpeg/ffprobe-utils.ts | 22 +++-- server/helpers/youtube-dl/youtube-dl-cli.ts | 28 ++++-- .../helpers/youtube-dl/youtube-dl-wrapper.ts | 10 +- server/lib/job-queue/handlers/video-import.ts | 9 +- .../job-queue/handlers/video-transcoding.ts | 2 +- server/lib/live/live-manager.ts | 4 +- server/lib/transcoding/transcoding.ts | 2 +- server/tests/api/videos/video-imports.ts | 98 ++++++++++++++++--- 12 files changed, 144 insertions(+), 43 deletions(-) diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index b7761597e..f8c0ed461 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts @@ -53,7 +53,7 @@ async function run () { if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { const resolutionsEnabled = options.resolution ? [ parseInt(options.resolution) ] - : computeResolutionsToTranscode({ inputResolution: maxResolution, type: 'vod', includeInputResolution: true }) + : computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false }) for (const resolution of resolutionsEnabled) { dataInput.push({ diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index ff2fa9d86..19bd2888c 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -10,6 +10,7 @@ import { CONFIG, reloadConfig } from '../../initializers/config' import { ClientHtml } from '../../lib/client-html' import { asyncMiddleware, authenticate, ensureUserHasRight, openapiOperationDoc } from '../../middlewares' import { customConfigUpdateValidator, ensureConfigIsEditable } from '../../middlewares/validators/config' +import { logger } from '@server/helpers/logger' const configRouter = express.Router() @@ -112,6 +113,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response) const data = customConfig() + logger.info('coucou', { data }) auditLogger.update( getAuditIdFromRes(res), new CustomConfigAuditView(data), diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 7576e77e7..b12953630 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -175,7 +175,11 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) const targetUrl = body.targetUrl const user = res.locals.oauth.token.User - const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) + const youtubeDL = new YoutubeDLWrapper( + targetUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) // Get video infos let youtubeDLInfo: YoutubeDLInfo diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts index 7d924c5ac..b2b71a870 100644 --- a/server/controllers/api/videos/transcoding.ts +++ b/server/controllers/api/videos/transcoding.ts @@ -32,7 +32,7 @@ async function createTranscoding (req: express.Request, res: express.Response) { const { resolution: maxResolution, audioStream } = await video.probeMaxQualityFile() const resolutions = await Hooks.wrapObject( - computeResolutionsToTranscode({ inputResolution: maxResolution, type: 'vod', includeInputResolution: true }), + computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false }), 'filter:transcoding.manual.resolutions-to-transcode.result', body ) diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts index 7bcd27665..c45f9ec99 100644 --- a/server/helpers/ffmpeg/ffprobe-utils.ts +++ b/server/helpers/ffmpeg/ffprobe-utils.ts @@ -91,11 +91,12 @@ async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { // --------------------------------------------------------------------------- function computeResolutionsToTranscode (options: { - inputResolution: number + input: number type: 'vod' | 'live' - includeInputResolution: boolean + includeInput: boolean + strictLower: boolean }) { - const { inputResolution, type, includeInputResolution } = options + const { input, type, includeInput, strictLower } = options const configResolutions = type === 'vod' ? CONFIG.TRANSCODING.RESOLUTIONS @@ -117,13 +118,18 @@ function computeResolutionsToTranscode (options: { ] for (const resolution of availableResolutions) { - if (configResolutions[resolution + 'p'] === true && inputResolution > resolution) { - resolutionsEnabled.add(resolution) - } + // Resolution not enabled + if (configResolutions[resolution + 'p'] !== true) continue + // Too big resolution for input file + if (input < resolution) continue + // We only want lower resolutions than input file + if (strictLower && input === resolution) continue + + resolutionsEnabled.add(resolution) } - if (includeInputResolution) { - resolutionsEnabled.add(inputResolution) + if (includeInput) { + resolutionsEnabled.add(input) } return Array.from(resolutionsEnabled) diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts index 728f096b5..13c990a1e 100644 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/helpers/youtube-dl/youtube-dl-cli.ts @@ -57,7 +57,7 @@ export class YoutubeDLCLI { } } - static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[]) { + static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) { /** * list of format selectors in order or preference * see https://github.com/ytdl-org/youtube-dl#format-selection @@ -69,18 +69,26 @@ export class YoutubeDLCLI { * * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 **/ - const resolution = enabledResolutions.length === 0 - ? VideoResolution.H_720P - : Math.max(...enabledResolutions) - return [ - `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 - `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 - `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3 - `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`, + let result: string[] = [] + + if (!useBestFormat) { + const resolution = enabledResolutions.length === 0 + ? VideoResolution.H_720P + : Math.max(...enabledResolutions) + + result = [ + `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 + `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 + `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case # + ] + } + + return result.concat([ + 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats 'best' // Ultimate fallback - ].join('/') + ]).join('/') } private constructor () { diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts index d585e9a95..176cf3b69 100644 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts @@ -21,7 +21,11 @@ const processOptions = { class YoutubeDLWrapper { - constructor (private readonly url: string = '', private readonly enabledResolutions: number[] = []) { + constructor ( + private readonly url: string, + private readonly enabledResolutions: number[], + private readonly useBestFormat: boolean + ) { } @@ -30,7 +34,7 @@ class YoutubeDLWrapper { const info = await youtubeDL.getInfo({ url: this.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions), + format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), additionalYoutubeDLArgs: youtubeDLArgs, processOptions }) @@ -80,7 +84,7 @@ class YoutubeDLWrapper { try { await youtubeDL.download({ url: this.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions), + format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), output: pathWithoutExtension, timeout, processOptions diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 40804e82e..4cde26aef 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -2,6 +2,7 @@ import { Job } from 'bull' import { move, remove, stat } from 'fs-extra' import { retryTransactionWrapper } from '@server/helpers/database-utils' import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' +import { CONFIG } from '@server/initializers/config' import { isPostImportVideoAccepted } from '@server/lib/moderation' import { generateWebTorrentVideoFilename } from '@server/lib/paths' import { Hooks } from '@server/lib/plugins/hooks' @@ -25,7 +26,7 @@ import { VideoResolution, VideoState } from '@shared/models' -import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg' import { logger } from '../../../helpers/logger' import { getSecureTorrentName } from '../../../helpers/utils' import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' @@ -80,7 +81,11 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul const options = { type: payload.type, videoImportId: videoImport.id } - const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod')) + const youtubeDL = new YoutubeDLWrapper( + videoImport.targetUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) return processFile( () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 890d34e3b..4e5e97919 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -265,7 +265,7 @@ async function createLowerResolutionsJobs (options: { // Create transcoding jobs if there are enabled resolutions const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ inputResolution: videoFileResolution, type: 'vod', includeInputResolution: false }), + computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true }), 'filter:transcoding.auto.resolutions-to-transcode.result', options ) diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 3ac57fa44..1410889a2 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -456,10 +456,10 @@ class LiveManager { } private buildAllResolutionsToTranscode (originResolution: number) { - const includeInputResolution = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED - ? computeResolutionsToTranscode({ inputResolution: originResolution, type: 'live', includeInputResolution }) + ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false }) : [] if (resolutionsEnabled.length === 0) { diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts index 3681de994..070c7ebda 100644 --- a/server/lib/transcoding/transcoding.ts +++ b/server/lib/transcoding/transcoding.ts @@ -366,7 +366,7 @@ async function generateHlsPlaylistCommon (options: { function buildOriginalFileResolution (inputResolution: number) { if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution) - const resolutions = computeResolutionsToTranscode({ inputResolution, type: 'vod', includeInputResolution: false }) + const resolutions = computeResolutionsToTranscode({ input: inputResolution, type: 'vod', includeInput: false, strictLower: false }) if (resolutions.length === 0) return toEven(inputResolution) return Math.max(...resolutions) diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index cf9f7d0cb..603e2d234 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -6,7 +6,7 @@ import { pathExists, readdir, remove } from 'fs-extra' import { join } from 'path' import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared' import { areHttpImportTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models' +import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models' import { cleanupTests, createMultipleServers, @@ -17,6 +17,7 @@ import { setDefaultVideoChannel, waitJobs } from '@shared/server-commands' +import { DeepPartial } from '@shared/typescript-utils' async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) { const videoHttp = await server.videos.get({ id: idHttp }) @@ -105,6 +106,16 @@ describe('Test video imports', function () { await setAccessTokensToServers(servers) await setDefaultVideoChannel(servers) + for (const server of servers) { + await server.config.updateExistingSubConfig({ + newConfig: { + transcoding: { + alwaysTranscodeOriginalResolution: false + } + } + }) + } + await doubleFollow(servers[0], servers[1]) }) @@ -306,10 +317,11 @@ describe('Test video imports', function () { it('Should import no HDR version on a HDR video', async function () { this.timeout(300_000) - const config = { + const config: DeepPartial = { transcoding: { enabled: true, resolutions: { + '0p': false, '144p': true, '240p': true, '360p': false, @@ -321,19 +333,9 @@ describe('Test video imports', function () { }, webtorrent: { enabled: true }, hls: { enabled: false } - }, - import: { - videos: { - http: { - enabled: true - }, - torrent: { - enabled: true - } - } } } - await servers[0].config.updateCustomSubConfig({ newConfig: config }) + await servers[0].config.updateExistingSubConfig({ newConfig: config }) const attributes = { name: 'hdr video', @@ -353,6 +355,76 @@ describe('Test video imports', function () { expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P) }) + it('Should not import resolution higher than enabled transcoding resolution', async function () { + this.timeout(300_000) + + const config: DeepPartial = { + transcoding: { + enabled: true, + resolutions: { + '0p': false, + '144p': true, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: false + } + } + await servers[0].config.updateExistingSubConfig({ newConfig: config }) + + const attributes = { + name: 'small resolution video', + targetUrl: FIXTURE_URLS.youtube, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = videoImported.uuid + + await waitJobs(servers) + + // test resolution + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.equal('small resolution video') + expect(video.files).to.have.lengthOf(1) + expect(video.files[0].resolution.id).to.equal(144) + }) + + it('Should import resolution higher than enabled transcoding resolution', async function () { + this.timeout(300_000) + + const config: DeepPartial = { + transcoding: { + alwaysTranscodeOriginalResolution: true + } + } + await servers[0].config.updateExistingSubConfig({ newConfig: config }) + + const attributes = { + name: 'bigger resolution video', + targetUrl: FIXTURE_URLS.youtube, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = videoImported.uuid + + await waitJobs(servers) + + // test resolution + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.equal('bigger resolution video') + + expect(video.files).to.have.lengthOf(2) + expect(video.files.find(f => f.resolution.id === 240)).to.exist + expect(video.files.find(f => f.resolution.id === 144)).to.exist + }) + it('Should import a peertube video', async function () { this.timeout(120_000)