forks/peertube
1
0
Fork 0
Code Releases Activity

Limit import depending on transcoding resolutions

Browse source
This commit is contained in:
Chocobozzz 2022-08-05 15:05:20 +02:00
parent 64fd6158fd
commit 5e2afe4290
No known key found for this signature in database
GPG key ID: 583A612D890159BE
12 changed files with 144 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
scripts/create-transcoding-job.ts
View file

@ -53,7 +53,7 @@ async function run () {
if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
const resolutionsEnabled = options.resolution
? [ parseInt(options.resolution) ]
: computeResolutionsToTranscode({ inputResolution: maxResolution, type: 'vod', includeInputResolution: true })
: computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false })
for (const resolution of resolutionsEnabled) {
dataInput.push({

2
server/controllers/api/config.ts
View file

@ -10,6 +10,7 @@ import { CONFIG, reloadConfig } from '../../initializers/config'
import { ClientHtml } from '../../lib/client-html'
import { asyncMiddleware, authenticate, ensureUserHasRight, openapiOperationDoc } from '../../middlewares'
import { customConfigUpdateValidator, ensureConfigIsEditable } from '../../middlewares/validators/config'
import { logger } from '@server/helpers/logger'
const configRouter = express.Router()
@ -112,6 +113,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response)
const data = customConfig()
logger.info('coucou', { data })
auditLogger.update(
getAuditIdFromRes(res),
new CustomConfigAuditView(data),

6
server/controllers/api/videos/import.ts
View file

@ -175,7 +175,11 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
const targetUrl = body.targetUrl
const user = res.locals.oauth.token.User
const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
const youtubeDL = new YoutubeDLWrapper(
targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
// Get video infos
let youtubeDLInfo: YoutubeDLInfo

2
server/controllers/api/videos/transcoding.ts
View file

@ -32,7 +32,7 @@ async function createTranscoding (req: express.Request, res: express.Response) {
const { resolution: maxResolution, audioStream } = await video.probeMaxQualityFile()
const resolutions = await Hooks.wrapObject(
computeResolutionsToTranscode({ inputResolution: maxResolution, type: 'vod', includeInputResolution: true }),
computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false }),
'filter:transcoding.manual.resolutions-to-transcode.result',
body
)

20
server/helpers/ffmpeg/ffprobe-utils.ts
View file

@ -91,11 +91,12 @@ async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
// ---------------------------------------------------------------------------
function computeResolutionsToTranscode (options: {
inputResolution: number
input: number
type: 'vod' | 'live'
includeInputResolution: boolean
includeInput: boolean
strictLower: boolean
}) {
const { inputResolution, type, includeInputResolution } = options
const { input, type, includeInput, strictLower } = options
const configResolutions = type === 'vod'
? CONFIG.TRANSCODING.RESOLUTIONS
@ -117,13 +118,18 @@ function computeResolutionsToTranscode (options: {
]
for (const resolution of availableResolutions) {
if (configResolutions[resolution + 'p'] === true && inputResolution > resolution) {
// Resolution not enabled
if (configResolutions[resolution + 'p'] !== true) continue
// Too big resolution for input file
if (input < resolution) continue
// We only want lower resolutions than input file
if (strictLower && input === resolution) continue
resolutionsEnabled.add(resolution)
}
}
if (includeInputResolution) {
resolutionsEnabled.add(inputResolution)
if (includeInput) {
resolutionsEnabled.add(input)
}
return Array.from(resolutionsEnabled)

18
server/helpers/youtube-dl/youtube-dl-cli.ts
View file

@ -57,7 +57,7 @@ export class YoutubeDLCLI {
}
}
static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[]) {
static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) {
/**
* list of format selectors in order or preference
* see https://github.com/ytdl-org/youtube-dl#format-selection
@ -69,18 +69,26 @@ export class YoutubeDLCLI {
*
* in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
**/
let result: string[] = []
if (!useBestFormat) {
const resolution = enabledResolutions.length === 0
? VideoResolution.H_720P
: Math.max(...enabledResolutions)
return [
result = [
`bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
`bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
`bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
`bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
`bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
]
}
return result.concat([
'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
'best' // Ultimate fallback
].join('/')
]).join('/')
}
private constructor () {

10
server/helpers/youtube-dl/youtube-dl-wrapper.ts
View file

@ -21,7 +21,11 @@ const processOptions = {
class YoutubeDLWrapper {
constructor (private readonly url: string = '', private readonly enabledResolutions: number[] = []) {
constructor (
private readonly url: string,
private readonly enabledResolutions: number[],
private readonly useBestFormat: boolean
) {
}
@ -30,7 +34,7 @@ class YoutubeDLWrapper {
const info = await youtubeDL.getInfo({
url: this.url,
format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
additionalYoutubeDLArgs: youtubeDLArgs,
processOptions
})
@ -80,7 +84,7 @@ class YoutubeDLWrapper {
try {
await youtubeDL.download({
url: this.url,
format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
output: pathWithoutExtension,
timeout,
processOptions

9
server/lib/job-queue/handlers/video-import.ts
View file

@ -2,6 +2,7 @@ import { Job } from 'bull'
import { move, remove, stat } from 'fs-extra'
import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
import { CONFIG } from '@server/initializers/config'
import { isPostImportVideoAccepted } from '@server/lib/moderation'
import { generateWebTorrentVideoFilename } from '@server/lib/paths'
import { Hooks } from '@server/lib/plugins/hooks'
@ -25,7 +26,7 @@ import {
VideoResolution,
VideoState
} from '@shared/models'
import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg'
import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg'
import { logger } from '../../../helpers/logger'
import { getSecureTorrentName } from '../../../helpers/utils'
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
@ -80,7 +81,11 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul
const options = { type: payload.type, videoImportId: videoImport.id }
const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
const youtubeDL = new YoutubeDLWrapper(
videoImport.targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
return processFile(
() => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),

2
server/lib/job-queue/handlers/video-transcoding.ts
View file

@ -265,7 +265,7 @@ async function createLowerResolutionsJobs (options: {
// Create transcoding jobs if there are enabled resolutions
const resolutionsEnabled = await Hooks.wrapObject(
computeResolutionsToTranscode({ inputResolution: videoFileResolution, type: 'vod', includeInputResolution: false }),
computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true }),
'filter:transcoding.auto.resolutions-to-transcode.result',
options
)

4
server/lib/live/live-manager.ts
View file

@ -456,10 +456,10 @@ class LiveManager {
}
private buildAllResolutionsToTranscode (originResolution: number) {
const includeInputResolution = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
? computeResolutionsToTranscode({ inputResolution: originResolution, type: 'live', includeInputResolution })
? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false })
: []
if (resolutionsEnabled.length === 0) {

2
server/lib/transcoding/transcoding.ts
View file

@ -366,7 +366,7 @@ async function generateHlsPlaylistCommon (options: {
function buildOriginalFileResolution (inputResolution: number) {
if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution)
const resolutions = computeResolutionsToTranscode({ inputResolution, type: 'vod', includeInputResolution: false })
const resolutions = computeResolutionsToTranscode({ input: inputResolution, type: 'vod', includeInput: false, strictLower: false })
if (resolutions.length === 0) return toEven(inputResolution)
return Math.max(...resolutions)

98
server/tests/api/videos/video-imports.ts
View file

@ -6,7 +6,7 @@ import { pathExists, readdir, remove } from 'fs-extra'
import { join } from 'path'
import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
@ -17,6 +17,7 @@ import {
setDefaultVideoChannel,
waitJobs
} from '@shared/server-commands'
import { DeepPartial } from '@shared/typescript-utils'
async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
const videoHttp = await server.videos.get({ id: idHttp })
@ -105,6 +106,16 @@ describe('Test video imports', function () {
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
for (const server of servers) {
await server.config.updateExistingSubConfig({
newConfig: {
transcoding: {
alwaysTranscodeOriginalResolution: false
}
}
})
}
await doubleFollow(servers[0], servers[1])
})
@ -306,10 +317,11 @@ describe('Test video imports', function () {
it('Should import no HDR version on a HDR video', async function () {
this.timeout(300_000)
const config = {
const config: DeepPartial<CustomConfig> = {
transcoding: {
enabled: true,
resolutions: {
'0p': false,
'144p': true,
'240p': true,
'360p': false,
@ -321,19 +333,9 @@ describe('Test video imports', function () {
},
webtorrent: { enabled: true },
hls: { enabled: false }
},
import: {
videos: {
http: {
enabled: true
},
torrent: {
enabled: true
}
}
}
}
await servers[0].config.updateCustomSubConfig({ newConfig: config })
await servers[0].config.updateExistingSubConfig({ newConfig: config })
const attributes = {
name: 'hdr video',
@ -353,6 +355,76 @@ describe('Test video imports', function () {
expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
})
it('Should not import resolution higher than enabled transcoding resolution', async function () {
this.timeout(300_000)
const config: DeepPartial<CustomConfig> = {
transcoding: {
enabled: true,
resolutions: {
'0p': false,
'144p': true,
'240p': false,
'360p': false,
'480p': false,
'720p': false,
'1080p': false,
'1440p': false,
'2160p': false
},
alwaysTranscodeOriginalResolution: false
}
}
await servers[0].config.updateExistingSubConfig({ newConfig: config })
const attributes = {
name: 'small resolution video',
targetUrl: FIXTURE_URLS.youtube,
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC
}
const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
const videoUUID = videoImported.uuid
await waitJobs(servers)
// test resolution
const video = await servers[0].videos.get({ id: videoUUID })
expect(video.name).to.equal('small resolution video')
expect(video.files).to.have.lengthOf(1)
expect(video.files[0].resolution.id).to.equal(144)
})
it('Should import resolution higher than enabled transcoding resolution', async function () {
this.timeout(300_000)
const config: DeepPartial<CustomConfig> = {
transcoding: {
alwaysTranscodeOriginalResolution: true
}
}
await servers[0].config.updateExistingSubConfig({ newConfig: config })
const attributes = {
name: 'bigger resolution video',
targetUrl: FIXTURE_URLS.youtube,
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC
}
const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
const videoUUID = videoImported.uuid
await waitJobs(servers)
// test resolution
const video = await servers[0].videos.get({ id: videoUUID })
expect(video.name).to.equal('bigger resolution video')
expect(video.files).to.have.lengthOf(2)
expect(video.files.find(f => f.resolution.id === 240)).to.exist
expect(video.files.find(f => f.resolution.id === 144)).to.exist
})
it('Should import a peertube video', async function () {
this.timeout(120_000)