From 805b8619c1b8c56e08c927d93293c06e1075b50d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 19 Jan 2021 16:28:55 +0100 Subject: [PATCH] Fix (again) youtube import --- server/controllers/api/videos/import.ts | 3 +- server/helpers/utils.ts | 16 ++-- server/helpers/youtube-dl.ts | 83 ++++++++++--------- server/lib/job-queue/handlers/video-import.ts | 2 +- shared/models/server/job.model.ts | 2 - 5 files changed, 53 insertions(+), 53 deletions(-) diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 9702e219a..96499dffb 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -220,8 +220,7 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) videoImportId: videoImport.id, generateThumbnail: !thumbnailModel, generatePreview: !previewModel, - fileExt: `.${youtubeDLInfo.ext || 'mp4'}`, - mergeExt: youtubeDLInfo.mergeExt ? `.${youtubeDLInfo.mergeExt}` : '' + fileExt: `.${youtubeDLInfo.ext || 'mp4'}` } await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload }) diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index ad3b7949e..0545e8996 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -1,11 +1,10 @@ +import { remove } from 'fs-extra' +import { Instance as ParseTorrent } from 'parse-torrent' +import { join } from 'path' import { ResultList } from '../../shared' +import { CONFIG } from '../initializers/config' import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' -import { join } from 'path' -import { Instance as ParseTorrent } from 'parse-torrent' -import { remove } from 'fs-extra' -import { CONFIG } from '../initializers/config' -import { isVideoFileExtnameValid } from './custom-validators/videos' function deleteFileAsync (path: string) { remove(path) @@ -31,16 +30,11 @@ function getFormattedObjects> (objects: } as ResultList } -function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) { +function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') { const id = typeof target === 'string' ? target : target.infoHash - let extension = '.mp4' - if (extensionArg && isVideoFileExtnameValid(extensionArg)) { - extension = extensionArg - } - const hash = sha256(id) return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) } diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 72d457bc1..6b9d8a5f7 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -1,15 +1,16 @@ -import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' -import { logger } from './logger' -import { generateVideoImportTmpPath } from './utils' -import { getEnabledResolutions } from '../lib/video-transcoding' -import { join } from 'path' -import { peertubeTruncate, root } from './core-utils' -import { ensureDir, remove, writeFile } from 'fs-extra' -import * as request from 'request' import { createWriteStream } from 'fs' +import { ensureDir, pathExists, remove, writeFile } from 'fs-extra' +import { join } from 'path' +import * as request from 'request' import { CONFIG } from '@server/initializers/config' import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' import { VideoResolution } from '../../shared/models/videos' +import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' +import { getEnabledResolutions } from '../lib/video-transcoding' +import { peertubeTruncate, root } from './core-utils' +import { isVideoFileExtnameValid } from './custom-validators/videos' +import { logger } from './logger' +import { generateVideoImportTmpPath } from './utils' export type YoutubeDLInfo = { name?: string @@ -21,7 +22,6 @@ export type YoutubeDLInfo = { tags?: string[] thumbnailUrl?: string ext?: string - mergeExt?: string originallyPublishedAt?: Date } @@ -51,16 +51,6 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise youtubeDL.getInfo(url, args, processOptions, (err, info) => { if (err) return rej(err) if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) - if (info.format_id?.includes('+')) { - // this is a merge format and its extension will be appended - if (info.ext === 'mp4') { - info.mergeExt = 'mp4' - } else if (info.ext === 'webm') { - info.mergeExt = 'webm' - } else { - info.mergeExt = 'mkv' - } - } const obj = buildVideoInfo(normalizeObject(info)) if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' @@ -133,18 +123,15 @@ function getYoutubeDLVideoFormat () { ].join('/') } -function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) { - let path = generateVideoImportTmpPath(url, extension) - - path = mergeExtension - ? path.replace(new RegExp(`${extension}$`), mergeExtension) - : path +function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) { + // Leave empty the extension, youtube-dl will add it + const pathWithoutExtension = generateVideoImportTmpPath(url, '') let timer - logger.info('Importing youtubeDL video %s to %s', url, path) + logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension) - let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ] + let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ] options = wrapWithProxyOptions(options) if (process.env.FFMPEG_PATH) { @@ -156,26 +143,33 @@ function downloadYoutubeDLVideo (url: string, extension: string, timeout: number return new Promise((res, rej) => { safeGetYoutubeDL() .then(youtubeDL => { - youtubeDL.exec(url, options, processOptions, err => { + youtubeDL.exec(url, options, processOptions, async err => { clearTimeout(timer) - if (err) { - remove(path) - .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) + try { + const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt) + if (err) { + remove(path) + .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) + + return rej(err) + } + + return res(path) + } catch (err) { return rej(err) } - - return res(path) }) timer = setTimeout(() => { const err = new Error('YoutubeDL download timeout.') - remove(path) + guessVideoPathWithExtension(pathWithoutExtension, fileExt) + .then(path => remove(path)) .finally(() => rej(err)) .catch(err => { - logger.error('Cannot remove %s in youtubeDL timeout.', path, { err }) + logger.error('Cannot remove file in youtubeDL timeout.', { err }) return rej(err) }) }, timeout) @@ -294,6 +288,22 @@ export { // --------------------------------------------------------------------------- +async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { + if (!isVideoFileExtnameValid(sourceExt)) { + throw new Error('Invalid video extension ' + sourceExt) + } + + const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] + + for (const extension of extensions) { + const path = tmpPath + extension + + if (await pathExists(path)) return path + } + + throw new Error('Cannot guess path of ' + tmpPath) +} + function normalizeObject (obj: any) { const newObj: any = {} @@ -324,8 +334,7 @@ function buildVideoInfo (obj: any): YoutubeDLInfo { tags: getTags(obj.tags), thumbnailUrl: obj.thumbnail || undefined, originallyPublishedAt: buildOriginallyPublishedAt(obj), - ext: obj.ext, - mergeExt: obj.mergeExt + ext: obj.ext } } diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index db3112418..c1d1866b0 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -80,7 +80,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub } return processFile( - () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT, payload.mergeExt), + () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options ) diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 2af2a25a6..f9a6250c9 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -80,9 +80,7 @@ export type VideoImportYoutubeDLPayload = { generateThumbnail: boolean generatePreview: boolean - fileExt?: string - mergeExt?: string } export type VideoImportTorrentPayload = { type: VideoImportTorrentPayloadType