diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts index 508055b85..fc4c40787 100644 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/helpers/youtube-dl/youtube-dl-cli.ts @@ -142,6 +142,11 @@ export class YoutubeDLCLI { }): Promise<{ upload_date: string, webpage_url: string }[]> { const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] + if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') { + // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel + additionalYoutubeDLArgs.push('--flat-playlist') + } + if (options.latestVideosCount !== undefined) { additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) } diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts index 2c3ba2feb..966b8df78 100644 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts @@ -1,5 +1,6 @@ import { move, pathExists, readdir, remove } from 'fs-extra' import { dirname, join } from 'path' +import { inspect } from 'util' import { CONFIG } from '@server/initializers/config' import { isVideoFileExtnameValid } from '../custom-validators/videos' import { logger, loggerTagsFactory } from '../logger' @@ -59,13 +60,9 @@ class YoutubeDLWrapper { processOptions }) - if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}`) + if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`) - return list.map(info => { - const infoBuilder = new YoutubeDLInfoBuilder(info) - - return infoBuilder.getInfo() - }) + return list.map(info => info.webpage_url) } async getSubtitles (): Promise { diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index eb5ca1703..3a81daac0 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts @@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config' import { buildYoutubeDLImport } from '@server/lib/video-import' import { UserModel } from '@server/models/user/user' import { VideoImportModel } from '@server/models/video/video-import' -import { MChannelAccountDefault, MChannelSync } from '@server/types/models' +import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' import { CreateJobArgument, JobQueue } from './job-queue' import { ServerConfigManager } from './server-config-manager' @@ -31,15 +31,7 @@ export async function synchronizeChannel (options: { CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION ) - const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) - - const targetUrls = infoList - .filter(videoInfo => { - if (!onlyAfter) return true - - return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime() - }) - .map(videoInfo => videoInfo.webpageUrl) + const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) logger.info( 'Fetched %d candidate URLs for sync channel %s.', @@ -58,10 +50,7 @@ export async function synchronizeChannel (options: { const children: CreateJobArgument[] = [] for (const targetUrl of targetUrls) { - if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { - logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl) - continue - } + if (await skipImport(channel, targetUrl, onlyAfter)) continue const { job } = await buildYoutubeDLImport({ user, @@ -86,3 +75,28 @@ export async function synchronizeChannel (options: { await JobQueue.Instance.createJobWithChildren(parent, children) } + +// --------------------------------------------------------------------------- + +async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { + if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { + logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl) + return true + } + + if (onlyAfter) { + const youtubeDL = new YoutubeDLWrapper( + targetUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) + + const videoInfo = await youtubeDL.getInfoForDownload() + + if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) { + return true + } + } + + return false +} diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 23790ba65..d47807a79 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -1021,7 +1021,7 @@ describe('Test multiple servers', function () { describe('With minimum parameters', function () { it('Should upload and propagate the video', async function () { - this.timeout(60000) + this.timeout(120000) const path = '/api/v1/videos/upload'