1
0
Fork 0

Optimize fetching playlist urls

This commit is contained in:
Chocobozzz 2022-09-07 17:30:21 +02:00
parent d4d9bbc6f2
commit e9fc9e03c1
No known key found for this signature in database
GPG key ID: 583A612D890159BE
4 changed files with 37 additions and 21 deletions

View file

@ -142,6 +142,11 @@ export class YoutubeDLCLI {
}): Promise<{ upload_date: string, webpage_url: string }[]> {
const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
// Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
additionalYoutubeDLArgs.push('--flat-playlist')
}
if (options.latestVideosCount !== undefined) {
additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
}

View file

@ -1,5 +1,6 @@
import { move, pathExists, readdir, remove } from 'fs-extra'
import { dirname, join } from 'path'
import { inspect } from 'util'
import { CONFIG } from '@server/initializers/config'
import { isVideoFileExtnameValid } from '../custom-validators/videos'
import { logger, loggerTagsFactory } from '../logger'
@ -59,13 +60,9 @@ class YoutubeDLWrapper {
processOptions
})
if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}`)
if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
return list.map(info => {
const infoBuilder = new YoutubeDLInfoBuilder(info)
return infoBuilder.getInfo()
})
return list.map(info => info.webpage_url)
}
async getSubtitles (): Promise<YoutubeDLSubs> {

View file

@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
import { buildYoutubeDLImport } from '@server/lib/video-import'
import { UserModel } from '@server/models/user/user'
import { VideoImportModel } from '@server/models/video/video-import'
import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
import { CreateJobArgument, JobQueue } from './job-queue'
import { ServerConfigManager } from './server-config-manager'
@ -31,15 +31,7 @@ export async function synchronizeChannel (options: {
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
const targetUrls = infoList
.filter(videoInfo => {
if (!onlyAfter) return true
return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
})
.map(videoInfo => videoInfo.webpageUrl)
const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
logger.info(
'Fetched %d candidate URLs for sync channel %s.',
@ -58,10 +50,7 @@ export async function synchronizeChannel (options: {
const children: CreateJobArgument[] = []
for (const targetUrl of targetUrls) {
if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
continue
}
if (await skipImport(channel, targetUrl, onlyAfter)) continue
const { job } = await buildYoutubeDLImport({
user,
@ -86,3 +75,28 @@ export async function synchronizeChannel (options: {
await JobQueue.Instance.createJobWithChildren(parent, children)
}
// ---------------------------------------------------------------------------
async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
return true
}
if (onlyAfter) {
const youtubeDL = new YoutubeDLWrapper(
targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
const videoInfo = await youtubeDL.getInfoForDownload()
if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
return true
}
}
return false
}

View file

@ -1021,7 +1021,7 @@ describe('Test multiple servers', function () {
describe('With minimum parameters', function () {
it('Should upload and propagate the video', async function () {
this.timeout(60000)
this.timeout(120000)
const path = '/api/v1/videos/upload'