Optimize fetching playlist urls
This commit is contained in:
parent
d4d9bbc6f2
commit
e9fc9e03c1
4 changed files with 37 additions and 21 deletions
|
@ -142,6 +142,11 @@ export class YoutubeDLCLI {
|
|||
}): Promise<{ upload_date: string, webpage_url: string }[]> {
|
||||
const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
|
||||
|
||||
if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
|
||||
// Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
|
||||
additionalYoutubeDLArgs.push('--flat-playlist')
|
||||
}
|
||||
|
||||
if (options.latestVideosCount !== undefined) {
|
||||
additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
|
||||
}
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
import { move, pathExists, readdir, remove } from 'fs-extra'
|
||||
import { dirname, join } from 'path'
|
||||
import { inspect } from 'util'
|
||||
import { CONFIG } from '@server/initializers/config'
|
||||
import { isVideoFileExtnameValid } from '../custom-validators/videos'
|
||||
import { logger, loggerTagsFactory } from '../logger'
|
||||
|
@ -59,13 +60,9 @@ class YoutubeDLWrapper {
|
|||
processOptions
|
||||
})
|
||||
|
||||
if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}`)
|
||||
if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
|
||||
|
||||
return list.map(info => {
|
||||
const infoBuilder = new YoutubeDLInfoBuilder(info)
|
||||
|
||||
return infoBuilder.getInfo()
|
||||
})
|
||||
return list.map(info => info.webpage_url)
|
||||
}
|
||||
|
||||
async getSubtitles (): Promise<YoutubeDLSubs> {
|
||||
|
|
|
@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
|
|||
import { buildYoutubeDLImport } from '@server/lib/video-import'
|
||||
import { UserModel } from '@server/models/user/user'
|
||||
import { VideoImportModel } from '@server/models/video/video-import'
|
||||
import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
|
||||
import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
|
||||
import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
|
||||
import { CreateJobArgument, JobQueue } from './job-queue'
|
||||
import { ServerConfigManager } from './server-config-manager'
|
||||
|
@ -31,15 +31,7 @@ export async function synchronizeChannel (options: {
|
|||
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
||||
)
|
||||
|
||||
const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
|
||||
|
||||
const targetUrls = infoList
|
||||
.filter(videoInfo => {
|
||||
if (!onlyAfter) return true
|
||||
|
||||
return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
|
||||
})
|
||||
.map(videoInfo => videoInfo.webpageUrl)
|
||||
const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
|
||||
|
||||
logger.info(
|
||||
'Fetched %d candidate URLs for sync channel %s.',
|
||||
|
@ -58,10 +50,7 @@ export async function synchronizeChannel (options: {
|
|||
const children: CreateJobArgument[] = []
|
||||
|
||||
for (const targetUrl of targetUrls) {
|
||||
if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
|
||||
logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
|
||||
continue
|
||||
}
|
||||
if (await skipImport(channel, targetUrl, onlyAfter)) continue
|
||||
|
||||
const { job } = await buildYoutubeDLImport({
|
||||
user,
|
||||
|
@ -86,3 +75,28 @@ export async function synchronizeChannel (options: {
|
|||
|
||||
await JobQueue.Instance.createJobWithChildren(parent, children)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
|
||||
if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
|
||||
logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
|
||||
return true
|
||||
}
|
||||
|
||||
if (onlyAfter) {
|
||||
const youtubeDL = new YoutubeDLWrapper(
|
||||
targetUrl,
|
||||
ServerConfigManager.Instance.getEnabledResolutions('vod'),
|
||||
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
||||
)
|
||||
|
||||
const videoInfo = await youtubeDL.getInfoForDownload()
|
||||
|
||||
if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
|
@ -1021,7 +1021,7 @@ describe('Test multiple servers', function () {
|
|||
|
||||
describe('With minimum parameters', function () {
|
||||
it('Should upload and propagate the video', async function () {
|
||||
this.timeout(60000)
|
||||
this.timeout(120000)
|
||||
|
||||
const path = '/api/v1/videos/upload'
|
||||
|
||||
|
|
Loading…
Reference in a new issue