forks/peertube
1
0
Fork 0
Code Releases Activity

Handle sync failure in synchronizeChannel fn

Browse source
This commit is contained in:
Chocobozzz 2022-09-27 09:47:38 +02:00
parent 3097acc7c2
commit 97922ecf64
No known key found for this signature in database
GPG key ID: 583A612D890159BE
3 changed files with 68 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
server/lib/job-queue/handlers/video-channel-import.ts
View file

@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { MChannelSync } from '@server/types/models'
import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models'
import { VideoChannelImportPayload } from '@shared/models'
export async function processVideoChannelImport (job: Job) {
const payload = job.data as VideoChannelImportPayload
@ -32,17 +32,11 @@ export async function processVideoChannelImport (job: Job) {
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
try {
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
await synchronizeChannel({
channel: videoChannel,
externalChannelUrl: payload.externalChannelUrl,
channelSync
})
} catch (err) {
logger.error(`Failed to import channel ${videoChannel.name}`, { err })
channelSync.state = VideoChannelSyncState.FAILED
await channelSync.save()
}
await synchronizeChannel({
channel: videoChannel,
externalChannelUrl: payload.externalChannelUrl,
channelSync
})
}

31
server/lib/schedulers/video-channel-sync-latest-scheduler.ts
View file

@ -2,7 +2,6 @@ import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { VideoChannelSyncState } from '@shared/models'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { synchronizeChannel } from '../sync-channel'
import { AbstractScheduler } from './abstract-scheduler'
@ -28,26 +27,20 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
for (const sync of channelSyncs) {
const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
try {
logger.info(
'Creating video import jobs for "%s" sync with external channel "%s"',
channel.Actor.preferredUsername, sync.externalChannelUrl
)
logger.info(
'Creating video import jobs for "%s" sync with external channel "%s"',
channel.Actor.preferredUsername, sync.externalChannelUrl
)
const onlyAfter = sync.lastSyncAt || sync.createdAt
const onlyAfter = sync.lastSyncAt || sync.createdAt
await synchronizeChannel({
channel,
externalChannelUrl: sync.externalChannelUrl,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
channelSync: sync,
onlyAfter
})
} catch (err) {
logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
sync.state = VideoChannelSyncState.FAILED
await sync.save()
}
await synchronizeChannel({
channel,
externalChannelUrl: sync.externalChannelUrl,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
channelSync: sync,
onlyAfter
})
}
}

92
server/lib/sync-channel.ts
View file

@ -24,56 +24,62 @@ export async function synchronizeChannel (options: {
await channelSync.save()
}
const user = await UserModel.loadByChannelActorId(channel.actorId)
const youtubeDL = new YoutubeDLWrapper(
externalChannelUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
try {
const user = await UserModel.loadByChannelActorId(channel.actorId)
const youtubeDL = new YoutubeDLWrapper(
externalChannelUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
logger.info(
'Fetched %d candidate URLs for sync channel %s.',
targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
)
logger.info(
'Fetched %d candidate URLs for sync channel %s.',
targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
)
if (targetUrls.length === 0) {
if (channelSync) {
channelSync.state = VideoChannelSyncState.SYNCED
await channelSync.save()
}
return
}
const children: CreateJobArgument[] = []
for (const targetUrl of targetUrls) {
if (await skipImport(channel, targetUrl, onlyAfter)) continue
const { job } = await buildYoutubeDLImport({
user,
channel,
targetUrl,
channelSync,
importDataOverride: {
privacy: VideoPrivacy.PUBLIC
if (targetUrls.length === 0) {
if (channelSync) {
channelSync.state = VideoChannelSyncState.SYNCED
await channelSync.save()
}
})
children.push(job)
}
// Will update the channel sync status
const parent: CreateJobArgument = {
type: 'after-video-channel-import',
payload: {
channelSyncId: channelSync?.id
return
}
}
await JobQueue.Instance.createJobWithChildren(parent, children)
const children: CreateJobArgument[] = []
for (const targetUrl of targetUrls) {
if (await skipImport(channel, targetUrl, onlyAfter)) continue
const { job } = await buildYoutubeDLImport({
user,
channel,
targetUrl,
channelSync,
importDataOverride: {
privacy: VideoPrivacy.PUBLIC
}
})
children.push(job)
}
// Will update the channel sync status
const parent: CreateJobArgument = {
type: 'after-video-channel-import',
payload: {
channelSyncId: channelSync?.id
}
}
await JobQueue.Instance.createJobWithChildren(parent, children)
} catch (err) {
logger.error(`Failed to import channel ${channel.name}`, { err })
channelSync.state = VideoChannelSyncState.FAILED
await channelSync.save()
}
}
// ---------------------------------------------------------------------------