1
0
Fork 0

Fix infinite playlist import

Using an hard videos limit in config
This commit is contained in:
Chocobozzz 2022-11-14 11:11:39 +01:00
parent 0e45e336f6
commit 4efa5535cc
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
8 changed files with 56 additions and 4 deletions

View File

@ -578,6 +578,9 @@ import:
# Number of latest published videos to check and to potentially import when syncing a channel
videos_limit_per_synchronization: 10
# Max number of videos to import when the user asks for full sync
full_sync_videos_limit: 1000
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:

View File

@ -588,6 +588,9 @@ import:
# Number of latest published videos to check and to potentially import when syncing a channel
videos_limit_per_synchronization: 10
# Max number of videos to import when the user asks for full sync
full_sync_videos_limit: 1000
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:

View File

@ -35,6 +35,7 @@ function checkMissedConfig () {
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
'import.video_channel_synchronization.full_sync_videos_limit',
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
'client.videos.miniature.display_author_avatar',
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',

View File

@ -418,6 +418,9 @@ const CONFIG = {
get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
},
get FULL_SYNC_VIDEOS_LIMIT () {
return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit')
}
}
},

View File

@ -37,6 +37,7 @@ export async function processVideoChannelImport (job: Job) {
await synchronizeChannel({
channel: videoChannel,
externalChannelUrl: payload.externalChannelUrl,
channelSync
channelSync,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
})
}

View File

@ -12,8 +12,8 @@ import { ServerConfigManager } from './server-config-manager'
export async function synchronizeChannel (options: {
channel: MChannelAccountDefault
externalChannelUrl: string
videosCountLimit: number
channelSync?: MChannelSync
videosCountLimit?: number
onlyAfter?: Date
}) {
const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options

View File

@ -109,6 +109,45 @@ describe('Test videos import in a channel', function () {
}
})
it('Should limit max amount of videos synced on full sync', async function () {
this.timeout(240_000)
await server.kill()
await server.run({
import: {
video_channel_synchronization: {
full_sync_videos_limit: 1
}
}
})
const { id } = await server.channels.create({ attributes: { name: 'channel3' } })
const channel3Id = id
const { videoChannelSync } = await server.channelSyncs.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: channel3Id
}
})
const syncId = videoChannelSync.id
await waitJobs(server)
await server.channels.importVideos({
channelName: 'channel3',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelSyncId: syncId
})
await waitJobs(server)
const { total, data } = await server.videos.listByChannel({ handle: 'channel3' })
expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
})
after(async function () {
await server?.kill()
})
@ -116,5 +155,7 @@ describe('Test videos import in a channel', function () {
}
runSuite('yt-dlp')
runSuite('youtube-dl')
// FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails
// runSuite('youtube-dl')
})

View File

@ -220,7 +220,7 @@ describe('Test channel synchronizations', function () {
expect(total).to.equal(0)
})
// FIXME: youtube-dl doesn't work when speicifying a port after the hostname
// FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
// it('Should import a remote PeerTube channel', async function () {
// this.timeout(240_000)