Fix infinite playlist import
Using an hard videos limit in config
This commit is contained in:
parent
0e45e336f6
commit
4efa5535cc
|
@ -578,6 +578,9 @@ import:
|
||||||
# Number of latest published videos to check and to potentially import when syncing a channel
|
# Number of latest published videos to check and to potentially import when syncing a channel
|
||||||
videos_limit_per_synchronization: 10
|
videos_limit_per_synchronization: 10
|
||||||
|
|
||||||
|
# Max number of videos to import when the user asks for full sync
|
||||||
|
full_sync_videos_limit: 1000
|
||||||
|
|
||||||
auto_blacklist:
|
auto_blacklist:
|
||||||
# New videos automatically blacklisted so moderators can review before publishing
|
# New videos automatically blacklisted so moderators can review before publishing
|
||||||
videos:
|
videos:
|
||||||
|
|
|
@ -588,6 +588,9 @@ import:
|
||||||
# Number of latest published videos to check and to potentially import when syncing a channel
|
# Number of latest published videos to check and to potentially import when syncing a channel
|
||||||
videos_limit_per_synchronization: 10
|
videos_limit_per_synchronization: 10
|
||||||
|
|
||||||
|
# Max number of videos to import when the user asks for full sync
|
||||||
|
full_sync_videos_limit: 1000
|
||||||
|
|
||||||
auto_blacklist:
|
auto_blacklist:
|
||||||
# New videos automatically blacklisted so moderators can review before publishing
|
# New videos automatically blacklisted so moderators can review before publishing
|
||||||
videos:
|
videos:
|
||||||
|
|
|
@ -35,6 +35,7 @@ function checkMissedConfig () {
|
||||||
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
|
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
|
||||||
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
|
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
|
||||||
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
|
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
|
||||||
|
'import.video_channel_synchronization.full_sync_videos_limit',
|
||||||
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
|
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
|
||||||
'client.videos.miniature.display_author_avatar',
|
'client.videos.miniature.display_author_avatar',
|
||||||
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
|
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',
|
||||||
|
|
|
@ -418,6 +418,9 @@ const CONFIG = {
|
||||||
get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
|
get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
|
||||||
get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
|
get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
|
||||||
return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
|
return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
|
||||||
|
},
|
||||||
|
get FULL_SYNC_VIDEOS_LIMIT () {
|
||||||
|
return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -37,6 +37,7 @@ export async function processVideoChannelImport (job: Job) {
|
||||||
await synchronizeChannel({
|
await synchronizeChannel({
|
||||||
channel: videoChannel,
|
channel: videoChannel,
|
||||||
externalChannelUrl: payload.externalChannelUrl,
|
externalChannelUrl: payload.externalChannelUrl,
|
||||||
channelSync
|
channelSync,
|
||||||
|
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,8 +12,8 @@ import { ServerConfigManager } from './server-config-manager'
|
||||||
export async function synchronizeChannel (options: {
|
export async function synchronizeChannel (options: {
|
||||||
channel: MChannelAccountDefault
|
channel: MChannelAccountDefault
|
||||||
externalChannelUrl: string
|
externalChannelUrl: string
|
||||||
|
videosCountLimit: number
|
||||||
channelSync?: MChannelSync
|
channelSync?: MChannelSync
|
||||||
videosCountLimit?: number
|
|
||||||
onlyAfter?: Date
|
onlyAfter?: Date
|
||||||
}) {
|
}) {
|
||||||
const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
|
const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
|
||||||
|
|
|
@ -109,6 +109,45 @@ describe('Test videos import in a channel', function () {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('Should limit max amount of videos synced on full sync', async function () {
|
||||||
|
this.timeout(240_000)
|
||||||
|
|
||||||
|
await server.kill()
|
||||||
|
await server.run({
|
||||||
|
import: {
|
||||||
|
video_channel_synchronization: {
|
||||||
|
full_sync_videos_limit: 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const { id } = await server.channels.create({ attributes: { name: 'channel3' } })
|
||||||
|
const channel3Id = id
|
||||||
|
|
||||||
|
const { videoChannelSync } = await server.channelSyncs.create({
|
||||||
|
attributes: {
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
videoChannelId: channel3Id
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const syncId = videoChannelSync.id
|
||||||
|
|
||||||
|
await waitJobs(server)
|
||||||
|
|
||||||
|
await server.channels.importVideos({
|
||||||
|
channelName: 'channel3',
|
||||||
|
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
|
||||||
|
videoChannelSyncId: syncId
|
||||||
|
})
|
||||||
|
|
||||||
|
await waitJobs(server)
|
||||||
|
|
||||||
|
const { total, data } = await server.videos.listByChannel({ handle: 'channel3' })
|
||||||
|
|
||||||
|
expect(total).to.equal(1)
|
||||||
|
expect(data).to.have.lengthOf(1)
|
||||||
|
})
|
||||||
|
|
||||||
after(async function () {
|
after(async function () {
|
||||||
await server?.kill()
|
await server?.kill()
|
||||||
})
|
})
|
||||||
|
@ -116,5 +155,7 @@ describe('Test videos import in a channel', function () {
|
||||||
}
|
}
|
||||||
|
|
||||||
runSuite('yt-dlp')
|
runSuite('yt-dlp')
|
||||||
runSuite('youtube-dl')
|
|
||||||
|
// FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails
|
||||||
|
// runSuite('youtube-dl')
|
||||||
})
|
})
|
||||||
|
|
|
@ -220,7 +220,7 @@ describe('Test channel synchronizations', function () {
|
||||||
expect(total).to.equal(0)
|
expect(total).to.equal(0)
|
||||||
})
|
})
|
||||||
|
|
||||||
// FIXME: youtube-dl doesn't work when speicifying a port after the hostname
|
// FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
|
||||||
// it('Should import a remote PeerTube channel', async function () {
|
// it('Should import a remote PeerTube channel', async function () {
|
||||||
// this.timeout(240_000)
|
// this.timeout(240_000)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue