forks/peertube
1
0
Fork 0
Code Releases Activity

Fix sync import of latest videos

Browse source
This commit is contained in:
Chocobozzz 2022-09-16 09:37:07 +02:00
parent c95fbe6553
commit 3204f4d17d
No known key found for this signature in database
GPG key ID: 583A612D890159BE
5 changed files with 106 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
server/helpers/youtube-dl/youtube-dl-info-builder.ts
View file

@ -12,7 +12,7 @@ type YoutubeDLInfo = {
tags?: string[]
thumbnailUrl?: string
ext?: string
originallyPublishedAt?: Date
originallyPublishedAtWithoutTime?: Date
webpageUrl?: string
urls?: string[]
@ -81,7 +81,7 @@ class YoutubeDLInfoBuilder {
tags: this.getTags(obj.tags),
thumbnailUrl: obj.thumbnail || undefined,
urls: this.buildAvailableUrl(obj),
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
ext: obj.ext,
webpageUrl: obj.webpage_url
}

4
server/lib/job-queue/handlers/video-channel-import.ts
View file

@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { MChannelSync } from '@server/types/models'
import { VideoChannelImportPayload } from '@shared/models'
import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models'
export async function processVideoChannelImport (job: Job) {
const payload = job.data as VideoChannelImportPayload
@ -42,5 +42,7 @@ export async function processVideoChannelImport (job: Job) {
})
} catch (err) {
logger.error(`Failed to import channel ${videoChannel.name}`, { err })
channelSync.state = VideoChannelSyncState.FAILED
await channelSync.save()
}
}

5
server/lib/sync-channel.ts
View file

@ -93,7 +93,10 @@ async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Dat
const videoInfo = await youtubeDL.getInfoForDownload()
if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
const onlyAfterWithoutTime = new Date(onlyAfter)
onlyAfterWithoutTime.setHours(0, 0, 0)
if (videoInfo.originallyPublishedAtWithoutTime.getTime() >= onlyAfterWithoutTime.getTime()) {
return true
}
}

2
server/lib/video-import.ts
View file

@ -125,7 +125,7 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride
channelId,
originallyPublishedAt: importDataOverride?.originallyPublishedAt
? new Date(importDataOverride?.originallyPublishedAt)
: importData.originallyPublishedAt
: importData.originallyPublishedAtWithoutTime
}
videoData = await Hooks.wrapObject(

138
server/tests/api/videos/video-channel-syncs.ts
View file

@ -3,9 +3,9 @@ import { FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
import {
ChannelSyncsCommand,
createSingleServer,
createMultipleServers,
getServerImportConfig,
killallServers,
PeerTubeServer,
setAccessTokensToServers,
setDefaultAccountAvatar,
@ -20,8 +20,7 @@ describe('Test channel synchronizations', function () {
function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
describe('Sync using ' + mode, function () {
let server: PeerTubeServer
let command: ChannelSyncsCommand
let servers: PeerTubeServer[]
let startTestDate: Date
@ -35,7 +34,7 @@ describe('Test channel synchronizations', function () {
}
async function changeDateForSync (channelSyncId: number, newDate: string) {
await server.sql.updateQuery(
await servers[0].sql.updateQuery(
`UPDATE "videoChannelSync" ` +
`SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
`WHERE id=${channelSyncId}`
@ -43,25 +42,23 @@ describe('Test channel synchronizations', function () {
}
before(async function () {
this.timeout(120_000)
this.timeout(240_000)
startTestDate = new Date()
server = await createSingleServer(1, getServerImportConfig(mode))
servers = await createMultipleServers(2, getServerImportConfig(mode))
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
await setDefaultChannelAvatar([ server ])
await setDefaultAccountAvatar([ server ])
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await setDefaultChannelAvatar(servers)
await setDefaultAccountAvatar(servers)
await server.config.enableChannelSync()
command = server.channelSyncs
await servers[0].config.enableChannelSync()
{
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)
const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
userInfo.channelId = videoChannels[0].id
}
})
@ -70,9 +67,9 @@ describe('Test channel synchronizations', function () {
this.timeout(120_000)
{
const { video } = await server.imports.importVideo({
const { video } = await servers[0].imports.importVideo({
attributes: {
channelId: server.store.channel.id,
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC,
targetUrl: FIXTURE_URLS.youtube
}
@ -80,16 +77,16 @@ describe('Test channel synchronizations', function () {
expect(video.name).to.equal('small video - youtube')
const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
expect(total).to.equal(1)
}
const { videoChannelSync } = await command.create({
const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: server.store.channel.id
videoChannelId: servers[0].store.channel.id
},
token: server.accessToken,
token: servers[0].accessToken,
expectedStatus: HttpStatusCode.OK_200
})
rootChannelSyncId = videoChannelSync.id
@ -97,16 +94,19 @@ describe('Test channel synchronizations', function () {
// Ensure any missing video not already fetched will be considered as new
await changeDateForSync(videoChannelSync.id, '1970-01-01')
await server.debug.sendCommand({
await servers[0].debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
}
})
{
await waitJobs(server)
await waitJobs(servers)
const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
const { total, data } = await servers[0].videos.listByChannel({
handle: 'root_channel',
include: VideoInclude.NOT_PUBLISHED_STATE
})
expect(total).to.equal(2)
expect(data[0].name).to.equal('test')
}
@ -115,18 +115,18 @@ describe('Test channel synchronizations', function () {
it('Should add another synchronization', async function () {
const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
const { videoChannelSync } = await command.create({
const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: {
externalChannelUrl,
videoChannelId: server.store.channel.id
videoChannelId: servers[0].store.channel.id
},
token: server.accessToken,
token: servers[0].accessToken,
expectedStatus: HttpStatusCode.OK_200
})
expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
expect(videoChannelSync.channel).to.include({
id: server.store.channel.id,
id: servers[0].store.channel.id,
name: 'root_channel'
})
expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
@ -134,7 +134,7 @@ describe('Test channel synchronizations', function () {
})
it('Should add a synchronization for another user', async function () {
const { videoChannelSync } = await command.create({
const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
videoChannelId: userInfo.channelId
@ -145,9 +145,9 @@ describe('Test channel synchronizations', function () {
})
it('Should not import a channel if not asked', async function () {
await waitJobs(server)
await waitJobs(servers)
const { data } = await command.listByAccount({ accountName: userInfo.username })
const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(data[0].state).to.contain({
id: VideoChannelSyncState.WAITING_FIRST_RUN,
@ -160,15 +160,15 @@ describe('Test channel synchronizations', function () {
await changeDateForSync(userInfo.syncId, '2019-03-01')
await server.debug.sendCommand({
await servers[0].debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
}
})
await waitJobs(server)
await waitJobs(servers)
const { data, total } = await server.videos.listByChannel({
const { data, total } = await servers[0].videos.listByChannel({
handle: userInfo.channelName,
include: VideoInclude.NOT_PUBLISHED_STATE
})
@ -180,7 +180,7 @@ describe('Test channel synchronizations', function () {
it('Should list channel synchronizations', async function () {
// Root
{
const { total, data } = await command.listByAccount({ accountName: 'root' })
const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
expect(total).to.equal(2)
expect(data[0]).to.deep.contain({
@ -193,13 +193,13 @@ describe('Test channel synchronizations', function () {
expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
expect(data[0].channel).to.contain({ id: server.store.channel.id })
expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
}
// User
{
const { total, data } = await command.listByAccount({ accountName: userInfo.username })
const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(1)
expect(data[0]).to.deep.contain({
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
@ -212,7 +212,7 @@ describe('Test channel synchronizations', function () {
})
it('Should list imports of a channel synchronization', async function () {
const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
@ -220,14 +220,68 @@ describe('Test channel synchronizations', function () {
})
it('Should remove user\'s channel synchronizations', async function () {
await command.delete({ channelSyncId: userInfo.syncId })
await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })
const { total } = await command.listByAccount({ accountName: userInfo.username })
const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(0)
})
// FIXME: youtube-dl doesn't work when speicifying a port after the hostname
// it('Should import a remote PeerTube channel', async function () {
// this.timeout(240_000)
// await servers[1].videos.quickUpload({ name: 'remote 1' })
// await waitJobs(servers)
// const { videoChannelSync } = await servers[0].channelSyncs.create({
// attributes: {
// externalChannelUrl: servers[1].url + '/c/root_channel',
// videoChannelId: userInfo.channelId
// },
// token: userInfo.accessToken
// })
// await servers[0].channels.importVideos({
// channelName: userInfo.channelName,
// externalChannelUrl: servers[1].url + '/c/root_channel',
// videoChannelSyncId: videoChannelSync.id,
// token: userInfo.accessToken
// })
// await waitJobs(servers)
// const { data, total } = await servers[0].videos.listByChannel({
// handle: userInfo.channelName,
// include: VideoInclude.NOT_PUBLISHED_STATE
// })
// expect(total).to.equal(2)
// expect(data[0].name).to.equal('remote 1')
// })
// it('Should keep synced a remote PeerTube channel', async function () {
// this.timeout(240_000)
// await servers[1].videos.quickUpload({ name: 'remote 2' })
// await waitJobs(servers)
// await servers[0].debug.sendCommand({
// body: {
// command: 'process-video-channel-sync-latest'
// }
// })
// await waitJobs(servers)
// const { data, total } = await servers[0].videos.listByChannel({
// handle: userInfo.channelName,
// include: VideoInclude.NOT_PUBLISHED_STATE
// })
// expect(total).to.equal(2)
// expect(data[0].name).to.equal('remote 2')
// })
after(async function () {
await server?.kill()
await killallServers(servers)
})
})
}