forks
/
peertube
1
0
Fork 0
Code Releases Activity

Refresh playlists

This commit is contained in:
Chocobozzz 2019-03-19 14:13:53 +01:00
parent 0e0c745b62
commit 9f79ade627
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
13 changed files with 217 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/controllers/api/video-playlist.ts
View File

@ -40,6 +40,7 @@ import { VideoPlaylistElementUpdate } from '../../../shared/models/videos/playli
import { copy, pathExists } from 'fs-extra'
import { AccountModel } from '../../models/account/account'
import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model'
import { JobQueue } from '../../lib/job-queue'
const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR })
@ -142,6 +143,11 @@ async function listVideoPlaylists (req: express.Request, res: express.Response)
function getVideoPlaylist (req: express.Request, res: express.Response) {
const videoPlaylist = res.locals.videoPlaylist
if (videoPlaylist.isOutdated()) {
JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: videoPlaylist.url } })
.catch(err => logger.error('Cannot create AP refresher job for playlist %s.', videoPlaylist.url, { err }))
}
return res.json(videoPlaylist.toFormattedJSON())
}

4
server/initializers/constants.ts
View File

@ -584,7 +584,8 @@ const ACTIVITY_PUB = {
},
MAX_RECURSION_COMMENTS: 100,
ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
}
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
@ -724,6 +725,7 @@ if (isTestInstance() === true) {
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB

3
server/lib/activitypub/actor.ts
View File

@ -375,7 +375,8 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
}
if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
return { result: undefined, statusCode: requestResult.response.statusCode }
}
const followersCount = await fetchActorTotalItems(actorJSON.followers)

56
server/lib/activitypub/playlist.ts
View File

@ -95,7 +95,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
return Promise.resolve()
})
// Empty playlists generally do not have a miniature, so skip it
// Empty playlists generally do not have a miniature, so skip this
if (accItems.length !== 0) {
try {
await generateThumbnailFromUrl(playlist, playlistObject.icon)
@ -107,13 +107,45 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
return resetVideoPlaylistElements(accItems, playlist)
}
async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
if (!videoPlaylist.isOutdated()) return videoPlaylist
try {
const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
if (statusCode === 404) {
logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
await videoPlaylist.destroy()
return undefined
}
if (playlistObject === undefined) {
logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
await videoPlaylist.setAsRefreshed()
return videoPlaylist
}
const byAccount = videoPlaylist.OwnerAccount
await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
return videoPlaylist
} catch (err) {
logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
await videoPlaylist.setAsRefreshed()
return videoPlaylist
}
}
// ---------------------------------------------------------------------------
export {
createAccountPlaylists,
playlistObjectToDBAttributes,
playlistElementObjectToDBAttributes,
createOrUpdateVideoPlaylist
createOrUpdateVideoPlaylist,
refreshVideoPlaylistIfNeeded
}
// ---------------------------------------------------------------------------
@ -162,3 +194,23 @@ function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityI
return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
}
async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
const options = {
uri: playlistUrl,
method: 'GET',
json: true,
activityPub: true
}
logger.info('Fetching remote playlist %s.', playlistUrl)
const { response, body } = await doRequest(options)
if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
logger.debug('Remote video playlist JSON is not valid.', { body })
return { statusCode: response.statusCode, playlistObject: undefined }
}
return { statusCode: response.statusCode, playlistObject: body }
}

16
server/lib/job-queue/handlers/activitypub-refresher.ts
View File

@ -1,11 +1,12 @@
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { fetchVideoByUrl } from '../../../helpers/video'
import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub'
import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
import { ActorModel } from '../../../models/activitypub/actor'
import { VideoPlaylistModel } from '../../../models/video/video-playlist'
export type RefreshPayload = {
type: 'video' | 'actor'
type: 'video' | 'video-playlist' | 'actor'
url: string
}
@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) {
logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url)
if (payload.type === 'video') return refreshVideo(payload.url)
if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
if (payload.type === 'actor') return refreshActor(payload.url)
}
// ---------------------------------------------------------------------------
export {
refreshActor,
refreshAPObject
}
@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) {
if (actor) {
await refreshActorIfNeeded(actor, fetchType)
}
}
async function refreshVideoPlaylist (playlistUrl: string) {
const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
if (playlist) {
await refreshVideoPlaylistIfNeeded(playlist)
}
}

3
server/middlewares/validators/videos/video-playlists.ts
View File

@ -8,8 +8,8 @@ import { doesVideoExist, isVideoImage } from '../../../helpers/custom-validators
import { CONSTRAINTS_FIELDS } from '../../../initializers'
import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc'
import {
isVideoPlaylistDescriptionValid,
doesVideoPlaylistExist,
isVideoPlaylistDescriptionValid,
isVideoPlaylistNameValid,
isVideoPlaylistPrivacyValid,
isVideoPlaylistTimestampValid,
@ -19,7 +19,6 @@ import { VideoPlaylistModel } from '../../../models/video/video-playlist'
import { cleanUpReqFiles } from '../../../helpers/express-utils'
import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels'
import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element'
import { VideoModel } from '../../../models/video/video'
import { authenticatePromiseIfNeeded } from '../../oauth'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model'

9
server/models/activitypub/actor.ts
View File

@ -34,7 +34,7 @@ import { ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } fr
import { AccountModel } from '../account/account'
import { AvatarModel } from '../avatar/avatar'
import { ServerModel } from '../server/server'
import { throwIfNotValid } from '../utils'
import { isOutdated, throwIfNotValid } from '../utils'
import { VideoChannelModel } from '../video/video-channel'
import { ActorFollowModel } from './actor-follow'
import { VideoModel } from '../video/video'
@ -532,11 +532,6 @@ export class ActorModel extends Model<ActorModel> {
isOutdated () {
if (this.isOwned()) return false
const now = Date.now()
const createdAtTime = this.createdAt.getTime()
const updatedAtTime = this.updatedAt.getTime()
return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL &&
(now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL
return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL)
}
}

12
server/models/utils.ts
View File

@ -1,5 +1,6 @@
import { Sequelize } from 'sequelize-typescript'
import * as validator from 'validator'
import { ACTIVITY_PUB } from '../initializers'
type SortType = { sortModel: any, sortValue: string }
@ -44,6 +45,14 @@ function getSortOnModel (model: any, value: string, lastSort: string[] = [ 'id',
return [ firstSort, lastSort ]
}
function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
const now = Date.now()
const createdAtTime = model.createdAt.getTime()
const updatedAtTime = model.updatedAt.getTime()
return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
}
function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') {
if (validator(value) === false) {
throw new Error(`"${value}" is not a valid ${fieldName}.`)
@ -108,7 +117,8 @@ export {
throwIfNotValid,
buildServerIdsFollowedBy,
buildTrigramSearchIndex,
buildWhereIdOrUUID
buildWhereIdOrUUID,
isOutdated
}
// ---------------------------------------------------------------------------

15
server/models/video/video-playlist.ts
View File

@ -17,7 +17,7 @@ import {
} from 'sequelize-typescript'
import * as Sequelize from 'sequelize'
import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, throwIfNotValid } from '../utils'
import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, isOutdated, throwIfNotValid } from '../utils'
import {
isVideoPlaylistDescriptionValid,
isVideoPlaylistNameValid,
@ -25,6 +25,7 @@ import {
} from '../../helpers/custom-validators/video-playlists'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import {
ACTIVITY_PUB,
CONFIG,
CONSTRAINTS_FIELDS,
STATIC_PATHS,
@ -429,10 +430,22 @@ export class VideoPlaylistModel extends Model<VideoPlaylistModel> {
.catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err }))
}
setAsRefreshed () {
this.changed('updatedAt', true)
return this.save()
}
isOwned () {
return this.OwnerAccount.isOwned()
}
isOutdated () {
if (this.isOwned()) return false
return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL)
}
toFormattedJSON (): VideoPlaylist {
return {
id: this.id,

11
server/models/video/video.ts
View File

@ -77,7 +77,7 @@ import {
buildTrigramSearchIndex,
buildWhereIdOrUUID,
createSimilarityAttribute,
getVideoSort,
getVideoSort, isOutdated,
throwIfNotValid
} from '../utils'
import { TagModel } from './tag'
@ -1547,7 +1547,7 @@ export class VideoModel extends Model<VideoModel> {
attributes: query.attributes,
order: [ // Keep original order
Sequelize.literal(
ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ')
ids.map(id => `"VideoModel".id = ${id}`).join(', ')
)
]
}
@ -1767,12 +1767,7 @@ export class VideoModel extends Model<VideoModel> {
isOutdated () {
if (this.isOwned()) return false
const now = Date.now()
const createdAtTime = this.createdAt.getTime()
const updatedAtTime = this.updatedAt.getTime()
return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
(now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL)
}
setAsRefreshed () {

132
server/tests/api/activitypub/refresher.ts
View File

@ -2,89 +2,155 @@
import 'mocha'
import {
createVideoPlaylist,
doubleFollow,
flushAndRunMultipleServers,
generateUserAccessToken,
getVideo,
getVideoPlaylist,
killallServers,
reRunServer,
ServerInfo,
setAccessTokensToServers,
uploadVideo,
wait,
setActorField,
setDefaultVideoChannel,
setPlaylistField,
setVideoField,
uploadVideo,
uploadVideoAndGetId,
wait,
waitJobs
} from '../../../../shared/utils'
import { getAccount } from '../../../../shared/utils/users/accounts'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos'
describe('Test AP refresher', function () {
let servers: ServerInfo[] = []
let videoUUID1: string
let videoUUID2: string
let videoUUID3: string
let playlistUUID1: string
let playlistUUID2: string
before(async function () {
this.timeout(60000)
servers = await flushAndRunMultipleServers(2)
servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: false } })
// Get the access tokens
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
{
const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' })
videoUUID1 = res.body.video.uuid
videoUUID1 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video1' })).uuid
videoUUID2 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video2' })).uuid
videoUUID3 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video3' })).uuid
}
{
const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
videoUUID2 = res.body.video.uuid
const a1 = await generateUserAccessToken(servers[1], 'user1')
await uploadVideo(servers[1].url, a1, { name: 'video4' })
const a2 = await generateUserAccessToken(servers[1], 'user2')
await uploadVideo(servers[1].url, a2, { name: 'video5' })
}
{
const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' })
videoUUID3 = res.body.video.uuid
const playlistAttrs = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
playlistUUID1 = res.body.videoPlaylist.uuid
}
{
const playlistAttrs = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
playlistUUID2 = res.body.videoPlaylist.uuid
}
await doubleFollow(servers[0], servers[1])
})
it('Should remove a deleted remote video', async function () {
this.timeout(60000)
describe('Videos refresher', function () {
await wait(10000)
it('Should remove a deleted remote video', async function () {
this.timeout(60000)
// Change UUID so the remote server returns a 404
await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
await wait(10000)
await getVideo(servers[0].url, videoUUID1)
await getVideo(servers[0].url, videoUUID2)
// Change UUID so the remote server returns a 404
await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
await waitJobs(servers)
await getVideo(servers[ 0 ].url, videoUUID1)
await getVideo(servers[ 0 ].url, videoUUID2)
await getVideo(servers[0].url, videoUUID1, 404)
await getVideo(servers[0].url, videoUUID2, 200)
await waitJobs(servers)
await getVideo(servers[ 0 ].url, videoUUID1, 404)
await getVideo(servers[ 0 ].url, videoUUID2, 200)
})
it('Should not update a remote video if the remote instance is down', async function () {
this.timeout(60000)
killallServers([ servers[ 1 ] ])
await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
// Video will need a refresh
await wait(10000)
await getVideo(servers[ 0 ].url, videoUUID3)
// The refresh should fail
await waitJobs([ servers[ 0 ] ])
await reRunServer(servers[ 1 ])
// Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
await getVideo(servers[ 0 ].url, videoUUID3)
await waitJobs(servers)
await getVideo(servers[ 0 ].url, videoUUID3, 200)
})
})
it('Should not update a remote video if the remote instance is down', async function () {
this.timeout(60000)
describe('Actors refresher', function () {
killallServers([ servers[1] ])
it('Should remove a deleted actor', async function () {
this.timeout(60000)
await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
await wait(10000)
// Video will need a refresh
await wait(10000)
// Change actor name so the remote server returns a 404
await setActorField(2, 'http://localhost:9002/accounts/user2', 'preferredUsername', 'toto')
await getVideo(servers[0].url, videoUUID3)
// The refresh should fail
await waitJobs([ servers[0] ])
await getAccount(servers[ 0 ].url, 'user1@localhost:9002')
await getAccount(servers[ 0 ].url, 'user2@localhost:9002')
await reRunServer(servers[1])
await waitJobs(servers)
// Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
await getVideo(servers[0].url, videoUUID3)
await waitJobs(servers)
await getAccount(servers[ 0 ].url, 'user1@localhost:9002', 200)
await getAccount(servers[ 0 ].url, 'user2@localhost:9002', 404)
})
})
await getVideo(servers[0].url, videoUUID3, 200)
describe('Playlist refresher', function () {
it('Should remove a deleted playlist', async function () {
this.timeout(60000)
await wait(10000)
// Change UUID so the remote server returns a 404
await setPlaylistField(2, playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e')
await getVideoPlaylist(servers[ 0 ].url, playlistUUID1)
await getVideoPlaylist(servers[ 0 ].url, playlistUUID2)
await waitJobs(servers)
await getVideoPlaylist(servers[ 0 ].url, playlistUUID1, 200)
await getVideoPlaylist(servers[ 0 ].url, playlistUUID2, 404)
})
})
after(async function () {

9
shared/utils/miscs/sql.ts
View File

@ -40,6 +40,14 @@ function setVideoField (serverNumber: number, uuid: string, field: string, value
return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
}
function setPlaylistField (serverNumber: number, uuid: string, field: string, value: string) {
const seq = getSequelize(serverNumber)
const options = { type: Sequelize.QueryTypes.UPDATE }
return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
}
async function closeAllSequelize (servers: any[]) {
for (let i = 1; i <= servers.length; i++) {
if (sequelizes[ i ]) {
@ -51,6 +59,7 @@ async function closeAllSequelize (servers: any[]) {
export {
setVideoField,
setPlaylistField,
setActorField,
closeAllSequelize
}

2
shared/utils/videos/video-playlists.ts
View File

@ -127,7 +127,7 @@ function createVideoPlaylist (options: {
playlistAttrs: VideoPlaylistCreate,
expectedStatus?: number
}) {
const path = '/api/v1/video-playlists/'
const path = '/api/v1/video-playlists'
const fields = omit(options.playlistAttrs, 'thumbnailfile')