Refactor AP playlists

server/lib/activitypub/cache-file.ts

@@ -1,8 +1,51 @@
-import { CacheFileObject } from '../../../shared/index'
-import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { Transaction } from 'sequelize'
-import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models'
+import { CacheFileObject } from '../../../shared/index'
+import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+
+async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
+  const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
+
+  if (redundancyModel) {
+    return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
+  }
+
+  return createCacheFile(cacheFileObject, video, byActor, t)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  createOrUpdateCacheFile
+}
+
+// ---------------------------------------------------------------------------
+
+function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
+  const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
+
+  return VideoRedundancyModel.create(attributes, { transaction: t })
+}
+
+function updateCacheFile (
+  cacheFileObject: CacheFileObject,
+  redundancyModel: MVideoRedundancy,
+  video: MVideoWithAllFiles,
+  byActor: MActorId,
+  t: Transaction
+) {
+  if (redundancyModel.actorId !== byActor.id) {
+    throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
+  }
+
+  const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
+
+  redundancyModel.expiresOn = attributes.expiresOn
+  redundancyModel.fileUrl = attributes.fileUrl
+
+  return redundancyModel.save({ transaction: t })
+}
 
 function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) {
 
@@ -38,45 +81,3 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
     actorId: byActor.id
   }
 }
-
-async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
-  const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
-
-  if (!redundancyModel) {
-    await createCacheFile(cacheFileObject, video, byActor, t)
-  } else {
-    await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
-  }
-}
-
-function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
-  const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
-
-  return VideoRedundancyModel.create(attributes, { transaction: t })
-}
-
-function updateCacheFile (
-  cacheFileObject: CacheFileObject,
-  redundancyModel: MVideoRedundancy,
-  video: MVideoWithAllFiles,
-  byActor: MActorId,
-  t: Transaction
-) {
-  if (redundancyModel.actorId !== byActor.id) {
-    throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.')
-  }
-
-  const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
-
-  redundancyModel.expiresOn = attributes.expiresOn
-  redundancyModel.fileUrl = attributes.fileUrl
-
-  return redundancyModel.save({ transaction: t })
-}
-
-export {
-  createOrUpdateCacheFile,
-  createCacheFile,
-  updateCacheFile,
-  cacheFileActivityObjectToDBAttributes
-}

server/lib/activitypub/playlist.ts

@@ -1,204 +0,0 @@
-import * as Bluebird from 'bluebird'
-import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
-import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
-import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
-import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
-import { checkUrlsSameHost } from '../../helpers/activitypub'
-import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
-import { isArray } from '../../helpers/custom-validators/misc'
-import { logger } from '../../helpers/logger'
-import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
-import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
-import { sequelizeTypescript } from '../../initializers/database'
-import { VideoPlaylistModel } from '../../models/video/video-playlist'
-import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
-import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
-import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
-import { FilteredModelAttributes } from '../../types/sequelize'
-import { createPlaylistMiniatureFromUrl } from '../thumbnail'
-import { getOrCreateActorAndServerAndModel } from './actor'
-import { crawlCollectionPage } from './crawl'
-import { getOrCreateAPVideo } from './videos'
-
-function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
-  const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
-    ? VideoPlaylistPrivacy.PUBLIC
-    : VideoPlaylistPrivacy.UNLISTED
-
-  return {
-    name: playlistObject.name,
-    description: playlistObject.content,
-    privacy,
-    url: playlistObject.id,
-    uuid: playlistObject.uuid,
-    ownerAccountId: byAccount.id,
-    videoChannelId: null,
-    createdAt: new Date(playlistObject.published),
-    updatedAt: new Date(playlistObject.updated)
-  }
-}
-
-function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
-  return {
-    position: elementObject.position,
-    url: elementObject.id,
-    startTimestamp: elementObject.startTimestamp || null,
-    stopTimestamp: elementObject.stopTimestamp || null,
-    videoPlaylistId: videoPlaylist.id,
-    videoId: video.id
-  }
-}
-
-async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
-  await Bluebird.map(playlistUrls, async playlistUrl => {
-    try {
-      const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
-      if (exists === true) return
-
-      // Fetch url
-      const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
-
-      if (!isPlaylistObjectValid(body)) {
-        throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
-      }
-
-      if (!isArray(body.to)) {
-        throw new Error('Playlist does not have an audience.')
-      }
-
-      return createOrUpdateVideoPlaylist(body, account, body.to)
-    } catch (err) {
-      logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
-    }
-  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
-}
-
-async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
-  const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
-
-  if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
-    const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
-
-    if (actor.VideoChannel) {
-      playlistAttributes.videoChannelId = actor.VideoChannel.id
-    } else {
-      logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
-    }
-  }
-
-  const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
-
-  let accItems: string[] = []
-  await crawlCollectionPage<string>(playlistObject.id, items => {
-    accItems = accItems.concat(items)
-
-    return Promise.resolve()
-  })
-
-  const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
-
-  if (playlistObject.icon) {
-    try {
-      const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
-      await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
-    } catch (err) {
-      logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
-    }
-  } else if (refreshedPlaylist.hasThumbnail()) {
-    await refreshedPlaylist.Thumbnail.destroy()
-    refreshedPlaylist.Thumbnail = null
-  }
-
-  return resetVideoPlaylistElements(accItems, refreshedPlaylist)
-}
-
-async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
-  if (!videoPlaylist.isOutdated()) return videoPlaylist
-
-  try {
-    const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
-
-    if (playlistObject === undefined) {
-      logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
-
-      await videoPlaylist.setAsRefreshed()
-      return videoPlaylist
-    }
-
-    const byAccount = videoPlaylist.OwnerAccount
-    await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
-
-    return videoPlaylist
-  } catch (err) {
-    if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
-      logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
-
-      await videoPlaylist.destroy()
-      return undefined
-    }
-
-    logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
-
-    await videoPlaylist.setAsRefreshed()
-    return videoPlaylist
-  }
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  createAccountPlaylists,
-  playlistObjectToDBAttributes,
-  playlistElementObjectToDBAttributes,
-  createOrUpdateVideoPlaylist,
-  refreshVideoPlaylistIfNeeded
-}
-
-// ---------------------------------------------------------------------------
-
-async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
-  const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
-
-  await Bluebird.map(elementUrls, async elementUrl => {
-    try {
-      const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
-
-      if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
-
-      if (checkUrlsSameHost(body.id, elementUrl) !== true) {
-        throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
-      }
-
-      const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' })
-
-      elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
-    } catch (err) {
-      logger.warn('Cannot add playlist element %s.', elementUrl, { err })
-    }
-  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
-
-  await sequelizeTypescript.transaction(async t => {
-    await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
-
-    for (const element of elementsToCreate) {
-      await VideoPlaylistElementModel.create(element, { transaction: t })
-    }
-  })
-
-  logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
-
-  return undefined
-}
-
-async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
-  logger.info('Fetching remote playlist %s.', playlistUrl)
-
-  const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
-
-  if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
-    logger.debug('Remote video playlist JSON is not valid.', { body })
-    return { statusCode, playlistObject: undefined }
-  }
-
-  return { statusCode, playlistObject: body }
-}

server/lib/activitypub/playlists/create-update.ts

@@ -0,0 +1,146 @@
+import { isArray } from '@server/helpers/custom-validators/misc'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
+import { sequelizeTypescript } from '@server/initializers/database'
+import { createPlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
+import { FilteredModelAttributes } from '@server/types'
+import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
+import { PlaylistObject } from '@shared/models'
+import { getOrCreateActorAndServerAndModel } from '../actor'
+import { crawlCollectionPage } from '../crawl'
+import { getOrCreateAPVideo } from '../videos'
+import {
+  fetchRemotePlaylistElement,
+  fetchRemoteVideoPlaylist,
+  playlistElementObjectToDBAttributes,
+  playlistObjectToDBAttributes
+} from './shared'
+
+import Bluebird = require('bluebird')
+
+const lTags = loggerTagsFactory('ap', 'video-playlist')
+
+async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
+  await Bluebird.map(playlistUrls, async playlistUrl => {
+    try {
+      const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
+      if (exists === true) return
+
+      const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
+
+      if (playlistObject === undefined) {
+        throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
+      }
+
+      return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
+    } catch (err) {
+      logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+}
+
+async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
+  const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
+
+  await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
+
+  const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
+
+  const playlistElementUrls = await fetchElementUrls(playlistObject)
+
+  // Refetch playlist from DB since elements fetching could be long in time
+  const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
+
+  try {
+    await updatePlaylistThumbnail(playlistObject, playlist)
+  } catch (err) {
+    logger.warn('Cannot update thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
+  }
+
+  return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  createAccountPlaylists,
+  createOrUpdateVideoPlaylist
+}
+
+// ---------------------------------------------------------------------------
+
+async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
+  if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
+
+  const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
+
+  if (!actor.VideoChannel) {
+    logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
+    return
+  }
+
+  playlistAttributes.videoChannelId = actor.VideoChannel.id
+}
+
+async function fetchElementUrls (playlistObject: PlaylistObject) {
+  let accItems: string[] = []
+  await crawlCollectionPage<string>(playlistObject.id, items => {
+    accItems = accItems.concat(items)
+
+    return Promise.resolve()
+  })
+
+  return accItems
+}
+
+async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
+  if (playlistObject.icon) {
+    const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
+    await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
+
+    return
+  }
+
+  // Playlist does not have an icon, destroy existing one
+  if (playlist.hasThumbnail()) {
+    await playlist.Thumbnail.destroy()
+    playlist.Thumbnail = null
+  }
+}
+
+async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
+  const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
+
+  await sequelizeTypescript.transaction(async t => {
+    await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
+
+    for (const element of elementsToCreate) {
+      await VideoPlaylistElementModel.create(element, { transaction: t })
+    }
+  })
+
+  logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
+
+  return undefined
+}
+
+async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
+  const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
+
+  await Bluebird.map(elementUrls, async elementUrl => {
+    try {
+      const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
+
+      const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
+
+      elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
+    } catch (err) {
+      logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+
+  return elementsToCreate
+}

server/lib/activitypub/playlists/index.ts

@@ -0,0 +1,2 @@
+export * from './create-update'
+export * from './refresh'

server/lib/activitypub/playlists/refresh.ts

@@ -0,0 +1,44 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { PeerTubeRequestError } from '@server/helpers/requests'
+import { MVideoPlaylistOwner } from '@server/types/models'
+import { HttpStatusCode } from '@shared/core-utils'
+import { createOrUpdateVideoPlaylist } from './create-update'
+import { fetchRemoteVideoPlaylist } from './shared'
+
+async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
+  if (!videoPlaylist.isOutdated()) return videoPlaylist
+
+  const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
+
+  try {
+    const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
+
+    if (playlistObject === undefined) {
+      logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags())
+
+      await videoPlaylist.setAsRefreshed()
+      return videoPlaylist
+    }
+
+    const byAccount = videoPlaylist.OwnerAccount
+    await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
+
+    return videoPlaylist
+  } catch (err) {
+    if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
+      logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags())
+
+      await videoPlaylist.destroy()
+      return undefined
+    }
+
+    logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() })
+
+    await videoPlaylist.setAsRefreshed()
+    return videoPlaylist
+  }
+}
+
+export {
+  refreshVideoPlaylistIfNeeded
+}

server/lib/activitypub/playlists/shared/index.ts

@@ -0,0 +1,2 @@
+export * from './object-to-model-attributes'
+export * from './url-to-object'

server/lib/activitypub/playlists/shared/object-to-model-attributes.ts

@@ -0,0 +1,40 @@
+import { ACTIVITY_PUB } from '@server/initializers/constants'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
+import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
+import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
+
+function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
+  const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
+    ? VideoPlaylistPrivacy.PUBLIC
+    : VideoPlaylistPrivacy.UNLISTED
+
+  return {
+    name: playlistObject.name,
+    description: playlistObject.content,
+    privacy,
+    url: playlistObject.id,
+    uuid: playlistObject.uuid,
+    ownerAccountId: byAccount.id,
+    videoChannelId: null,
+    createdAt: new Date(playlistObject.published),
+    updatedAt: new Date(playlistObject.updated)
+  } as AttributesOnly<VideoPlaylistModel>
+}
+
+function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
+  return {
+    position: elementObject.position,
+    url: elementObject.id,
+    startTimestamp: elementObject.startTimestamp || null,
+    stopTimestamp: elementObject.stopTimestamp || null,
+    videoPlaylistId: videoPlaylist.id,
+    videoId: video.id
+  } as AttributesOnly<VideoPlaylistElementModel>
+}
+
+export {
+  playlistObjectToDBAttributes,
+  playlistElementObjectToDBAttributes
+}

server/lib/activitypub/playlists/shared/url-to-object.ts

@@ -0,0 +1,47 @@
+import { isArray } from 'lodash'
+import { checkUrlsSameHost } from '@server/helpers/activitypub'
+import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { doJSONRequest } from '@server/helpers/requests'
+import { PlaylistElementObject, PlaylistObject } from '@shared/models'
+
+async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
+  const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl)
+
+  logger.info('Fetching remote playlist %s.', playlistUrl, lTags())
+
+  const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
+
+  if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
+    logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() })
+    return { statusCode, playlistObject: undefined }
+  }
+
+  if (!isArray(body.to)) {
+    logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() })
+    return { statusCode, playlistObject: undefined }
+  }
+
+  return { statusCode, playlistObject: body }
+}
+
+async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> {
+  const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl)
+
+  logger.debug('Fetching remote playlist element %s.', elementUrl, lTags())
+
+  const { body, statusCode } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
+
+  if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`)
+
+  if (checkUrlsSameHost(body.id, elementUrl) !== true) {
+    throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
+  }
+
+  return { statusCode, elementObject: body }
+}
+
+export {
+  fetchRemoteVideoPlaylist,
+  fetchRemotePlaylistElement
+}

server/lib/activitypub/process/process-create.ts

@@ -1,3 +1,4 @@
+import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
 import { isRedundancyAccepted } from '@server/lib/redundancy'
 import { ActivityCreate, CacheFileObject, VideoObject } from '../../../../shared'
 import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
@@ -9,11 +10,10 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
 import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models'
 import { Notifier } from '../../notifier'
 import { createOrUpdateCacheFile } from '../cache-file'
-import { createOrUpdateVideoPlaylist } from '../playlist'
+import { createOrUpdateVideoPlaylist } from '../playlists'
 import { forwardVideoRelatedActivity } from '../send/utils'
 import { resolveThread } from '../video-comments'
 import { getOrCreateAPVideo } from '../videos'
-import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
 
 async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
   const { activity, byActor } = options

server/lib/activitypub/process/process-update.ts

@@ -15,7 +15,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
 import { MActorSignature } from '../../../types/models'
 import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor'
 import { createOrUpdateCacheFile } from '../cache-file'
-import { createOrUpdateVideoPlaylist } from '../playlist'
+import { createOrUpdateVideoPlaylist } from '../playlists'
 import { forwardVideoRelatedActivity } from '../send/utils'
 import { APVideoUpdater, getOrCreateAPVideo } from '../videos'
 

server/lib/activitypub/share.ts

@@ -40,6 +40,22 @@ async function changeVideoChannelShare (
 async function addVideoShares (shareUrls: string[], video: MVideoId) {
   await Bluebird.map(shareUrls, async shareUrl => {
     try {
+      await addVideoShare(shareUrl, video)
+    } catch (err) {
+      logger.warn('Cannot add share %s.', shareUrl, { err })
+    }
+  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
+}
+
+export {
+  changeVideoChannelShare,
+  addVideoShares,
+  shareVideoByServerAndChannel
+}
+
+// ---------------------------------------------------------------------------
+
+async function addVideoShare (shareUrl: string, video: MVideoId) {
   const { body } = await doJSONRequest<any>(shareUrl, { activityPub: true })
   if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
@@ -57,19 +73,7 @@ async function addVideoShares (shareUrls: string[], video: MVideoId) {
   }
 
   await VideoShareModel.upsert(entry)
-    } catch (err) {
-      logger.warn('Cannot add share %s.', shareUrl, { err })
 }
-  }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
-}
-
-export {
-  changeVideoChannelShare,
-  addVideoShares,
-  shareVideoByServerAndChannel
-}
-
-// ---------------------------------------------------------------------------
 
 async function shareByServer (video: MVideo, t: Transaction) {
   const serverActor = await getServerActor()

server/lib/activitypub/video-comments.ts

@@ -29,10 +29,11 @@ async function addVideoComments (commentUrls: string[]) {
 
 async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
   const { url, isVideo } = params
+
   if (params.commentCreated === undefined) params.commentCreated = false
   if (params.comments === undefined) params.comments = []
 
-  // If it is not a video, or if we don't know if it's a video
+  // If it is not a video, or if we don't know if it's a video, try to get the thread from DB
   if (isVideo === false || isVideo === undefined) {
     const result = await resolveCommentFromDB(params)
     if (result) return result
@@ -42,7 +43,7 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult
     // If it is a video, or if we don't know if it's a video
     if (isVideo === true || isVideo === undefined) {
       // Keep await so we catch the exception
-      return await tryResolveThreadFromVideo(params)
+      return await tryToResolveThreadFromVideo(params)
     }
   } catch (err) {
     logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err })
@@ -62,7 +63,8 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
   const { url, comments, commentCreated } = params
 
   const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
-  if (commentFromDatabase) {
+  if (!commentFromDatabase) return undefined
+
   let parentComments = comments.concat([ commentFromDatabase ])
 
   // Speed up things and resolve directly the thread
@@ -80,10 +82,7 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
   })
 }
 
-  return undefined
+async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
-}
-
-async function tryResolveThreadFromVideo (params: ResolveThreadParams) {
   const { url, comments, commentCreated } = params
 
   // Maybe it's a reply to a video?

server/lib/activitypub/video-rates.ts

@@ -15,30 +15,7 @@ import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlBy
 async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) {
   await Bluebird.map(ratesUrl, async rateUrl => {
     try {
-      // Fetch url
+      await createRate(rateUrl, video, rate)
-      const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
-      if (!body || !body.actor) throw new Error('Body or body actor is invalid')
-
-      const actorUrl = getAPId(body.actor)
-      if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
-        throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
-      }
-
-      if (checkUrlsSameHost(body.id, rateUrl) !== true) {
-        throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
-      }
-
-      const actor = await getOrCreateActorAndServerAndModel(actorUrl)
-
-      const entry = {
-        videoId: video.id,
-        accountId: actor.Account.id,
-        type: rate,
-        url: body.id
-      }
-
-      // Video "likes"/"dislikes" will be updated by the caller
-      await AccountVideoRateModel.upsert(entry)
     } catch (err) {
       logger.warn('Cannot add rate %s.', rateUrl, { err })
     }
@@ -73,8 +50,39 @@ function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVid
     : getVideoDislikeActivityPubUrlByLocalActor(actor, video)
 }
 
+// ---------------------------------------------------------------------------
+
 export {
   getLocalRateUrl,
   createRates,
   sendVideoRateChange
 }
+
+// ---------------------------------------------------------------------------
+
+async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) {
+  // Fetch url
+  const { body } = await doJSONRequest<any>(rateUrl, { activityPub: true })
+  if (!body || !body.actor) throw new Error('Body or body actor is invalid')
+
+  const actorUrl = getAPId(body.actor)
+  if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
+    throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
+  }
+
+  if (checkUrlsSameHost(body.id, rateUrl) !== true) {
+    throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`)
+  }
+
+  const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+
+  const entry = {
+    videoId: video.id,
+    accountId: actor.Account.id,
+    type: rate,
+    url: body.id
+  }
+
+  // Video "likes"/"dislikes" will be updated by the caller
+  await AccountVideoRateModel.upsert(entry)
+}

server/lib/job-queue/handlers/activitypub-http-fetcher.ts

@@ -8,7 +8,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { MAccountDefault, MVideoFullLight } from '../../../types/models'
 import { crawlCollectionPage } from '../../activitypub/crawl'
-import { createAccountPlaylists } from '../../activitypub/playlist'
+import { createAccountPlaylists } from '../../activitypub/playlists'
 import { processActivities } from '../../activitypub/process'
 import { addVideoShares } from '../../activitypub/share'
 import { addVideoComments } from '../../activitypub/video-comments'

server/lib/job-queue/handlers/activitypub-refresher.ts

@@ -1,5 +1,5 @@
 import * as Bull from 'bull'
-import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist'
+import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists'
 import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos'
 import { RefreshPayload } from '@shared/models'
 import { logger } from '../../../helpers/logger'

server/models/video/video-playlist.ts

@@ -18,6 +18,7 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { v4 as uuidv4 } from 'uuid'
+import { setAsUpdated } from '@server/helpers/database-utils'
 import { MAccountId, MChannelId } from '@server/types/models'
 import { AttributesOnly } from '@shared/core-utils'
 import { ActivityIconObject } from '../../../shared/models/activitypub/objects'
@@ -531,9 +532,7 @@ export class VideoPlaylistModel extends Model<Partial<AttributesOnly<VideoPlayli
   }
 
   setAsRefreshed () {
-    this.changed('updatedAt', true)
+    return setAsUpdated('videoPlaylist', this.id)
-
-    return this.save()
   }
 
   isOwned () {

shared/models/activitypub/objects/index.ts

@@ -2,5 +2,9 @@ export * from './abuse-object'
 export * from './cache-file-object'
 export * from './common-objects'
 export * from './dislike-object'
+export * from './object.model'
+export * from './playlist-element-object'
+export * from './playlist-object'
+export * from './video-comment-object'
 export * from './video-torrent-object'
 export * from './view-object'