diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 0cb5674c2..ef0f4285d 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -15,7 +15,7 @@ import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/expr import { logger } from '../../helpers/logger' import { getFormattedObjects } from '../../helpers/utils' import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' -import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor' +import { getOrCreateAPActor } from '../../lib/activitypub/actors' import { asyncMiddleware, commonVideosFiltersValidator, @@ -145,7 +145,7 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean if (isUserAbleToSearchRemoteURI(res)) { try { - const actor = await getOrCreateActorAndServerAndModel(uri, 'all', true, true) + const actor = await getOrCreateAPActor(uri, 'all', true, true) videoChannel = actor.VideoChannel } catch (err) { logger.info('Cannot search remote video channel %s.', uri, { err }) diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 810e4295e..1f2b2f9dd 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -11,7 +11,7 @@ import { CONFIG } from '../../../initializers/config' import { MIMETYPES } from '../../../initializers/constants' import { sequelizeTypescript } from '../../../initializers/database' import { sendUpdateActor } from '../../../lib/activitypub/send' -import { deleteLocalActorImageFile, updateLocalActorImageFile } from '../../../lib/actor-image' +import { deleteLocalActorImageFile, updateLocalActorImageFile } from '../../../lib/local-actor' import { getOriginalVideoFileTotalDailyFromUser, getOriginalVideoFileTotalFromUser, sendVerifyUserEmail } from '../../../lib/user' import { asyncMiddleware, diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 34207ea8a..03aa918d3 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -13,8 +13,8 @@ import { CONFIG } from '../../initializers/config' import { MIMETYPES } from '../../initializers/constants' import { sequelizeTypescript } from '../../initializers/database' import { sendUpdateActor } from '../../lib/activitypub/send' -import { deleteLocalActorImageFile, updateLocalActorImageFile } from '../../lib/actor-image' import { JobQueue } from '../../lib/job-queue' +import { deleteLocalActorImageFile, updateLocalActorImageFile } from '../../lib/local-actor' import { createLocalVideoChannel, federateAllVideosOfChannel } from '../../lib/video-channel' import { asyncMiddleware, diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts index 9f260cef0..27b1b7160 100644 --- a/server/controllers/lazy-static.ts +++ b/server/controllers/lazy-static.ts @@ -4,8 +4,8 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' import { logger } from '../helpers/logger' import { LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' -import { actorImagePathUnsafeCache, pushActorImageProcessInQueue } from '../lib/actor-image' import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' +import { actorImagePathUnsafeCache, pushActorImageProcessInQueue } from '../lib/local-actor' import { asyncMiddleware } from '../middlewares' import { ActorImageModel } from '../models/actor/actor-image' diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts deleted file mode 100644 index 1bcee7ef9..000000000 --- a/server/lib/activitypub/actor.ts +++ /dev/null @@ -1,593 +0,0 @@ -import * as Bluebird from 'bluebird' -import { extname } from 'path' -import { Op, Transaction } from 'sequelize' -import { URL } from 'url' -import { v4 as uuidv4 } from 'uuid' -import { getServerActor } from '@server/models/application/application' -import { ActorImageType } from '@shared/models' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' -import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' -import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' -import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' -import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' -import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' -import { logger } from '../../helpers/logger' -import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests' -import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { MIMETYPES, WEBSERVER } from '../../initializers/constants' -import { sequelizeTypescript } from '../../initializers/database' -import { AccountModel } from '../../models/account/account' -import { ActorModel } from '../../models/actor/actor' -import { ActorImageModel } from '../../models/actor/actor-image' -import { ServerModel } from '../../models/server/server' -import { VideoChannelModel } from '../../models/video/video-channel' -import { - MAccount, - MAccountDefault, - MActor, - MActorAccountChannelId, - MActorAccountChannelIdActor, - MActorAccountId, - MActorFull, - MActorFullActor, - MActorId, - MActorImage, - MActorImages, - MChannel -} from '../../types/models' -import { JobQueue } from '../job-queue' - -// Set account keys, this could be long so process after the account creation and do not block the client -async function generateAndSaveActorKeys (actor: T) { - const { publicKey, privateKey } = await createPrivateAndPublicKeys() - - actor.publicKey = publicKey - actor.privateKey = privateKey - - return actor.save() -} - -function getOrCreateActorAndServerAndModel ( - activityActor: string | ActivityPubActor, - fetchType: 'all', - recurseIfNeeded?: boolean, - updateCollections?: boolean -): Promise - -function getOrCreateActorAndServerAndModel ( - activityActor: string | ActivityPubActor, - fetchType?: 'association-ids', - recurseIfNeeded?: boolean, - updateCollections?: boolean -): Promise - -async function getOrCreateActorAndServerAndModel ( - activityActor: string | ActivityPubActor, - fetchType: ActorFetchByUrlType = 'association-ids', - recurseIfNeeded = true, - updateCollections = false -): Promise { - const actorUrl = getAPId(activityActor) - let created = false - let accountPlaylistsUrl: string - - let actor = await fetchActorByUrl(actorUrl, fetchType) - // Orphan actor (not associated to an account of channel) so recreate it - if (actor && (!actor.Account && !actor.VideoChannel)) { - await actor.destroy() - actor = null - } - - // We don't have this actor in our database, fetch it on remote - if (!actor) { - const { result } = await fetchRemoteActor(actorUrl) - if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) - - // Create the attributed to actor - // In PeerTube a video channel is owned by an account - let ownerActor: MActorFullActor - if (recurseIfNeeded === true && result.actor.type === 'Group') { - const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') - if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) - - if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { - throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) - } - - try { - // Don't recurse another time - const recurseIfNeeded = false - ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded) - } catch (err) { - logger.error('Cannot get or create account attributed to video channel ' + actorUrl) - throw new Error(err) - } - } - - actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor) - created = true - accountPlaylistsUrl = result.playlists - } - - if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor - if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor - - const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) - if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') - - if ((created === true || refreshed === true) && updateCollections === true) { - const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } - await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) - } - - // We created a new account: fetch the playlists - if (created === true && actor.Account && accountPlaylistsUrl) { - const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' } - await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) - } - - return actorRefreshed -} - -function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { - return new ActorModel({ - type, - url, - preferredUsername, - publicKey: null, - privateKey: null, - followersCount: 0, - followingCount: 0, - inboxUrl: url + '/inbox', - outboxUrl: url + '/outbox', - sharedInboxUrl: WEBSERVER.URL + '/inbox', - followersUrl: url + '/followers', - followingUrl: url + '/following' - }) as MActor -} - -async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) { - const followersCount = await fetchActorTotalItems(attributes.followers) - const followingCount = await fetchActorTotalItems(attributes.following) - - actorInstance.type = attributes.type - actorInstance.preferredUsername = attributes.preferredUsername - actorInstance.url = attributes.id - actorInstance.publicKey = attributes.publicKey.publicKeyPem - actorInstance.followersCount = followersCount - actorInstance.followingCount = followingCount - actorInstance.inboxUrl = attributes.inbox - actorInstance.outboxUrl = attributes.outbox - actorInstance.followersUrl = attributes.followers - actorInstance.followingUrl = attributes.following - - if (attributes.published) actorInstance.remoteCreatedAt = new Date(attributes.published) - - if (attributes.endpoints?.sharedInbox) { - actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox - } -} - -type ImageInfo = { - name: string - fileUrl: string - height: number - width: number - onDisk?: boolean -} -async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) { - const oldImageModel = type === ActorImageType.AVATAR - ? actor.Avatar - : actor.Banner - - if (oldImageModel) { - // Don't update the avatar if the file URL did not change - if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor - - try { - await oldImageModel.destroy({ transaction: t }) - - setActorImage(actor, type, null) - } catch (err) { - logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) - } - } - - if (imageInfo) { - const imageModel = await ActorImageModel.create({ - filename: imageInfo.name, - onDisk: imageInfo.onDisk ?? false, - fileUrl: imageInfo.fileUrl, - height: imageInfo.height, - width: imageInfo.width, - type - }, { transaction: t }) - - setActorImage(actor, type, imageModel) - } - - return actor -} - -async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) { - try { - if (type === ActorImageType.AVATAR) { - await actor.Avatar.destroy({ transaction: t }) - - actor.avatarId = null - actor.Avatar = null - } else { - await actor.Banner.destroy({ transaction: t }) - - actor.bannerId = null - actor.Banner = null - } - } catch (err) { - logger.error('Cannot remove old image of actor %s.', actor.url, { err }) - } - - return actor -} - -async function fetchActorTotalItems (url: string) { - try { - const { body } = await doJSONRequest>(url, { activityPub: true }) - - return body.totalItems || 0 - } catch (err) { - logger.warn('Cannot fetch remote actor count %s.', url, { err }) - return 0 - } -} - -function getImageInfoIfExists (actorJSON: ActivityPubActor, type: ActorImageType) { - const mimetypes = MIMETYPES.IMAGE - const icon = type === ActorImageType.AVATAR - ? actorJSON.icon - : actorJSON.image - - if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined - - let extension: string - - if (icon.mediaType) { - extension = mimetypes.MIMETYPE_EXT[icon.mediaType] - } else { - const tmp = extname(icon.url) - - if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp - } - - if (!extension) return undefined - - return { - name: uuidv4() + extension, - fileUrl: icon.url, - height: icon.height, - width: icon.width, - type - } -} - -async function addFetchOutboxJob (actor: Pick) { - // Don't fetch ourselves - const serverActor = await getServerActor() - if (serverActor.id === actor.id) { - logger.error('Cannot fetch our own outbox!') - return undefined - } - - const payload = { - uri: actor.outboxUrl, - type: 'activity' as 'activity' - } - - return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) -} - -async function refreshActorIfNeeded ( - actorArg: T, - fetchedType: ActorFetchByUrlType -): Promise<{ actor: T | MActorFull, refreshed: boolean }> { - if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } - - // We need more attributes - const actor = fetchedType === 'all' - ? actorArg as MActorFull - : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) - - try { - let actorUrl: string - try { - actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) - } catch (err) { - logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) - actorUrl = actor.url - } - - const { result } = await fetchRemoteActor(actorUrl) - - if (result === undefined) { - logger.warn('Cannot fetch remote actor in refresh actor.') - return { actor, refreshed: false } - } - - return sequelizeTypescript.transaction(async t => { - updateInstanceWithAnother(actor, result.actor) - - await updateActorImageInstance(actor, ActorImageType.AVATAR, result.avatar, t) - await updateActorImageInstance(actor, ActorImageType.BANNER, result.banner, t) - - // Force update - actor.setDataValue('updatedAt', new Date()) - await actor.save({ transaction: t }) - - if (actor.Account) { - actor.Account.name = result.name - actor.Account.description = result.summary - - await actor.Account.save({ transaction: t }) - } else if (actor.VideoChannel) { - actor.VideoChannel.name = result.name - actor.VideoChannel.description = result.summary - actor.VideoChannel.support = result.support - - await actor.VideoChannel.save({ transaction: t }) - } - - return { refreshed: true, actor } - }) - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) - actor.Account - ? await actor.Account.destroy() - : await actor.VideoChannel.destroy() - - return { actor: undefined, refreshed: false } - } - - logger.warn('Cannot refresh actor %s.', actor.url, { err }) - return { actor, refreshed: false } - } -} - -export { - getOrCreateActorAndServerAndModel, - buildActorInstance, - generateAndSaveActorKeys, - fetchActorTotalItems, - getImageInfoIfExists, - updateActorInstance, - deleteActorImageInstance, - refreshActorIfNeeded, - updateActorImageInstance, - addFetchOutboxJob -} - -// --------------------------------------------------------------------------- - -function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) { - const id = imageModel - ? imageModel.id - : null - - if (type === ActorImageType.AVATAR) { - actorModel.avatarId = id - actorModel.Avatar = imageModel - } else { - actorModel.bannerId = id - actorModel.Banner = imageModel - } - - return actorModel -} - -function saveActorAndServerAndModelIfNotExist ( - result: FetchRemoteActorResult, - ownerActor?: MActorFullActor, - t?: Transaction -): Bluebird | Promise { - const actor = result.actor - - if (t !== undefined) return save(t) - - return sequelizeTypescript.transaction(t => save(t)) - - async function save (t: Transaction) { - const actorHost = new URL(actor.url).host - - const serverOptions = { - where: { - host: actorHost - }, - defaults: { - host: actorHost - }, - transaction: t - } - const [ server ] = await ServerModel.findOrCreate(serverOptions) - - // Save our new account in database - actor.serverId = server.id - - // Avatar? - if (result.avatar) { - const avatar = await ActorImageModel.create({ - filename: result.avatar.name, - fileUrl: result.avatar.fileUrl, - width: result.avatar.width, - height: result.avatar.height, - onDisk: false, - type: ActorImageType.AVATAR - }, { transaction: t }) - - actor.avatarId = avatar.id - } - - // Banner? - if (result.banner) { - const banner = await ActorImageModel.create({ - filename: result.banner.name, - fileUrl: result.banner.fileUrl, - width: result.banner.width, - height: result.banner.height, - onDisk: false, - type: ActorImageType.BANNER - }, { transaction: t }) - - actor.bannerId = banner.id - } - - // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists - // (which could be false in a retried query) - const [ actorCreated, created ] = await ActorModel.findOrCreate({ - defaults: actor.toJSON(), - where: { - [Op.or]: [ - { - url: actor.url - }, - { - serverId: actor.serverId, - preferredUsername: actor.preferredUsername - } - ] - }, - transaction: t - }) - - // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards - if (created !== true && actorCreated.url !== actor.url) { - // Only fix http://example.com/account/djidane to https://example.com/account/djidane - if (actorCreated.url.replace(/^http:\/\//, '') !== actor.url.replace(/^https:\/\//, '')) { - throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${actor.url}`) - } - - actorCreated.url = actor.url - await actorCreated.save({ transaction: t }) - } - - if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { - actorCreated.Account = await saveAccount(actorCreated, result, t) as MAccountDefault - actorCreated.Account.Actor = actorCreated - } else if (actorCreated.type === 'Group') { // Video channel - const channel = await saveVideoChannel(actorCreated, result, ownerActor, t) - actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: ownerActor.Account }) - } - - actorCreated.Server = server - - return actorCreated - } -} - -type ImageResult = { - name: string - fileUrl: string - height: number - width: number -} - -type FetchRemoteActorResult = { - actor: MActor - name: string - summary: string - support?: string - playlists?: string - avatar?: ImageResult - banner?: ImageResult - attributedTo: ActivityPubAttributedTo[] -} -async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { - logger.info('Fetching remote actor %s.', actorUrl) - - const requestResult = await doJSONRequest(actorUrl, { activityPub: true }) - const actorJSON = requestResult.body - - if (sanitizeAndCheckActorObject(actorJSON) === false) { - logger.debug('Remote actor JSON is not valid.', { actorJSON }) - return { result: undefined, statusCode: requestResult.statusCode } - } - - if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { - logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) - return { result: undefined, statusCode: requestResult.statusCode } - } - - const followersCount = await fetchActorTotalItems(actorJSON.followers) - const followingCount = await fetchActorTotalItems(actorJSON.following) - - const actor = new ActorModel({ - type: actorJSON.type, - preferredUsername: actorJSON.preferredUsername, - url: actorJSON.id, - publicKey: actorJSON.publicKey.publicKeyPem, - privateKey: null, - followersCount: followersCount, - followingCount: followingCount, - inboxUrl: actorJSON.inbox, - outboxUrl: actorJSON.outbox, - followersUrl: actorJSON.followers, - followingUrl: actorJSON.following, - - sharedInboxUrl: actorJSON.endpoints?.sharedInbox - ? actorJSON.endpoints.sharedInbox - : null - }) - - const avatarInfo = getImageInfoIfExists(actorJSON, ActorImageType.AVATAR) - const bannerInfo = getImageInfoIfExists(actorJSON, ActorImageType.BANNER) - - const name = actorJSON.name || actorJSON.preferredUsername - return { - statusCode: requestResult.statusCode, - result: { - actor, - name, - avatar: avatarInfo, - banner: bannerInfo, - summary: actorJSON.summary, - support: actorJSON.support, - playlists: actorJSON.playlists, - attributedTo: actorJSON.attributedTo - } - } -} - -async function saveAccount (actor: MActorId, result: FetchRemoteActorResult, t: Transaction) { - const [ accountCreated ] = await AccountModel.findOrCreate({ - defaults: { - name: result.name, - description: result.summary, - actorId: actor.id - }, - where: { - actorId: actor.id - }, - transaction: t - }) - - return accountCreated as MAccount -} - -async function saveVideoChannel (actor: MActorId, result: FetchRemoteActorResult, ownerActor: MActorAccountId, t: Transaction) { - const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ - defaults: { - name: result.name, - description: result.summary, - support: result.support, - actorId: actor.id, - accountId: ownerActor.Account.id - }, - where: { - actorId: actor.id - }, - transaction: t - }) - - return videoChannelCreated as MChannel -} diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts new file mode 100644 index 000000000..0d5bea789 --- /dev/null +++ b/server/lib/activitypub/actors/get.ts @@ -0,0 +1,119 @@ + +import { checkUrlsSameHost, getAPId } from '@server/helpers/activitypub' +import { ActorFetchByUrlType, fetchActorByUrl } from '@server/helpers/actor' +import { retryTransactionWrapper } from '@server/helpers/database-utils' +import { logger } from '@server/helpers/logger' +import { JobQueue } from '@server/lib/job-queue' +import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' +import { ActivityPubActor } from '@shared/models' +import { refreshActorIfNeeded } from './refresh' +import { APActorCreator, fetchRemoteActor } from './shared' + +function getOrCreateAPActor ( + activityActor: string | ActivityPubActor, + fetchType: 'all', + recurseIfNeeded?: boolean, + updateCollections?: boolean +): Promise + +function getOrCreateAPActor ( + activityActor: string | ActivityPubActor, + fetchType?: 'association-ids', + recurseIfNeeded?: boolean, + updateCollections?: boolean +): Promise + +async function getOrCreateAPActor ( + activityActor: string | ActivityPubActor, + fetchType: ActorFetchByUrlType = 'association-ids', + recurseIfNeeded = true, + updateCollections = false +): Promise { + const actorUrl = getAPId(activityActor) + let actor = await loadActorFromDB(actorUrl, fetchType) + + let created = false + let accountPlaylistsUrl: string + + // We don't have this actor in our database, fetch it on remote + if (!actor) { + const { actorObject } = await fetchRemoteActor(actorUrl) + if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) + + // Create the attributed to actor + // In PeerTube a video channel is owned by an account + let ownerActor: MActorFullActor + if (recurseIfNeeded === true && actorObject.type === 'Group') { + ownerActor = await getOrCreateAPOwner(actorObject, actorUrl) + } + + const creator = new APActorCreator(actorObject, ownerActor) + actor = await retryTransactionWrapper(creator.create.bind(creator)) + created = true + accountPlaylistsUrl = actorObject.playlists + } + + if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor + if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor + + const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) + if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') + + await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) + await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl) + + return actorRefreshed +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPActor +} + +// --------------------------------------------------------------------------- + +async function loadActorFromDB (actorUrl: string, fetchType: ActorFetchByUrlType) { + let actor = await fetchActorByUrl(actorUrl, fetchType) + + // Orphan actor (not associated to an account of channel) so recreate it + if (actor && (!actor.Account && !actor.VideoChannel)) { + await actor.destroy() + actor = null + } + + return actor +} + +function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { + const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person') + if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl) + + if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { + throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) + } + + try { + // Don't recurse another time + const recurseIfNeeded = false + return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded) + } catch (err) { + logger.error('Cannot get or create account attributed to video channel ' + actorUrl) + throw new Error(err) + } +} + +async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { + if ((created === true || refreshed === true) && updateCollections === true) { + const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } + await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) + } +} + +async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { + // We created a new account: fetch the playlists + if (created === true && actor.Account && accountPlaylistsUrl) { + const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' } + await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) + } +} diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts new file mode 100644 index 000000000..443ad0a63 --- /dev/null +++ b/server/lib/activitypub/actors/image.ts @@ -0,0 +1,94 @@ +import { Transaction } from 'sequelize/types' +import { logger } from '@server/helpers/logger' +import { ActorImageModel } from '@server/models/actor/actor-image' +import { MActorImage, MActorImages } from '@server/types/models' +import { ActorImageType } from '@shared/models' + +type ImageInfo = { + name: string + fileUrl: string + height: number + width: number + onDisk?: boolean +} + +async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) { + const oldImageModel = type === ActorImageType.AVATAR + ? actor.Avatar + : actor.Banner + + if (oldImageModel) { + // Don't update the avatar if the file URL did not change + if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor + + try { + await oldImageModel.destroy({ transaction: t }) + + setActorImage(actor, type, null) + } catch (err) { + logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) + } + } + + if (imageInfo) { + const imageModel = await ActorImageModel.create({ + filename: imageInfo.name, + onDisk: imageInfo.onDisk ?? false, + fileUrl: imageInfo.fileUrl, + height: imageInfo.height, + width: imageInfo.width, + type + }, { transaction: t }) + + setActorImage(actor, type, imageModel) + } + + return actor +} + +async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) { + try { + if (type === ActorImageType.AVATAR) { + await actor.Avatar.destroy({ transaction: t }) + + actor.avatarId = null + actor.Avatar = null + } else { + await actor.Banner.destroy({ transaction: t }) + + actor.bannerId = null + actor.Banner = null + } + } catch (err) { + logger.error('Cannot remove old image of actor %s.', actor.url, { err }) + } + + return actor +} + +// --------------------------------------------------------------------------- + +export { + ImageInfo, + + updateActorImageInstance, + deleteActorImageInstance +} + +// --------------------------------------------------------------------------- + +function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) { + const id = imageModel + ? imageModel.id + : null + + if (type === ActorImageType.AVATAR) { + actorModel.avatarId = id + actorModel.Avatar = imageModel + } else { + actorModel.bannerId = id + actorModel.Banner = imageModel + } + + return actorModel +} diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts new file mode 100644 index 000000000..a54da6798 --- /dev/null +++ b/server/lib/activitypub/actors/index.ts @@ -0,0 +1,5 @@ +export * from './get' +export * from './image' +export * from './keys' +export * from './refresh' +export * from './updater' diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts new file mode 100644 index 000000000..c3d18abd8 --- /dev/null +++ b/server/lib/activitypub/actors/keys.ts @@ -0,0 +1,16 @@ +import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto' +import { MActor } from '@server/types/models' + +// Set account keys, this could be long so process after the account creation and do not block the client +async function generateAndSaveActorKeys (actor: T) { + const { publicKey, privateKey } = await createPrivateAndPublicKeys() + + actor.publicKey = publicKey + actor.privateKey = privateKey + + return actor.save() +} + +export { + generateAndSaveActorKeys +} diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts new file mode 100644 index 000000000..ff3b249d0 --- /dev/null +++ b/server/lib/activitypub/actors/refresh.ts @@ -0,0 +1,63 @@ +import { ActorFetchByUrlType } from '@server/helpers/actor' +import { logger } from '@server/helpers/logger' +import { PeerTubeRequestError } from '@server/helpers/requests' +import { getUrlFromWebfinger } from '@server/helpers/webfinger' +import { ActorModel } from '@server/models/actor/actor' +import { MActorAccountChannelId, MActorFull } from '@server/types/models' +import { HttpStatusCode } from '@shared/core-utils' +import { fetchRemoteActor } from './shared' +import { APActorUpdater } from './updater' + +async function refreshActorIfNeeded ( + actorArg: T, + fetchedType: ActorFetchByUrlType +): Promise<{ actor: T | MActorFull, refreshed: boolean }> { + if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } + + // We need more attributes + const actor = fetchedType === 'all' + ? actorArg as MActorFull + : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) + + try { + const actorUrl = await getActorUrl(actor) + const { actorObject } = await fetchRemoteActor(actorUrl) + + if (actorObject === undefined) { + logger.warn('Cannot fetch remote actor in refresh actor.') + return { actor, refreshed: false } + } + + const updater = new APActorUpdater(actorObject, actor) + await updater.update() + + return { refreshed: true, actor } + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) + + actor.Account + ? await actor.Account.destroy() + : await actor.VideoChannel.destroy() + + return { actor: undefined, refreshed: false } + } + + logger.warn('Cannot refresh actor %s.', actor.url, { err }) + return { actor, refreshed: false } + } +} + +export { + refreshActorIfNeeded +} + +// --------------------------------------------------------------------------- + +function getActorUrl (actor: MActorFull) { + return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + .catch(err => { + logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err) + return actor.url + }) +} diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts new file mode 100644 index 000000000..999aed97d --- /dev/null +++ b/server/lib/activitypub/actors/shared/creator.ts @@ -0,0 +1,149 @@ +import { Op, Transaction } from 'sequelize' +import { sequelizeTypescript } from '@server/initializers/database' +import { AccountModel } from '@server/models/account/account' +import { ActorModel } from '@server/models/actor/actor' +import { ServerModel } from '@server/models/server/server' +import { VideoChannelModel } from '@server/models/video/video-channel' +import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models' +import { ActivityPubActor, ActorImageType } from '@shared/models' +import { updateActorImageInstance } from '../image' +import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImageInfoFromObject } from './object-to-model-attributes' +import { fetchActorFollowsCount } from './url-to-object' + +export class APActorCreator { + + constructor ( + private readonly actorObject: ActivityPubActor, + private readonly ownerActor?: MActorFullActor + ) { + + } + + async create (): Promise { + const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject) + + const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount)) + + return sequelizeTypescript.transaction(async t => { + const server = await this.setServer(actorInstance, t) + + await this.setImageIfNeeded(actorInstance, ActorImageType.AVATAR, t) + await this.setImageIfNeeded(actorInstance, ActorImageType.BANNER, t) + + const { actorCreated, created } = await this.saveActor(actorInstance, t) + + await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t) + + if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance + actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault + actorCreated.Account.Actor = actorCreated + } + + if (actorCreated.type === 'Group') { // Video channel + const channel = await this.saveVideoChannel(actorCreated, t) + actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account }) + } + + actorCreated.Server = server + + return actorCreated + }) + } + + private async setServer (actor: MActor, t: Transaction) { + const actorHost = new URL(actor.url).host + + const serverOptions = { + where: { + host: actorHost + }, + defaults: { + host: actorHost + }, + transaction: t + } + const [ server ] = await ServerModel.findOrCreate(serverOptions) + + // Save our new account in database + actor.serverId = server.id + + return server as MServer + } + + private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) { + const imageInfo = getImageInfoFromObject(this.actorObject, type) + if (!imageInfo) return + + return updateActorImageInstance(actor as MActorImages, type, imageInfo, t) + } + + private async saveActor (actor: MActor, t: Transaction) { + // Force the actor creation using findOrCreate() instead of save() + // Sometimes Sequelize skips the save() when it thinks the instance already exists + // (which could be false in a retried query) + const [ actorCreated, created ] = await ActorModel.findOrCreate({ + defaults: actor.toJSON(), + where: { + [Op.or]: [ + { + url: actor.url + }, + { + serverId: actor.serverId, + preferredUsername: actor.preferredUsername + } + ] + }, + transaction: t + }) + + return { actorCreated, created } + } + + private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) { + // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards + if (created !== true && actorCreated.url !== newActor.url) { + // Only fix http://example.com/account/djidane to https://example.com/account/djidane + if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) { + throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`) + } + + actorCreated.url = newActor.url + await actorCreated.save({ transaction: t }) + } + } + + private async saveAccount (actor: MActorId, t: Transaction) { + const [ accountCreated ] = await AccountModel.findOrCreate({ + defaults: { + name: getActorDisplayNameFromObject(this.actorObject), + description: this.actorObject.summary, + actorId: actor.id + }, + where: { + actorId: actor.id + }, + transaction: t + }) + + return accountCreated as MAccount + } + + private async saveVideoChannel (actor: MActorId, t: Transaction) { + const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ + defaults: { + name: getActorDisplayNameFromObject(this.actorObject), + description: this.actorObject.summary, + support: this.actorObject.support, + actorId: actor.id, + accountId: this.ownerActor.Account.id + }, + where: { + actorId: actor.id + }, + transaction: t + }) + + return videoChannelCreated as MChannel + } +} diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts new file mode 100644 index 000000000..a2ff468cf --- /dev/null +++ b/server/lib/activitypub/actors/shared/index.ts @@ -0,0 +1,3 @@ +export * from './creator' +export * from './url-to-object' +export * from './object-to-model-attributes' diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..66b22c952 --- /dev/null +++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts @@ -0,0 +1,70 @@ +import { extname } from 'path' +import { v4 as uuidv4 } from 'uuid' +import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' +import { MIMETYPES } from '@server/initializers/constants' +import { ActorModel } from '@server/models/actor/actor' +import { FilteredModelAttributes } from '@server/types' +import { ActivityPubActor, ActorImageType } from '@shared/models' + +function getActorAttributesFromObject ( + actorObject: ActivityPubActor, + followersCount: number, + followingCount: number +): FilteredModelAttributes { + return { + type: actorObject.type, + preferredUsername: actorObject.preferredUsername, + url: actorObject.id, + publicKey: actorObject.publicKey.publicKeyPem, + privateKey: null, + followersCount, + followingCount, + inboxUrl: actorObject.inbox, + outboxUrl: actorObject.outbox, + followersUrl: actorObject.followers, + followingUrl: actorObject.following, + + sharedInboxUrl: actorObject.endpoints?.sharedInbox + ? actorObject.endpoints.sharedInbox + : null + } +} + +function getImageInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) { + const mimetypes = MIMETYPES.IMAGE + const icon = type === ActorImageType.AVATAR + ? actorObject.icon + : actorObject.image + + if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined + + let extension: string + + if (icon.mediaType) { + extension = mimetypes.MIMETYPE_EXT[icon.mediaType] + } else { + const tmp = extname(icon.url) + + if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp + } + + if (!extension) return undefined + + return { + name: uuidv4() + extension, + fileUrl: icon.url, + height: icon.height, + width: icon.width, + type + } +} + +function getActorDisplayNameFromObject (actorObject: ActivityPubActor) { + return actorObject.name || actorObject.preferredUsername +} + +export { + getActorAttributesFromObject, + getImageInfoFromObject, + getActorDisplayNameFromObject +} diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts new file mode 100644 index 000000000..f4f16b044 --- /dev/null +++ b/server/lib/activitypub/actors/shared/url-to-object.ts @@ -0,0 +1,54 @@ + +import { checkUrlsSameHost } from '@server/helpers/activitypub' +import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor' +import { logger } from '@server/helpers/logger' +import { doJSONRequest } from '@server/helpers/requests' +import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models' + +async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { + logger.info('Fetching remote actor %s.', actorUrl) + + const { body, statusCode } = await doJSONRequest(actorUrl, { activityPub: true }) + + if (sanitizeAndCheckActorObject(body) === false) { + logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) + return { actorObject: undefined, statusCode: statusCode } + } + + if (checkUrlsSameHost(body.id, actorUrl) !== true) { + logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id) + return { actorObject: undefined, statusCode: statusCode } + } + + return { + statusCode, + + actorObject: body + } +} + +async function fetchActorFollowsCount (actorObject: ActivityPubActor) { + const followersCount = await fetchActorTotalItems(actorObject.followers) + const followingCount = await fetchActorTotalItems(actorObject.following) + + return { followersCount, followingCount } +} + +// --------------------------------------------------------------------------- +export { + fetchActorFollowsCount, + fetchRemoteActor +} + +// --------------------------------------------------------------------------- + +async function fetchActorTotalItems (url: string) { + try { + const { body } = await doJSONRequest>(url, { activityPub: true }) + + return body.totalItems || 0 + } catch (err) { + logger.warn('Cannot fetch remote actor count %s.', url, { err }) + return 0 + } +} diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts new file mode 100644 index 000000000..471688f11 --- /dev/null +++ b/server/lib/activitypub/actors/updater.ts @@ -0,0 +1,90 @@ +import { resetSequelizeInstance } from '@server/helpers/database-utils' +import { logger } from '@server/helpers/logger' +import { sequelizeTypescript } from '@server/initializers/database' +import { VideoChannelModel } from '@server/models/video/video-channel' +import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' +import { ActivityPubActor, ActorImageType } from '@shared/models' +import { updateActorImageInstance } from './image' +import { fetchActorFollowsCount } from './shared' +import { getImageInfoFromObject } from './shared/object-to-model-attributes' + +export class APActorUpdater { + + private accountOrChannel: MAccount | MChannel + + private readonly actorFieldsSave: object + private readonly accountOrChannelFieldsSave: object + + constructor ( + private readonly actorObject: ActivityPubActor, + private readonly actor: MActorFull + ) { + this.actorFieldsSave = this.actor.toJSON() + + if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel + else this.accountOrChannel = this.actor.Account + + this.accountOrChannelFieldsSave = this.accountOrChannel.toJSON() + } + + async update () { + const avatarInfo = getImageInfoFromObject(this.actorObject, ActorImageType.AVATAR) + const bannerInfo = getImageInfoFromObject(this.actorObject, ActorImageType.BANNER) + + try { + await sequelizeTypescript.transaction(async t => { + await this.updateActorInstance(this.actor, this.actorObject) + + await updateActorImageInstance(this.actor, ActorImageType.AVATAR, avatarInfo, t) + await updateActorImageInstance(this.actor, ActorImageType.BANNER, bannerInfo, t) + + await this.actor.save({ transaction: t }) + + this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername + this.accountOrChannel.description = this.actorObject.summary + + if (this.accountOrChannel instanceof VideoChannelModel) this.accountOrChannel.support = this.actorObject.support + + await this.accountOrChannel.save({ transaction: t }) + }) + + logger.info('Remote account %s updated', this.actorObject.url) + } catch (err) { + if (this.actor !== undefined && this.actorFieldsSave !== undefined) { + resetSequelizeInstance(this.actor, this.actorFieldsSave) + } + + if (this.accountOrChannel !== undefined && this.accountOrChannelFieldsSave !== undefined) { + resetSequelizeInstance(this.accountOrChannel, this.accountOrChannelFieldsSave) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote account.', { err }) + throw err + } + } + + private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) { + const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject) + + actorInstance.type = actorObject.type + actorInstance.preferredUsername = actorObject.preferredUsername + actorInstance.url = actorObject.id + actorInstance.publicKey = actorObject.publicKey.publicKeyPem + actorInstance.followersCount = followersCount + actorInstance.followingCount = followingCount + actorInstance.inboxUrl = actorObject.inbox + actorInstance.outboxUrl = actorObject.outbox + actorInstance.followersUrl = actorObject.followers + actorInstance.followingUrl = actorObject.following + + if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published) + + if (actorObject.endpoints?.sharedInbox) { + actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox + } + + // Force actor update + actorInstance.changed('updatedAt', true) + } +} diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts new file mode 100644 index 000000000..ecdc33a77 --- /dev/null +++ b/server/lib/activitypub/outbox.ts @@ -0,0 +1,24 @@ +import { logger } from '@server/helpers/logger' +import { ActorModel } from '@server/models/actor/actor' +import { getServerActor } from '@server/models/application/application' +import { JobQueue } from '../job-queue' + +async function addFetchOutboxJob (actor: Pick) { + // Don't fetch ourselves + const serverActor = await getServerActor() + if (serverActor.id === actor.id) { + logger.error('Cannot fetch our own outbox!') + return undefined + } + + const payload = { + uri: actor.outboxUrl, + type: 'activity' as 'activity' + } + + return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) +} + +export { + addFetchOutboxJob +} diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts index 886b1f288..fcfcc41a2 100644 --- a/server/lib/activitypub/playlists/create-update.ts +++ b/server/lib/activitypub/playlists/create-update.ts @@ -9,7 +9,7 @@ import { FilteredModelAttributes } from '@server/types' import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { PlaylistObject } from '@shared/models' -import { getOrCreateActorAndServerAndModel } from '../actor' +import { getOrCreateAPActor } from '../actors' import { crawlCollectionPage } from '../crawl' import { getOrCreateAPVideo } from '../videos' import { @@ -75,7 +75,7 @@ export { async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return - const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0]) + const actor = await getOrCreateAPActor(playlistObject.attributedTo[0]) if (!actor.VideoChannel) { logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts index 8ad470cf4..077b01eda 100644 --- a/server/lib/activitypub/process/process-accept.ts +++ b/server/lib/activitypub/process/process-accept.ts @@ -2,7 +2,7 @@ import { ActivityAccept } from '../../../../shared/models/activitypub' import { ActorFollowModel } from '../../../models/actor/actor-follow' import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorDefault, MActorSignature } from '../../../types/models' -import { addFetchOutboxJob } from '../actor' +import { addFetchOutboxJob } from '../outbox' async function processAcceptActivity (options: APProcessorOptions) { const { byActor: targetActor, inboxActor } = options diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d2b63c901..aa80d5d09 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -1,19 +1,16 @@ import { isRedundancyAccepted } from '@server/lib/redundancy' -import { ActorImageType } from '@shared/models' import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' -import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' +import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers/database' -import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/actor/actor' -import { VideoChannelModel } from '../../../models/video/video-channel' import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { getImageInfoIfExists, updateActorImageInstance, updateActorInstance } from '../actor' +import { MActorFull, MActorSignature } from '../../../types/models' +import { APActorUpdater } from '../actors/updater' import { createOrUpdateCacheFile } from '../cache-file' import { createOrUpdateVideoPlaylist } from '../playlists' import { forwardVideoRelatedActivity } from '../send/utils' @@ -99,56 +96,13 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ } } -async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { - const actorAttributesToUpdate = activity.object as ActivityPubActor +async function processUpdateActor (actor: MActorFull, activity: ActivityUpdate) { + const actorObject = activity.object as ActivityPubActor - logger.debug('Updating remote account "%s".', actorAttributesToUpdate.url) - let accountOrChannelInstance: AccountModel | VideoChannelModel - let actorFieldsSave: object - let accountOrChannelFieldsSave: object + logger.debug('Updating remote account "%s".', actorObject.url) - // Fetch icon? - const avatarInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.AVATAR) - const bannerInfo = getImageInfoIfExists(actorAttributesToUpdate, ActorImageType.BANNER) - - try { - await sequelizeTypescript.transaction(async t => { - actorFieldsSave = actor.toJSON() - - if (actorAttributesToUpdate.type === 'Group') accountOrChannelInstance = actor.VideoChannel - else accountOrChannelInstance = actor.Account - - accountOrChannelFieldsSave = accountOrChannelInstance.toJSON() - - await updateActorInstance(actor, actorAttributesToUpdate) - - await updateActorImageInstance(actor, ActorImageType.AVATAR, avatarInfo, t) - await updateActorImageInstance(actor, ActorImageType.BANNER, bannerInfo, t) - - await actor.save({ transaction: t }) - - accountOrChannelInstance.name = actorAttributesToUpdate.name || actorAttributesToUpdate.preferredUsername - accountOrChannelInstance.description = actorAttributesToUpdate.summary - - if (accountOrChannelInstance instanceof VideoChannelModel) accountOrChannelInstance.support = actorAttributesToUpdate.support - - await accountOrChannelInstance.save({ transaction: t }) - }) - - logger.info('Remote account %s updated', actorAttributesToUpdate.url) - } catch (err) { - if (actor !== undefined && actorFieldsSave !== undefined) { - resetSequelizeInstance(actor, actorFieldsSave) - } - - if (accountOrChannelInstance !== undefined && accountOrChannelFieldsSave !== undefined) { - resetSequelizeInstance(accountOrChannelInstance, accountOrChannelFieldsSave) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote account.', { err }) - throw err - } + const updater = new APActorUpdater(actorObject, actor) + return updater.update() } async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) { diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts index 5cef75665..02a23d098 100644 --- a/server/lib/activitypub/process/process.ts +++ b/server/lib/activitypub/process/process.ts @@ -1,22 +1,22 @@ +import { StatsManager } from '@server/lib/stat-manager' import { Activity, ActivityType } from '../../../../shared/models/activitypub' import { checkUrlsSameHost, getAPId } from '../../../helpers/activitypub' import { logger } from '../../../helpers/logger' +import { APProcessorOptions } from '../../../types/activitypub-processor.model' +import { MActorDefault, MActorSignature } from '../../../types/models' +import { getOrCreateAPActor } from '../actors' import { processAcceptActivity } from './process-accept' import { processAnnounceActivity } from './process-announce' import { processCreateActivity } from './process-create' import { processDeleteActivity } from './process-delete' +import { processDislikeActivity } from './process-dislike' +import { processFlagActivity } from './process-flag' import { processFollowActivity } from './process-follow' import { processLikeActivity } from './process-like' import { processRejectActivity } from './process-reject' import { processUndoActivity } from './process-undo' import { processUpdateActivity } from './process-update' -import { getOrCreateActorAndServerAndModel } from '../actor' -import { processDislikeActivity } from './process-dislike' -import { processFlagActivity } from './process-flag' import { processViewActivity } from './process-view' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorDefault, MActorSignature } from '../../../types/models' -import { StatsManager } from '@server/lib/stat-manager' const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions) => Promise } = { Create: processCreateActivity, @@ -65,7 +65,7 @@ async function processActivities ( continue } - const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateActorAndServerAndModel(actorUrl) + const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl) actorsCache[actorUrl] = byActor const activityProcessor = processActivity[activity.type] diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 327955dd2..1ff01a175 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -7,7 +7,7 @@ import { doJSONRequest } from '../../helpers/requests' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoShareModel } from '../../models/video/video-share' import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' -import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateAPActor } from './actors' import { sendUndoAnnounce, sendVideoAnnounce } from './send' import { getLocalVideoAnnounceActivityPubUrl } from './url' @@ -64,7 +64,7 @@ async function addVideoShare (shareUrl: string, video: MVideoId) { throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) } - const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const actor = await getOrCreateAPActor(actorUrl) const entry = { actorId: actor.id, diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 760da719d..6b7f9504f 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -6,7 +6,7 @@ import { doJSONRequest } from '../../helpers/requests' import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoCommentModel } from '../../models/video/video-comment' import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' -import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateAPActor } from './actors' import { getOrCreateAPVideo } from './videos' type ResolveThreadParams = { @@ -147,7 +147,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { } const actor = actorUrl - ? await getOrCreateActorAndServerAndModel(actorUrl, 'all') + ? await getOrCreateAPActor(actorUrl, 'all') : null const comment = new VideoCommentModel({ diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 091f4ec23..0eec806f9 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -7,7 +7,7 @@ import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' -import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateAPActor } from './actors' import { sendLike, sendUndoDislike, sendUndoLike } from './send' import { sendDislike } from './send/send-dislike' import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' @@ -74,7 +74,7 @@ async function createRate (rateUrl: string, video: MVideo, rate: VideoRateType) throw new Error(`Rate url ${rateUrl} host is different from the AP object id ${body.id}`) } - const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const actor = await getOrCreateAPActor(actorUrl) const entry = { videoId: video.id, diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index 953710f6c..f8e4d6aa3 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -10,7 +10,7 @@ import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models' import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' -import { getOrCreateActorAndServerAndModel } from '../../actor' +import { getOrCreateAPActor } from '../../actors' import { getCaptionAttributesFromObject, getFileAttributesFromUrl, @@ -34,7 +34,7 @@ export abstract class APVideoAbstractBuilder { throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${this.videoObject.id}`) } - return getOrCreateActorAndServerAndModel(channel.id, 'all') + return getOrCreateAPActor(channel.id, 'all') } protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise { diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 9e1c74969..6745e2efd 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts @@ -126,7 +126,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { this.video.views = videoData.views this.video.isLive = videoData.isLive - // Ensures we update the updated video attribute + // Ensures we update the updatedAt attribute, even if main attributes did not change this.video.changed('updatedAt', true) return this.video.save({ transaction }) as Promise diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts index ec8df8969..76b6fcaae 100644 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ b/server/lib/job-queue/handlers/activitypub-follow.ts @@ -10,7 +10,7 @@ import { sequelizeTypescript } from '../../../initializers/database' import { ActorModel } from '../../../models/actor/actor' import { ActorFollowModel } from '../../../models/actor/actor-follow' import { MActor, MActorFollowActors, MActorFull } from '../../../types/models' -import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor' +import { getOrCreateAPActor } from '../../activitypub/actors' import { sendFollow } from '../../activitypub/send' import { Notifier } from '../../notifier' @@ -26,7 +26,7 @@ async function processActivityPubFollow (job: Bull.Job) { } else { const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) - targetActor = await getOrCreateActorAndServerAndModel(actorUrl, 'all') + targetActor = await getOrCreateAPActor(actorUrl, 'all') } if (payload.assertIsChannel && !targetActor.VideoChannel) { diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 10e6895da..29483f310 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -6,7 +6,7 @@ import { logger } from '../../../helpers/logger' import { fetchVideoByUrl } from '../../../helpers/video' import { ActorModel } from '../../../models/actor/actor' import { VideoPlaylistModel } from '../../../models/video/video-playlist' -import { refreshActorIfNeeded } from '../../activitypub/actor' +import { refreshActorIfNeeded } from '../../activitypub/actors' async function refreshAPObject (job: Bull.Job) { const payload = job.data as RefreshPayload diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts index 3eef565d0..60ac61afd 100644 --- a/server/lib/job-queue/handlers/actor-keys.ts +++ b/server/lib/job-queue/handlers/actor-keys.ts @@ -1,5 +1,5 @@ import * as Bull from 'bull' -import { generateAndSaveActorKeys } from '@server/lib/activitypub/actor' +import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors' import { ActorModel } from '@server/models/actor/actor' import { ActorKeysPayload } from '@shared/models' import { logger } from '../../../helpers/logger' diff --git a/server/lib/actor-image.ts b/server/lib/local-actor.ts similarity index 78% rename from server/lib/actor-image.ts rename to server/lib/local-actor.ts index f271f0b5b..55e77dd04 100644 --- a/server/lib/actor-image.ts +++ b/server/lib/local-actor.ts @@ -3,17 +3,35 @@ import { queue } from 'async' import * as LRUCache from 'lru-cache' import { extname, join } from 'path' import { v4 as uuidv4 } from 'uuid' -import { ActorImageType } from '@shared/models' +import { ActorModel } from '@server/models/actor/actor' +import { ActivityPubActorType, ActorImageType } from '@shared/models' import { retryTransactionWrapper } from '../helpers/database-utils' import { processImage } from '../helpers/image-utils' import { downloadImage } from '../helpers/requests' import { CONFIG } from '../initializers/config' -import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants' +import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY, WEBSERVER } from '../initializers/constants' import { sequelizeTypescript } from '../initializers/database' -import { MAccountDefault, MChannelDefault } from '../types/models' -import { deleteActorImageInstance, updateActorImageInstance } from './activitypub/actor' +import { MAccountDefault, MActor, MChannelDefault } from '../types/models' +import { deleteActorImageInstance, updateActorImageInstance } from './activitypub/actors' import { sendUpdateActor } from './activitypub/send' +function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { + return new ActorModel({ + type, + url, + preferredUsername, + publicKey: null, + privateKey: null, + followersCount: 0, + followingCount: 0, + inboxUrl: url + '/inbox', + outboxUrl: url + '/outbox', + sharedInboxUrl: WEBSERVER.URL + '/inbox', + followersUrl: url + '/followers', + followingUrl: url + '/following' + }) as MActor +} + async function updateLocalActorImageFile ( accountOrChannel: MAccountDefault | MChannelDefault, imagePhysicalFile: Express.Multer.File, @@ -93,5 +111,6 @@ export { actorImagePathUnsafeCache, updateLocalActorImageFile, deleteLocalActorImageFile, - pushActorImageProcessInQueue + pushActorImageProcessInQueue, + buildActorInstance } diff --git a/server/lib/user.ts b/server/lib/user.ts index 8a6fcebc7..a2163abb1 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -11,10 +11,11 @@ import { ActorModel } from '../models/actor/actor' import { UserNotificationSettingModel } from '../models/user/user-notification-setting' import { MAccountDefault, MChannelActor } from '../types/models' import { MUser, MUserDefault, MUserId } from '../types/models/user' -import { buildActorInstance, generateAndSaveActorKeys } from './activitypub/actor' +import { generateAndSaveActorKeys } from './activitypub/actors' import { getLocalAccountActivityPubUrl } from './activitypub/url' import { Emailer } from './emailer' import { LiveManager } from './live-manager' +import { buildActorInstance } from './local-actor' import { Redis } from './redis' import { createLocalVideoChannel } from './video-channel' import { createWatchLaterPlaylist } from './video-playlist' diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index d57e832fe..2fd63a8c4 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts @@ -3,9 +3,9 @@ import { VideoChannelCreate } from '../../shared/models' import { VideoModel } from '../models/video/video' import { VideoChannelModel } from '../models/video/video-channel' import { MAccountId, MChannelId } from '../types/models' -import { buildActorInstance } from './activitypub/actor' import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' import { federateVideoIfNeeded } from './activitypub/videos' +import { buildActorInstance } from './local-actor' async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) diff --git a/server/middlewares/activitypub.ts b/server/middlewares/activitypub.ts index 6cd23f230..a1fdfafcf 100644 --- a/server/middlewares/activitypub.ts +++ b/server/middlewares/activitypub.ts @@ -3,7 +3,7 @@ import { ActivityDelete, ActivityPubSignature } from '../../shared' import { logger } from '../helpers/logger' import { isHTTPSignatureVerified, isJsonLDSignatureVerified, parseHTTPSignature } from '../helpers/peertube-crypto' import { ACCEPT_HEADERS, ACTIVITY_PUB, HTTP_SIGNATURE } from '../initializers/constants' -import { getOrCreateActorAndServerAndModel } from '../lib/activitypub/actor' +import { getOrCreateAPActor } from '../lib/activitypub/actors' import { loadActorUrlOrGetFromWebfinger } from '../helpers/webfinger' import { isActorDeleteActivityValid } from '@server/helpers/custom-validators/activitypub/actor' import { getAPId } from '@server/helpers/activitypub' @@ -100,7 +100,7 @@ async function checkHttpSignature (req: Request, res: Response) { actorUrl = await loadActorUrlOrGetFromWebfinger(actorUrl.replace(/^acct:/, '')) } - const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const actor = await getOrCreateAPActor(actorUrl) const verified = isHTTPSignatureVerified(parsed, actor) if (verified !== true) { @@ -135,7 +135,7 @@ async function checkJsonLDSignature (req: Request, res: Response) { logger.debug('Checking JsonLD signature of actor %s...', creator) - const actor = await getOrCreateActorAndServerAndModel(creator) + const actor = await getOrCreateAPActor(creator) const verified = await isJsonLDSignatureVerified(actor, req.body) if (verified !== true) {