From 8795d6f254bd8f88c385bf77b82cc6f177c94df9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 26 Feb 2021 14:22:25 +0100 Subject: [PATCH] Fix broken local actors Some channels can't federate because they don't have public/private keys, maybe because the generation failed for various reasons --- .../app/+admin/system/jobs/jobs.component.ts | 2 ++ .../video-trending-header.component.ts | 6 ++-- server/controllers/api/users/index.ts | 2 +- server/controllers/api/video-channel.ts | 7 ++-- server/initializers/constants.ts | 7 ++-- .../migrations/0605-actor-missing-keys.ts | 34 +++++++++++++++++++ server/lib/activitypub/actor.ts | 20 +++++------ server/lib/job-queue/handlers/actor-keys.ts | 20 +++++++++++ server/lib/job-queue/job-queue.ts | 5 +++ server/lib/user.ts | 8 ++--- shared/models/server/job.model.ts | 5 +++ 11 files changed, 90 insertions(+), 26 deletions(-) create mode 100644 server/initializers/migrations/0605-actor-missing-keys.ts create mode 100644 server/lib/job-queue/handlers/actor-keys.ts diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 38b4066a4..9f5c04406 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts @@ -22,11 +22,13 @@ export class JobsComponent extends RestTable implements OnInit { jobType: JobTypeClient = 'all' jobTypes: JobTypeClient[] = [ 'all', + 'activitypub-follow', 'activitypub-http-broadcast', 'activitypub-http-fetcher', 'activitypub-http-unicast', 'activitypub-refresher', + 'actor-keys', 'email', 'video-file-import', 'video-import', diff --git a/client/src/app/+videos/video-list/trending/video-trending-header.component.ts b/client/src/app/+videos/video-list/trending/video-trending-header.component.ts index a4a1e358f..46e65f305 100644 --- a/client/src/app/+videos/video-list/trending/video-trending-header.component.ts +++ b/client/src/app/+videos/video-list/trending/video-trending-header.component.ts @@ -39,21 +39,21 @@ export class VideoTrendingHeaderComponent extends VideoListHeaderComponent imple label: $localize`:A variant of Trending videos based on the number of recent interactions, minus user history:Best`, iconName: 'award', value: 'best', - tooltip: $localize`Videos totalizing the most interactions for recent videos, minus user history`, + tooltip: $localize`Videos with the most interactions for recent videos, minus user history`, hidden: true }, { label: $localize`:A variant of Trending videos based on the number of recent interactions:Hot`, iconName: 'flame', value: 'hot', - tooltip: $localize`Videos totalizing the most interactions for recent videos`, + tooltip: $localize`Videos with the most interactions for recent videos`, hidden: true }, { label: $localize`:Main variant of Trending videos based on number of recent views:Views`, iconName: 'trending', value: 'most-viewed', - tooltip: $localize`Videos totalizing the most views during the last 24 hours` + tooltip: $localize`Videos with the most views during the last 24 hours` }, { label: $localize`:A variant of Trending videos based on the number of likes:Likes`, diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index fa0688a9e..3be1d55ae 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -221,7 +221,7 @@ async function createUser (req: express.Request, res: express.Response) { id: account.id } } - }).end() + }) } async function registerUser (req: express.Request, res: express.Response) { diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 14bd64730..03617dc8d 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -3,6 +3,7 @@ import { Hooks } from '@server/lib/plugins/hooks' import { getServerActor } from '@server/models/application/application' import { MChannelAccountDefault } from '@server/types/models' import { VideoChannelCreate, VideoChannelUpdate } from '../../../shared' +import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' @@ -11,7 +12,6 @@ import { getFormattedObjects } from '../../helpers/utils' import { CONFIG } from '../../initializers/config' import { MIMETYPES } from '../../initializers/constants' import { sequelizeTypescript } from '../../initializers/database' -import { setAsyncActorKeys } from '../../lib/activitypub/actor' import { sendUpdateActor } from '../../lib/activitypub/send' import { deleteLocalActorAvatarFile, updateLocalActorAvatarFile } from '../../lib/avatar' import { JobQueue } from '../../lib/job-queue' @@ -39,7 +39,6 @@ import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' import { VideoPlaylistModel } from '../../models/video/video-playlist' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' const auditLogger = auditLoggerFactory('channels') const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) @@ -168,8 +167,8 @@ async function addVideoChannel (req: express.Request, res: express.Response) { return createLocalVideoChannel(videoChannelInfo, account, t) }) - setAsyncActorKeys(videoChannelCreated.Actor) - .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.url, { err })) + const payload = { actorId: videoChannelCreated.actorId } + await JobQueue.Instance.createJobWithPromise({ type: 'actor-keys', payload }) auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())) logger.info('Video channel %s created.', videoChannelCreated.Actor.url) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index bbf55d7a4..74192d590 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -1,8 +1,8 @@ -import { randomInt } from '../../shared/core-utils/miscs/miscs' import { CronRepeatOptions, EveryRepeatOptions } from 'bull' import { randomBytes } from 'crypto' import { invert } from 'lodash' import { join } from 'path' +import { randomInt } from '../../shared/core-utils/miscs/miscs' import { AbuseState, JobType, @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 600 +const LAST_MIGRATION_VERSION = 605 // --------------------------------------------------------------------------- @@ -141,6 +141,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { 'video-transcoding': 1, 'video-import': 1, 'email': 5, + 'actor-keys': 3, 'videos-views': 1, 'activitypub-refresher': 1, 'video-redundancy': 1, @@ -153,6 +154,7 @@ const JOB_CONCURRENCY: { [id in JobType]?: number } = { 'activitypub-follow': 1, 'video-file-import': 1, 'email': 5, + 'actor-keys': 1, 'videos-views': 1, 'activitypub-refresher': 1, 'video-redundancy': 1, @@ -167,6 +169,7 @@ const JOB_TTL: { [id in JobType]: number } = { 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long 'video-import': 1000 * 3600 * 2, // 2 hours 'email': 60000 * 10, // 10 minutes + 'actor-keys': 60000 * 20, // 20 minutes 'videos-views': undefined, // Unlimited 'activitypub-refresher': 60000 * 10, // 10 minutes 'video-redundancy': 1000 * 3600 * 3, // 3 hours diff --git a/server/initializers/migrations/0605-actor-missing-keys.ts b/server/initializers/migrations/0605-actor-missing-keys.ts new file mode 100644 index 000000000..72d9b359d --- /dev/null +++ b/server/initializers/migrations/0605-actor-missing-keys.ts @@ -0,0 +1,34 @@ +import * as Sequelize from 'sequelize' +import { createPrivateKey, getPublicKey } from '../../helpers/core-utils' +import { PRIVATE_RSA_KEY_SIZE } from '../constants' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + + { + const query = 'SELECT * FROM "actor" WHERE "serverId" IS NULL AND "publicKey" IS NULL' + const options = { type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT } + const actors = await utils.sequelize.query(query, options) + + for (const actor of actors) { + const { key } = await createPrivateKey(PRIVATE_RSA_KEY_SIZE) + const { publicKey } = await getPublicKey(key) + + const queryUpdate = `UPDATE "actor" SET "publicKey" = '${publicKey}', "privateKey" = '${key}' WHERE id = ${actor.id}` + await utils.sequelize.query(queryUpdate) + } + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index dbb243d3a..a726f9e20 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -39,17 +39,13 @@ import { getServerActor } from '@server/models/application/application' import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' // Set account keys, this could be long so process after the account creation and do not block the client -function setAsyncActorKeys (actor: T) { - return createPrivateAndPublicKeys() - .then(({ publicKey, privateKey }) => { - actor.publicKey = publicKey - actor.privateKey = privateKey - return actor.save() - }) - .catch(err => { - logger.error('Cannot set public/private keys of actor %d.', actor.url, { err }) - return actor - }) +async function generateAndSaveActorKeys (actor: T) { + const { publicKey, privateKey } = await createPrivateAndPublicKeys() + + actor.publicKey = publicKey + actor.privateKey = privateKey + + return actor.save() } function getOrCreateActorAndServerAndModel ( @@ -346,7 +342,7 @@ async function refreshActorIfNeeded Promise } = { 'videos-views': processVideosViews, 'activitypub-refresher': refreshAPObject, 'video-live-ending': processVideoLiveEnding, + 'actor-keys': processActorKeys, 'video-redundancy': processVideoRedundancy } @@ -78,6 +82,7 @@ const jobTypes: JobType[] = [ 'videos-views', 'activitypub-refresher', 'video-redundancy', + 'actor-keys', 'video-live-ending' ] diff --git a/server/lib/user.ts b/server/lib/user.ts index 6b0fd9b88..e1892f22c 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -10,7 +10,7 @@ import { UserNotificationSettingModel } from '../models/account/user-notificatio import { ActorModel } from '../models/activitypub/actor' import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models' import { MUser, MUserDefault, MUserId } from '../types/models/user' -import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor' +import { buildActorInstance, generateAndSaveActorKeys } from './activitypub/actor' import { getLocalAccountActivityPubUrl } from './activitypub/url' import { Emailer } from './emailer' import { LiveManager } from './live-manager' @@ -55,8 +55,8 @@ async function createUserAccountAndChannelAndPlaylist (parameters: { }) const [ accountActorWithKeys, channelActorWithKeys ] = await Promise.all([ - setAsyncActorKeys(account.Actor), - setAsyncActorKeys(videoChannel.Actor) + generateAndSaveActorKeys(account.Actor), + generateAndSaveActorKeys(videoChannel.Actor) ]) account.Actor = accountActorWithKeys @@ -101,7 +101,7 @@ async function createApplicationActor (applicationId: number) { type: 'Application' }) - accountCreated.Actor = await setAsyncActorKeys(accountCreated.Actor) + accountCreated.Actor = await generateAndSaveActorKeys(accountCreated.Actor) return accountCreated } diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 07d7efe22..c693827b0 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -17,6 +17,7 @@ export type JobType = | 'activitypub-refresher' | 'video-redundancy' | 'video-live-ending' + | 'actor-keys' export interface Job { id: number @@ -131,3 +132,7 @@ export type VideoTranscodingPayload = export interface VideoLiveEndingPayload { videoId: number } + +export interface ActorKeysPayload { + actorId: number +}