From bafaba0bcda0c9fb553b9eebef3764994bb4ff60 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 7 Jun 2023 08:53:14 +0200 Subject: [PATCH] Support lazy download of remote video miniatures --- server/controllers/lazy-static.ts | 19 ++++- server/controllers/static.ts | 2 +- server/initializers/constants.ts | 2 + .../videos/shared/abstract-builder.ts | 25 +++--- .../lib/activitypub/videos/shared/creator.ts | 81 +++++++------------ server/lib/activitypub/videos/updater.ts | 11 ++- .../avatar-permanent-file-cache.ts | 4 +- server/lib/files-cache/index.ts | 1 + .../video-miniature-permanent-file-cache.ts | 28 +++++++ server/lib/thumbnail.ts | 70 ++++++++-------- server/lib/video-pre-import.ts | 5 +- server/models/video/thumbnail.ts | 8 +- server/models/video/video-playlist.ts | 6 +- server/tests/api/videos/video-imports.ts | 2 +- support/doc/api/openapi.yaml | 2 +- 15 files changed, 152 insertions(+), 114 deletions(-) create mode 100644 server/lib/files-cache/video-miniature-permanent-file-cache.ts diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts index 8e18b0642..dad30365c 100644 --- a/server/controllers/lazy-static.ts +++ b/server/controllers/lazy-static.ts @@ -6,6 +6,7 @@ import { FILES_CACHE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/ import { AvatarPermanentFileCache, VideoCaptionsSimpleFileCache, + VideoMiniaturePermanentFileCache, VideoPreviewsSimpleFileCache, VideoStoryboardsSimpleFileCache, VideoTorrentsSimpleFileCache @@ -39,6 +40,12 @@ lazyStaticRouter.use( handleStaticError ) +lazyStaticRouter.use( + LAZY_STATIC_PATHS.THUMBNAILS + ':filename', + asyncMiddleware(getThumbnail), + handleStaticError +) + lazyStaticRouter.use( LAZY_STATIC_PATHS.PREVIEWS + ':filename', asyncMiddleware(getPreview), @@ -72,7 +79,6 @@ export { } // --------------------------------------------------------------------------- - const avatarPermanentFileCache = new AvatarPermanentFileCache() function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) { @@ -81,6 +87,17 @@ function getActorImage (req: express.Request, res: express.Response, next: expre return avatarPermanentFileCache.lazyServe({ filename, res, next }) } +// --------------------------------------------------------------------------- +const videoMiniaturePermanentFileCache = new VideoMiniaturePermanentFileCache() + +function getThumbnail (req: express.Request, res: express.Response, next: express.NextFunction) { + const filename = req.params.filename + + return videoMiniaturePermanentFileCache.lazyServe({ filename, res, next }) +} + +// --------------------------------------------------------------------------- + async function getPreview (req: express.Request, res: express.Response) { const result = await VideoPreviewsSimpleFileCache.Instance.getFilePath(req.params.filename) if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 9baff94c0..bbd0dd011 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -72,7 +72,7 @@ staticRouter.use( handleStaticError ) -// Thumbnails path for express +// FIXME: deprecated in v6, to remove const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR staticRouter.use( STATIC_PATHS.THUMBNAILS, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 511aa91cc..ced18eef0 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -747,6 +747,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { // Express static paths (router) const STATIC_PATHS = { + // TODO: deprecated in v6, to remove THUMBNAILS: '/static/thumbnails/', WEBSEED: '/static/webseed/', @@ -765,6 +766,7 @@ const STATIC_DOWNLOAD_PATHS = { HLS_VIDEOS: '/download/streaming-playlists/hls/videos/' } const LAZY_STATIC_PATHS = { + THUMBNAILS: '/lazy-static/thumbnails/', BANNERS: '/lazy-static/banners/', AVATARS: '/lazy-static/avatars/', PREVIEWS: '/lazy-static/previews/', diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index e50bf29dc..4f74316d3 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -1,7 +1,7 @@ import { CreationAttributes, Transaction } from 'sequelize/types' import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' import { logger, LoggerTagsFn } from '@server/helpers/logger' -import { updateRemoteThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' +import { updateRemoteThumbnail } from '@server/lib/thumbnail' import { setVideoTags } from '@server/lib/video' import { StoryboardModel } from '@server/models/video/storyboard' import { VideoCaptionModel } from '@server/models/video/video-caption' @@ -11,7 +11,6 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin import { MStreamingPlaylistFiles, MStreamingPlaylistFilesVideo, - MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, @@ -42,16 +41,22 @@ export abstract class APVideoAbstractBuilder { return getOrCreateAPActor(channel.id, 'all') } - protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise { - return updateVideoMiniatureFromUrl({ - downloadUrl: getThumbnailFromIcons(this.videoObject).url, - video, - type: ThumbnailType.MINIATURE - }).catch(err => { - logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags() }) - + protected async setThumbnail (video: MVideoThumbnail, t?: Transaction) { + const miniatureIcon = getThumbnailFromIcons(this.videoObject) + if (!miniatureIcon) { + logger.warn('Cannot find thumbnail in video object', { object: this.videoObject }) return undefined + } + + const miniatureModel = updateRemoteThumbnail({ + fileUrl: miniatureIcon.url, + video, + type: ThumbnailType.MINIATURE, + size: miniatureIcon, + onDisk: false // Lazy download remote thumbnails }) + + await video.addAndSaveThumbnail(miniatureModel, t) } protected async setPreview (video: MVideoFullLight, t?: Transaction) { diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts index e6d7bc23c..3d646ef66 100644 --- a/server/lib/activitypub/videos/shared/creator.ts +++ b/server/lib/activitypub/videos/shared/creator.ts @@ -4,7 +4,7 @@ import { sequelizeTypescript } from '@server/initializers/database' import { Hooks } from '@server/lib/plugins/hooks' import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' import { VideoModel } from '@server/models/video/video' -import { MThumbnail, MVideoFullLight, MVideoThumbnail } from '@server/types/models' +import { MVideoFullLight, MVideoThumbnail } from '@server/types/models' import { VideoObject } from '@shared/models' import { APVideoAbstractBuilder } from './abstract-builder' import { getVideoAttributesFromObject } from './object-to-model-attributes' @@ -27,64 +27,37 @@ export class APVideoCreator extends APVideoAbstractBuilder { const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail - const promiseThumbnail = this.tryToGenerateThumbnail(video) - - let thumbnailModel: MThumbnail - if (waitThumbnail === true) { - thumbnailModel = await promiseThumbnail - } - const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { - try { - const videoCreated = await video.save({ transaction: t }) as MVideoFullLight - videoCreated.VideoChannel = channel + const videoCreated = await video.save({ transaction: t }) as MVideoFullLight + videoCreated.VideoChannel = channel - if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) + await this.setThumbnail(videoCreated, t) + await this.setPreview(videoCreated, t) + await this.setWebTorrentFiles(videoCreated, t) + await this.setStreamingPlaylists(videoCreated, t) + await this.setTags(videoCreated, t) + await this.setTrackers(videoCreated, t) + await this.insertOrReplaceCaptions(videoCreated, t) + await this.insertOrReplaceLive(videoCreated, t) + await this.insertOrReplaceStoryboard(videoCreated, t) - await this.setPreview(videoCreated, t) - await this.setWebTorrentFiles(videoCreated, t) - await this.setStreamingPlaylists(videoCreated, t) - await this.setTags(videoCreated, t) - await this.setTrackers(videoCreated, t) - await this.insertOrReplaceCaptions(videoCreated, t) - await this.insertOrReplaceLive(videoCreated, t) - await this.insertOrReplaceStoryboard(videoCreated, t) + // We added a video in this channel, set it as updated + await channel.setAsUpdated(t) - // We added a video in this channel, set it as updated - await channel.setAsUpdated(t) - - const autoBlacklisted = await autoBlacklistVideoIfNeeded({ - video: videoCreated, - user: undefined, - isRemote: true, - isNew: true, - transaction: t - }) - - logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) - - Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) - - return { autoBlacklisted, videoCreated } - } catch (err) { - // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released - if (thumbnailModel) await thumbnailModel.removeThumbnail() - - throw err - } - }) - - if (waitThumbnail === false) { - // Error is already caught above - // eslint-disable-next-line @typescript-eslint/no-floating-promises - promiseThumbnail.then(thumbnailModel => { - if (!thumbnailModel) return - - thumbnailModel = videoCreated.id - - return thumbnailModel.save() + const autoBlacklisted = await autoBlacklistVideoIfNeeded({ + video: videoCreated, + user: undefined, + isRemote: true, + isNew: true, + transaction: t }) - } + + logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) + + Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) + + return { autoBlacklisted, videoCreated } + }) return { autoBlacklisted, videoCreated } } diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 3a0886523..c98bce662 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts @@ -41,7 +41,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { try { const channelActor = await this.getOrCreateVideoChannelFromVideoObject() - const thumbnailModel = await this.tryToGenerateThumbnail(this.video) + const thumbnailModel = await this.setThumbnail(this.video) this.checkChannelUpdateOrThrow(channelActor) @@ -58,8 +58,13 @@ export class APVideoUpdater extends APVideoAbstractBuilder { runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), - this.setOrDeleteLive(videoUpdated), - this.setPreview(videoUpdated) + runInReadCommittedTransaction(t => { + return Promise.all([ + this.setPreview(videoUpdated, t), + this.setThumbnail(videoUpdated, t) + ]) + }), + this.setOrDeleteLive(videoUpdated) ]) await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts index 89228c5a5..1d77c5bc1 100644 --- a/server/lib/files-cache/avatar-permanent-file-cache.ts +++ b/server/lib/files-cache/avatar-permanent-file-cache.ts @@ -1,10 +1,10 @@ +import { CONFIG } from '@server/initializers/config' import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' import { ActorImageModel } from '@server/models/actor/actor-image' import { MActorImage } from '@server/types/models' import { AbstractPermanentFileCache } from './shared' -import { CONFIG } from '@server/initializers/config' -export class AvatarPermanentFileCache extends AbstractPermanentFileCache { +export class AvatarPermanentFileCache extends AbstractPermanentFileCache { constructor () { super(CONFIG.STORAGE.ACTOR_IMAGES) diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts index cc11d5385..5630a9b80 100644 --- a/server/lib/files-cache/index.ts +++ b/server/lib/files-cache/index.ts @@ -1,4 +1,5 @@ export * from './avatar-permanent-file-cache' +export * from './video-miniature-permanent-file-cache' export * from './video-captions-simple-file-cache' export * from './video-previews-simple-file-cache' export * from './video-storyboards-simple-file-cache' diff --git a/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/lib/files-cache/video-miniature-permanent-file-cache.ts new file mode 100644 index 000000000..35d9466f7 --- /dev/null +++ b/server/lib/files-cache/video-miniature-permanent-file-cache.ts @@ -0,0 +1,28 @@ +import { CONFIG } from '@server/initializers/config' +import { THUMBNAILS_SIZE } from '@server/initializers/constants' +import { ThumbnailModel } from '@server/models/video/thumbnail' +import { MThumbnail } from '@server/types/models' +import { ThumbnailType } from '@shared/models' +import { AbstractPermanentFileCache } from './shared' + +export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache { + + constructor () { + super(CONFIG.STORAGE.THUMBNAILS_DIR) + } + + protected loadModel (filename: string) { + return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE) + } + + protected getImageSize (image: MThumbnail): { width: number, height: number } { + if (image.width && image.height) { + return { + height: image.height, + width: image.width + } + } + + return THUMBNAILS_SIZE + } +} diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index e792567ff..90f5dc2c8 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts @@ -60,38 +60,6 @@ function updatePlaylistMiniatureFromUrl (options: { return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) } -function updateVideoMiniatureFromUrl (options: { - downloadUrl: string - video: MVideoThumbnail - type: ThumbnailType - size?: ImageSize -}) { - const { downloadUrl, video, type, size } = options - const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - // Only save the file URL if it is a remote video - const fileUrl = video.isOwned() - ? null - : downloadUrl - - const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) - - // Do not change the thumbnail filename if the file did not change - const filename = thumbnailUrlChanged - ? updatedFilename - : existingThumbnail.filename - - const thumbnailCreator = () => { - if (thumbnailUrlChanged) { - return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) - } - - return Promise.resolve() - } - - return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) -} - function updateLocalVideoMiniatureFromExisting (options: { inputPath: string video: MVideoThumbnail @@ -157,6 +125,40 @@ function generateLocalVideoMiniature (options: { }) } +// --------------------------------------------------------------------------- + +function updateVideoMiniatureFromUrl (options: { + downloadUrl: string + video: MVideoThumbnail + type: ThumbnailType + size?: ImageSize +}) { + const { downloadUrl, video, type, size } = options + const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) + + // Only save the file URL if it is a remote video + const fileUrl = video.isOwned() + ? null + : downloadUrl + + const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) + + // Do not change the thumbnail filename if the file did not change + const filename = thumbnailUrlChanged + ? updatedFilename + : existingThumbnail.filename + + const thumbnailCreator = () => { + if (thumbnailUrlChanged) { + return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) + } + + return Promise.resolve() + } + + return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) +} + function updateRemoteThumbnail (options: { fileUrl: string video: MVideoThumbnail @@ -167,12 +169,10 @@ function updateRemoteThumbnail (options: { const { fileUrl, video, type, size, onDisk } = options const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, fileUrl, video) - const thumbnail = existingThumbnail || new ThumbnailModel() // Do not change the thumbnail filename if the file did not change - if (thumbnailUrlChanged) { + if (hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)) { thumbnail.filename = generatedFilename } diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts index ef9c38731..1471d4091 100644 --- a/server/lib/video-pre-import.ts +++ b/server/lib/video-pre-import.ts @@ -262,13 +262,16 @@ async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { type, automaticallyGenerated: false }) - } else if (downloadUrl) { + } + + if (downloadUrl) { try { return await updateVideoMiniatureFromUrl({ downloadUrl, video, type }) } catch (err) { logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) } } + return null } diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index 2a1f6a7b4..1722acdb4 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -21,7 +21,7 @@ import { AttributesOnly } from '@shared/typescript-utils' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { logger } from '../../helpers/logger' import { CONFIG } from '../../initializers/config' -import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants' +import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants' import { VideoModel } from './video' import { VideoPlaylistModel } from './video-playlist' @@ -110,7 +110,7 @@ export class ThumbnailModel extends Model [ThumbnailType.MINIATURE]: { label: 'miniature', directory: CONFIG.STORAGE.THUMBNAILS_DIR, - staticPath: STATIC_PATHS.THUMBNAILS + staticPath: LAZY_STATIC_PATHS.THUMBNAILS }, [ThumbnailType.PREVIEW]: { label: 'preview', @@ -201,4 +201,8 @@ export class ThumbnailModel extends Model this.previousThumbnailFilename = undefined } + + isOwned () { + return !this.fileUrl + } } diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index faf4bea78..15999d409 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -32,7 +32,7 @@ import { import { ACTIVITY_PUB, CONSTRAINTS_FIELDS, - STATIC_PATHS, + LAZY_STATIC_PATHS, THUMBNAILS_SIZE, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES, @@ -592,13 +592,13 @@ export class VideoPlaylistModel extends Model