From 8319d6ae72d4da6de51bd3d4b5c68040fc8dc3b4 Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Tue, 10 Mar 2020 14:39:40 +0100 Subject: [PATCH] Add video file metadata to download modal, via ffprobe (#2411) * Add video file metadata via ffprobe * Federate video file metadata * Add tests for file metadata generation * Complete tests for videoFile metadata federation * Lint migration and video-file for metadata * Objectify metadata from getter in ffmpeg-utils * Add metadataUrl to all videoFiles * Simplify metadata API middleware * Load playlist in videoFile when requesting metadata --- .../modals/video-download.component.html | 38 +++++- .../modals/video-download.component.scss | 35 ++++++ .../video/modals/video-download.component.ts | 109 +++++++++++++++++- client/src/app/shared/video/video.service.ts | 9 ++ client/src/sass/bootstrap.scss | 7 +- server/controllers/api/videos/index.ts | 18 ++- server/helpers/ffmpeg-utils.ts | 35 +++--- server/helpers/middlewares/videos.ts | 14 +++ server/initializers/constants.ts | 2 +- .../migrations/0485-video-file-metadata.ts | 30 +++++ server/lib/activitypub/videos.ts | 16 ++- server/lib/video-transcoding.ts | 8 ++ .../middlewares/validators/videos/videos.ts | 22 +++- server/models/redundancy/video-redundancy.ts | 6 +- server/models/utils.ts | 18 +++ server/models/video/video-file.ts | 98 ++++++++++++++-- server/models/video/video-format-utils.ts | 13 ++- server/models/video/video.ts | 13 ++- server/tests/api/videos/video-transcoder.ts | 73 +++++++++++- shared/extra-utils/videos/videos.ts | 9 ++ .../activitypub/objects/common-objects.ts | 11 ++ shared/models/videos/video-file-metadata.ts | 18 +++ shared/models/videos/video-file.model.ts | 3 + 23 files changed, 553 insertions(+), 52 deletions(-) create mode 100644 server/initializers/migrations/0485-video-file-metadata.ts create mode 100644 shared/models/videos/video-file-metadata.ts diff --git a/client/src/app/shared/video/modals/video-download.component.html b/client/src/app/shared/video/modals/video-download.component.html index 976da03f3..391fe245e 100644 --- a/client/src/app/shared/video/modals/video-download.component.html +++ b/client/src/app/shared/video/modals/video-download.component.html @@ -20,7 +20,7 @@
- @@ -38,6 +38,42 @@
+ + + Format + +
+
+ {{ item.value.label }} + {{ item.value.value }} +
+
+ + + + Video stream + +
+
+ {{ item.value.label }} + {{ item.value.value }} +
+
+ + + + Audio stream + +
+
+ {{ item.value.label }} + {{ item.value.value }} +
+
+ + + +
diff --git a/client/src/app/shared/video/modals/video-download.component.scss b/client/src/app/shared/video/modals/video-download.component.scss index 09dd91aa9..f28bc34ed 100644 --- a/client/src/app/shared/video/modals/video-download.component.scss +++ b/client/src/app/shared/video/modals/video-download.component.scss @@ -27,3 +27,38 @@ margin-right: 30px; } } + +.file-metadata { + padding: 1rem; +} + +.file-metadata .metadata-attribute { + font-size: 13px; + display: block; + margin-bottom: 12px; + + .metadata-attribute-label { + min-width: 142px; + padding-right: 5px; + display: inline-block; + color: $grey-foreground-color; + font-weight: $font-bold; + } + + a.metadata-attribute-value { + @include disable-default-a-behaviour; + color: var(--mainForegroundColor); + + &:hover { + opacity: 0.9; + } + } + + &.metadata-attribute-tags { + .metadata-attribute-value:not(:nth-child(2)) { + &::before { + content: ', ' + } + } + } +} diff --git a/client/src/app/shared/video/modals/video-download.component.ts b/client/src/app/shared/video/modals/video-download.component.ts index 6909c4279..d77187821 100644 --- a/client/src/app/shared/video/modals/video-download.component.ts +++ b/client/src/app/shared/video/modals/video-download.component.ts @@ -3,9 +3,15 @@ import { VideoDetails } from '../../../shared/video/video-details.model' import { NgbActiveModal, NgbModal } from '@ng-bootstrap/ng-bootstrap' import { I18n } from '@ngx-translate/i18n-polyfill' import { AuthService, Notifier } from '@app/core' -import { VideoPrivacy, VideoCaption } from '@shared/models' +import { VideoPrivacy, VideoCaption, VideoFile } from '@shared/models' +import { FfprobeFormat, FfprobeStream } from 'fluent-ffmpeg' +import { mapValues, pick } from 'lodash-es' +import { NumberFormatterPipe } from '@app/shared/angular/number-formatter.pipe' +import { BytesPipe } from 'ngx-pipes' +import { VideoService } from '../video.service' type DownloadType = 'video' | 'subtitles' +type FileMetadata = { [key: string]: { label: string, value: string }} @Component({ selector: 'my-video-download', @@ -20,17 +26,28 @@ export class VideoDownloadComponent { subtitleLanguageId: string video: VideoDetails + videoFile: VideoFile + videoFileMetadataFormat: FileMetadata + videoFileMetadataVideoStream: FileMetadata | undefined + videoFileMetadataAudioStream: FileMetadata | undefined videoCaptions: VideoCaption[] activeModal: NgbActiveModal type: DownloadType = 'video' + private bytesPipe: BytesPipe + private numbersPipe: NumberFormatterPipe + constructor ( private notifier: Notifier, private modalService: NgbModal, + private videoService: VideoService, private auth: AuthService, private i18n: I18n - ) { } + ) { + this.bytesPipe = new BytesPipe() + this.numbersPipe = new NumberFormatterPipe() + } get typeText () { return this.type === 'video' @@ -51,6 +68,7 @@ export class VideoDownloadComponent { this.activeModal = this.modalService.open(this.modal, { centered: true }) this.resolutionId = this.getVideoFiles()[0].resolution.id + this.onResolutionIdChange() if (this.videoCaptions) this.subtitleLanguageId = this.videoCaptions[0].language.id } @@ -67,10 +85,27 @@ export class VideoDownloadComponent { getLink () { return this.type === 'subtitles' && this.videoCaptions ? this.getSubtitlesLink() - : this.getVideoLink() + : this.getVideoFileLink() } - getVideoLink () { + async onResolutionIdChange () { + this.videoFile = this.getVideoFile() + if (this.videoFile.metadata || !this.videoFile.metadataUrl) return + + await this.hydrateMetadataFromMetadataUrl(this.videoFile) + + this.videoFileMetadataFormat = this.videoFile + ? this.getMetadataFormat(this.videoFile.metadata.format) + : undefined + this.videoFileMetadataVideoStream = this.videoFile + ? this.getMetadataStream(this.videoFile.metadata.streams, 'video') + : undefined + this.videoFileMetadataAudioStream = this.videoFile + ? this.getMetadataStream(this.videoFile.metadata.streams, 'audio') + : undefined + } + + getVideoFile () { // HTML select send us a string, so convert it to a number this.resolutionId = parseInt(this.resolutionId.toString(), 10) @@ -79,6 +114,12 @@ export class VideoDownloadComponent { console.error('Could not find file with resolution %d.', this.resolutionId) return } + return file + } + + getVideoFileLink () { + const file = this.videoFile + if (!file) return const suffix = this.video.privacy.id === VideoPrivacy.PRIVATE || this.video.privacy.id === VideoPrivacy.INTERNAL ? '?access_token=' + this.auth.getAccessToken() @@ -104,4 +145,64 @@ export class VideoDownloadComponent { switchToType (type: DownloadType) { this.type = type } + + getMetadataFormat (format: FfprobeFormat) { + const keyToTranslateFunction = { + 'encoder': (value: string) => ({ label: this.i18n('Encoder'), value }), + 'format_long_name': (value: string) => ({ label: this.i18n('Format name'), value }), + 'size': (value: number) => ({ label: this.i18n('Size'), value: this.bytesPipe.transform(value, 2) }), + 'bit_rate': (value: number) => ({ + label: this.i18n('Bitrate'), + value: `${this.numbersPipe.transform(value)}bps` + }) + } + + // flattening format + const sanitizedFormat = Object.assign(format, format.tags) + delete sanitizedFormat.tags + + return mapValues( + pick(sanitizedFormat, Object.keys(keyToTranslateFunction)), + (val, key) => keyToTranslateFunction[key](val) + ) + } + + getMetadataStream (streams: FfprobeStream[], type: 'video' | 'audio') { + const stream = streams.find(s => s.codec_type === type) + if (!stream) return undefined + + let keyToTranslateFunction = { + 'codec_long_name': (value: string) => ({ label: this.i18n('Codec'), value }), + 'profile': (value: string) => ({ label: this.i18n('Profile'), value }), + 'bit_rate': (value: number) => ({ + label: this.i18n('Bitrate'), + value: `${this.numbersPipe.transform(value)}bps` + }) + } + + if (type === 'video') { + keyToTranslateFunction = Object.assign(keyToTranslateFunction, { + 'width': (value: number) => ({ label: this.i18n('Resolution'), value: `${value}x${stream.height}` }), + 'display_aspect_ratio': (value: string) => ({ label: this.i18n('Aspect ratio'), value }), + 'avg_frame_rate': (value: string) => ({ label: this.i18n('Average frame rate'), value }), + 'pix_fmt': (value: string) => ({ label: this.i18n('Pixel format'), value }) + }) + } else { + keyToTranslateFunction = Object.assign(keyToTranslateFunction, { + 'sample_rate': (value: number) => ({ label: this.i18n('Sample rate'), value }), + 'channel_layout': (value: number) => ({ label: this.i18n('Channel Layout'), value }) + }) + } + + return mapValues( + pick(stream, Object.keys(keyToTranslateFunction)), + (val, key) => keyToTranslateFunction[key](val) + ) + } + + private hydrateMetadataFromMetadataUrl (file: VideoFile) { + const observable = this.videoService.getVideoFileMetadata(file.metadataUrl) + observable.subscribe(res => file.metadata = res) + return observable.toPromise() + } } diff --git a/client/src/app/shared/video/video.service.ts b/client/src/app/shared/video/video.service.ts index a51b9cab9..3aaf14990 100644 --- a/client/src/app/shared/video/video.service.ts +++ b/client/src/app/shared/video/video.service.ts @@ -32,6 +32,7 @@ import { UserSubscriptionService } from '@app/shared/user-subscription/user-subs import { VideoChannel } from '@app/shared/video-channel/video-channel.model' import { I18n } from '@ngx-translate/i18n-polyfill' import { NSFWPolicyType } from '@shared/models/videos/nsfw-policy.type' +import { FfprobeData } from 'fluent-ffmpeg' export interface VideosProvider { getVideos (parameters: { @@ -291,6 +292,14 @@ export class VideoService implements VideosProvider { return this.buildBaseFeedUrls(params) } + getVideoFileMetadata (metadataUrl: string) { + return this.authHttp + .get(metadataUrl) + .pipe( + catchError(err => this.restExtractor.handleError(err)) + ) + } + removeVideo (id: number) { return this.authHttp .delete(VideoService.BASE_VIDEO_URL + id) diff --git a/client/src/sass/bootstrap.scss b/client/src/sass/bootstrap.scss index e167fd02b..f718791eb 100644 --- a/client/src/sass/bootstrap.scss +++ b/client/src/sass/bootstrap.scss @@ -109,6 +109,11 @@ $icon-font-path: '~@neos21/bootstrap3-glyphicons/assets/fonts/'; margin: 0; padding: 0; opacity: .5; + + &[iconName="cross"] { + @include icon(16px); + top: -3px; + } } } @@ -153,7 +158,7 @@ $icon-font-path: '~@neos21/bootstrap3-glyphicons/assets/fonts/'; } } -ngb-tabset.bootstrap { +ngb-tabset { .nav-link { &, & a { diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index eb46ea01f..9b19c394d 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -1,7 +1,7 @@ import * as express from 'express' import { extname } from 'path' import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared' -import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' +import { getVideoFileFPS, getVideoFileResolution, getMetadataFromFile } from '../../../helpers/ffmpeg-utils' import { logger } from '../../../helpers/logger' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { getFormattedObjects, getServerActor } from '../../../helpers/utils' @@ -37,7 +37,8 @@ import { videosGetValidator, videosRemoveValidator, videosSortValidator, - videosUpdateValidator + videosUpdateValidator, + videoFileMetadataGetValidator } from '../../../middlewares' import { TagModel } from '../../../models/video/tag' import { VideoModel } from '../../../models/video/video' @@ -66,6 +67,7 @@ import { Hooks } from '../../../lib/plugins/hooks' import { MVideoDetails, MVideoFullLight } from '@server/typings/models' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getVideoFilePath } from '@server/lib/video-paths' +import toInt from 'validator/lib/toInt' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -128,6 +130,10 @@ videosRouter.get('/:id/description', asyncMiddleware(videosGetValidator), asyncMiddleware(getVideoDescription) ) +videosRouter.get('/:id/metadata/:videoFileId', + asyncMiddleware(videoFileMetadataGetValidator), + asyncMiddleware(getVideoFileMetadata) +) videosRouter.get('/:id', optionalAuthenticate, asyncMiddleware(videosCustomGetValidator('only-video-with-rights')), @@ -206,7 +212,8 @@ async function addVideo (req: express.Request, res: express.Response) { const videoFile = new VideoFileModel({ extname: extname(videoPhysicalFile.filename), size: videoPhysicalFile.size, - videoStreamingPlaylistId: null + videoStreamingPlaylistId: null, + metadata: await getMetadataFromFile(videoPhysicalFile.path) }) if (videoFile.isAudio()) { @@ -493,6 +500,11 @@ async function getVideoDescription (req: express.Request, res: express.Response) return res.json({ description }) } +async function getVideoFileMetadata (req: express.Request, res: express.Response) { + const videoFile = await VideoFileModel.loadWithMetadata(toInt(req.params.videoFileId)) + return res.json(videoFile.metadata) +} + async function listVideos (req: express.Request, res: express.Response) { const countVideos = getCountVideos(req) diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 084516e55..5ee295635 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -7,6 +7,7 @@ import { logger } from './logger' import { checkFFmpegEncoders } from '../initializers/checker-before-init' import { readFile, remove, writeFile } from 'fs-extra' import { CONFIG } from '../initializers/config' +import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata' /** * A toolbox to play with audio @@ -169,24 +170,26 @@ async function getVideoFileFPS (path: string) { return 0 } -async function getVideoFileBitrate (path: string) { - return new Promise((res, rej) => { +async function getMetadataFromFile (path: string, cb = metadata => metadata) { + return new Promise((res, rej) => { ffmpeg.ffprobe(path, (err, metadata) => { if (err) return rej(err) - return res(metadata.format.bit_rate) + return res(cb(new VideoFileMetadata(metadata))) }) }) } -function getDurationFromVideoFile (path: string) { - return new Promise((res, rej) => { - ffmpeg.ffprobe(path, (err, metadata) => { - if (err) return rej(err) +async function getVideoFileBitrate (path: string) { + return getMetadataFromFile(path, metadata => metadata.format.bit_rate) +} - return res(Math.floor(metadata.format.duration)) - }) - }) +function getDurationFromVideoFile (path: string) { + return getMetadataFromFile(path, metadata => Math.floor(metadata.format.duration)) +} + +function getVideoStreamFromFile (path: string) { + return getMetadataFromFile(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null) } async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) { @@ -341,6 +344,7 @@ export { getAudioStreamCodec, getVideoStreamSize, getVideoFileResolution, + getMetadataFromFile, getDurationFromVideoFile, generateImageFromVideoFile, TranscodeOptions, @@ -450,17 +454,6 @@ async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) { await writeFile(options.outputPath, newContent) } -function getVideoStreamFromFile (path: string) { - return new Promise((res, rej) => { - ffmpeg.ffprobe(path, (err, metadata) => { - if (err) return rej(err) - - const videoStream = metadata.streams.find(s => s.codec_type === 'video') - return res(videoStream || null) - }) - }) -} - /** * A slightly customised version of the 'veryfast' x264 preset * diff --git a/server/helpers/middlewares/videos.ts b/server/helpers/middlewares/videos.ts index 409f78650..a0bbcdb21 100644 --- a/server/helpers/middlewares/videos.ts +++ b/server/helpers/middlewares/videos.ts @@ -12,6 +12,7 @@ import { MVideoThumbnail, MVideoWithRights } from '@server/typings/models' +import { VideoFileModel } from '@server/models/video/video-file' async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') { const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined @@ -51,6 +52,18 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi return true } +async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) { + if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) { + res.status(404) + .json({ error: 'VideoFile matching Video not found' }) + .end() + + return false + } + + return true +} + async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) { if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) { const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId) @@ -107,5 +120,6 @@ function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: export { doesVideoChannelOfAccountExist, doesVideoExist, + doesVideoFileOfVideoExist, checkUserCanManageVideo } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 3da06402c..8b040aa2c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 480 +const LAST_MIGRATION_VERSION = 485 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0485-video-file-metadata.ts b/server/initializers/migrations/0485-video-file-metadata.ts new file mode 100644 index 000000000..5d95be024 --- /dev/null +++ b/server/initializers/migrations/0485-video-file-metadata.ts @@ -0,0 +1,30 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + + const metadata = { + type: Sequelize.JSONB, + allowNull: true + } + await utils.queryInterface.addColumn('videoFile', 'metadata', metadata) + + const metadataUrl = { + type: Sequelize.STRING, + allowNull: true + } + await utils.queryInterface.addColumn('videoFile', 'metadataUrl', metadataUrl) + +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index bce1666be..30de4714c 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -10,7 +10,8 @@ import { ActivityTagObject, ActivityUrlObject, ActivityVideoUrlObject, - VideoState + VideoState, + ActivityVideoFileMetadataObject } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' @@ -526,6 +527,10 @@ function isAPHashTagObject (url: any): url is ActivityHashTagObject { return url && url.type === 'Hashtag' } +function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject { + return url && url.type === 'Link' && url.mediaType === 'application/json' && url.hasAttribute('rel') && url.rel.includes('metadata') +} + async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) { logger.debug('Adding remote video %s.', videoObject.id) @@ -694,6 +699,14 @@ function videoFileActivityUrlToDBAttributes ( throw new Error('Cannot parse magnet URI ' + magnet.href) } + // Fetch associated metadata url, if any + const metadata = urls.filter(isAPVideoFileMetadataObject) + .find(u => + u.height === fileUrl.height && + u.fps === fileUrl.fps && + u.rel.includes(fileUrl.mediaType) + ) + const mediaType = fileUrl.mediaType const attribute = { extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType], @@ -701,6 +714,7 @@ function videoFileActivityUrlToDBAttributes ( resolution: fileUrl.height, size: fileUrl.size, fps: fileUrl.fps || -1, + metadataUrl: metadata?.href, // This is a video file owned by a video or by a streaming playlist videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id, diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 0d5b3ae39..444b0d954 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -2,6 +2,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSER import { basename, extname as extnameUtil, join } from 'path' import { canDoQuickTranscode, + getMetadataFromFile, getDurationFromVideoFile, getVideoFileFPS, transcode, @@ -19,6 +20,7 @@ import { CONFIG } from '../initializers/config' import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoWithAllFiles, MVideoWithFile } from '@server/typings/models' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { generateVideoStreamingPlaylistName, getVideoFilename, getVideoFilePath } from './video-paths' +import { extractVideo } from './videos' /** * Optimize the original video file and replace it. The resolution is not changed. @@ -202,6 +204,7 @@ async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoReso newVideoFile.size = stats.size newVideoFile.fps = await getVideoFileFPS(videoFilePath) + newVideoFile.metadata = await getMetadataFromFile(videoFilePath) await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) @@ -230,11 +233,16 @@ export { async function onVideoFileTranscoding (video: MVideoWithFile, videoFile: MVideoFile, transcodingPath: string, outputPath: string) { const stats = await stat(transcodingPath) const fps = await getVideoFileFPS(transcodingPath) + const metadata = await getMetadataFromFile(transcodingPath) await move(transcodingPath, outputPath) + const extractedVideo = extractVideo(video) + videoFile.size = stats.size videoFile.fps = fps + videoFile.metadata = metadata + videoFile.metadataUrl = extractedVideo.getVideoFileMetadataUrl(videoFile, extractedVideo.getBaseUrls().baseUrlHttp) await createTorrentAndSetInfoHash(video, videoFile) diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index a027c4840..96e0d6600 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -42,7 +42,12 @@ import { getServerActor } from '../../../helpers/utils' import { CONFIG } from '../../../initializers/config' import { isLocalVideoAccepted } from '../../../lib/moderation' import { Hooks } from '../../../lib/plugins/hooks' -import { checkUserCanManageVideo, doesVideoChannelOfAccountExist, doesVideoExist } from '../../../helpers/middlewares' +import { + checkUserCanManageVideo, + doesVideoChannelOfAccountExist, + doesVideoExist, + doesVideoFileOfVideoExist +} from '../../../helpers/middlewares' import { MVideoFullLight } from '@server/typings/models' import { getVideoWithAttributes } from '../../../helpers/video' @@ -198,6 +203,20 @@ const videosCustomGetValidator = ( const videosGetValidator = videosCustomGetValidator('all') const videosDownloadValidator = videosCustomGetValidator('all', true) +const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ + param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), + param('videoFileId').custom(isIdValid).not().isEmpty().withMessage('Should have a valid videoFileId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoFileMetadataGet parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + if (!await doesVideoFileOfVideoExist(+req.params.videoFileId, req.params.id, res)) return + + return next() + } +]) + const videosRemoveValidator = [ param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'), @@ -411,6 +430,7 @@ export { videosAddValidator, videosUpdateValidator, videosGetValidator, + videoFileMetadataGetValidator, videosDownloadValidator, checkVideoFollowConstraints, videosCustomGetValidator, diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 1b63d3818..857b9eca6 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -528,7 +528,7 @@ export class VideoRedundancyModel extends Model { include: [ { required: false, - model: VideoFileModel.unscoped(), + model: VideoFileModel, include: [ { model: VideoRedundancyModel.unscoped(), @@ -547,7 +547,7 @@ export class VideoRedundancyModel extends Model { where: redundancyWhere }, { - model: VideoFileModel.unscoped(), + model: VideoFileModel, required: false } ] @@ -699,7 +699,7 @@ export class VideoRedundancyModel extends Model { return { attributes: [], - model: VideoFileModel.unscoped(), + model: VideoFileModel, required: true, where: { id: { diff --git a/server/models/utils.ts b/server/models/utils.ts index 674ddcbe4..06ff05864 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -3,6 +3,23 @@ import validator from 'validator' import { Col } from 'sequelize/types/lib/utils' import { literal, OrderItem } from 'sequelize' +type Primitive = string | Function | number | boolean | Symbol | undefined | null +type DeepOmitHelper = { + [P in K]: // extra level of indirection needed to trigger homomorhic behavior + T[P] extends infer TP // distribute over unions + ? TP extends Primitive + ? TP // leave primitives and functions alone + : TP extends any[] + ? DeepOmitArray // Array special handling + : DeepOmit + : never +} +type DeepOmit = T extends Primitive ? T : DeepOmitHelper> + +type DeepOmitArray = { + [P in keyof T]: DeepOmit +} + type SortType = { sortModel: string, sortValue: string } // Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ] @@ -193,6 +210,7 @@ function buildDirectionAndField (value: string) { // --------------------------------------------------------------------------- export { + DeepOmit, buildBlockedAccountSQL, buildLocalActorIdsIn, SortType, diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index e08999385..029468004 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -10,7 +10,9 @@ import { Is, Model, Table, - UpdatedAt + UpdatedAt, + Scopes, + DefaultScope } from 'sequelize-typescript' import { isVideoFileExtnameValid, @@ -29,6 +31,60 @@ import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '. import { MStreamingPlaylistVideo, MVideo } from '@server/typings/models' import * as memoizee from 'memoizee' +export enum ScopeNames { + WITH_VIDEO = 'WITH_VIDEO', + WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST', + WITH_METADATA = 'WITH_METADATA' +} + +const METADATA_FIELDS = [ 'metadata', 'metadataUrl' ] + +@DefaultScope(() => ({ + attributes: { + exclude: [ METADATA_FIELDS[0] ] + } +})) +@Scopes(() => ({ + [ScopeNames.WITH_VIDEO]: { + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + }, + [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (videoIdOrUUID: string | number) => { + const where = (typeof videoIdOrUUID === 'number') + ? { id: videoIdOrUUID } + : { uuid: videoIdOrUUID } + + return { + include: [ + { + model: VideoModel.unscoped(), + required: false, + where + }, + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where + } + ] + } + ] + } + }, + [ScopeNames.WITH_METADATA]: { + attributes: { + include: METADATA_FIELDS + } + } +})) @Table({ tableName: 'videoFile', indexes: [ @@ -106,6 +162,14 @@ export class VideoFileModel extends Model { @Column fps: number + @AllowNull(true) + @Column(DataType.JSONB) + metadata: any + + @AllowNull(true) + @Column + metadataUrl: string + @ForeignKey(() => VideoModel) @Column videoId: number @@ -157,17 +221,29 @@ export class VideoFileModel extends Model { .then(results => results.length === 1) } - static loadWithVideo (id: number) { - const options = { - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] - } + static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { + const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID) + return (videoFile?.Video.id === videoIdOrUUID) || + (videoFile?.Video.uuid === videoIdOrUUID) || + (videoFile?.VideoStreamingPlaylist?.Video?.id === videoIdOrUUID) || + (videoFile?.VideoStreamingPlaylist?.Video?.uuid === videoIdOrUUID) + } - return VideoFileModel.findByPk(id, options) + static loadWithMetadata (id: number) { + return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) + } + + static loadWithVideo (id: number) { + return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id) + } + + static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { + return VideoFileModel.scope({ + method: [ + ScopeNames.WITH_VIDEO_OR_PLAYLIST, + videoIdOrUUID + ] + }).findByPk(id) } static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 1fa66fd63..21f0e0a68 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -23,6 +23,7 @@ import { import { MVideoFileRedundanciesOpt } from '../../typings/models/video/video-file' import { VideoFile } from '@shared/models/videos/video-file.model' import { generateMagnetUri } from '@server/helpers/webtorrent' +import { extractVideo } from '@server/lib/videos' export type VideoFormattingJSONOptions = { completeDescription?: boolean @@ -193,7 +194,8 @@ function videoFilesModelToFormattedJSON ( torrentUrl: model.getTorrentUrl(videoFile, baseUrlHttp), torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp), fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp), - fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp) + fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp), + metadataUrl: videoFile.metadataUrl // only send the metadataUrl and not the metadata over the wire } as VideoFile }) .sort((a, b) => { @@ -220,6 +222,15 @@ function addVideoFilesInAPAcc ( fps: file.fps }) + acc.push({ + type: 'Link', + rel: [ 'metadata', MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] ], + mediaType: 'application/json' as 'application/json', + href: extractVideo(model).getVideoFileMetadataUrl(file, baseUrlHttp), + height: file.resolution, + fps: file.fps + }) + acc.push({ type: 'Link', mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 7f94e834a..5e4b7d44c 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -216,7 +216,7 @@ export type AvailableForListIDsOptions = { if (options.withFiles === true) { query.include.push({ - model: VideoFileModel.unscoped(), + model: VideoFileModel, required: true }) } @@ -337,7 +337,7 @@ export type AvailableForListIDsOptions = { return { include: [ { - model: VideoFileModel.unscoped(), + model: VideoFileModel, separate: true, // We may have multiple files, having multiple redundancies so let's separate this join required: false, include: subInclude @@ -348,7 +348,7 @@ export type AvailableForListIDsOptions = { [ScopeNames.WITH_STREAMING_PLAYLISTS]: (withRedundancies = false) => { const subInclude: IncludeOptions[] = [ { - model: VideoFileModel.unscoped(), + model: VideoFileModel, required: false } ] @@ -1847,6 +1847,13 @@ export class VideoModel extends Model { return baseUrlHttp + STATIC_PATHS.WEBSEED + getVideoFilename(this, videoFile) } + getVideoFileMetadataUrl (videoFile: MVideoFile, baseUrlHttp: string) { + const path = '/api/v1/videos/' + return videoFile.metadata + ? baseUrlHttp + path + this.uuid + '/metadata/' + videoFile.id + : videoFile.metadataUrl + } + getVideoRedundancyUrl (videoFile: MVideoFile, baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.REDUNDANCY + getVideoFilename(this, videoFile) } diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 3e73ccbfa..ce0dd14d5 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -4,7 +4,14 @@ import * as chai from 'chai' import 'mocha' import { omit } from 'lodash' import { getMaxBitrate, VideoDetails, VideoResolution, VideoState } from '../../../../shared/models/videos' -import { audio, canDoQuickTranscode, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' +import { + audio, + canDoQuickTranscode, + getVideoFileBitrate, + getVideoFileFPS, + getVideoFileResolution, + getMetadataFromFile +} from '../../../helpers/ffmpeg-utils' import { buildAbsoluteFixturePath, cleanupTests, @@ -14,6 +21,7 @@ import { generateVideoWithFramerate, getMyVideos, getVideo, + getVideoFileMetadataUrl, getVideosList, makeGetRequest, root, @@ -25,6 +33,7 @@ import { } from '../../../../shared/extra-utils' import { join } from 'path' import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants' +import { FfprobeData } from 'fluent-ffmpeg' const expect = chai.expect @@ -458,6 +467,68 @@ describe('Test video transcoding', function () { } }) + it('Should provide valid ffprobe data', async function () { + this.timeout(160000) + + const videoAttributes = { + name: 'my super name for server 1', + description: 'my super description for server 1', + fixture: 'video_short.webm' + } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) + + await waitJobs(servers) + + const res = await getVideosList(servers[1].url) + + const videoOnOrigin = res.body.data.find(v => v.name === videoAttributes.name) + const res2 = await getVideo(servers[1].url, videoOnOrigin.id) + const videoOnOriginDetails: VideoDetails = res2.body + + { + const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', videoOnOrigin.uuid + '-240.mp4') + const metadata = await getMetadataFromFile(path) + for (const p of [ + // expected format properties + 'format.encoder', + 'format.format_long_name', + 'format.size', + 'format.bit_rate', + // expected stream properties + 'stream[0].codec_long_name', + 'stream[0].profile', + 'stream[0].width', + 'stream[0].height', + 'stream[0].display_aspect_ratio', + 'stream[0].avg_frame_rate', + 'stream[0].pix_fmt' + ]) { + expect(metadata).to.have.nested.property(p) + } + expect(metadata).to.not.have.nested.property('format.filename') + } + + for (const server of servers) { + const res = await getVideosList(server.url) + + const video = res.body.data.find(v => v.name === videoAttributes.name) + const res2 = await getVideo(server.url, video.id) + const videoDetails = res2.body + + const videoFiles = videoDetails.files + for (const [ index, file ] of videoFiles.entries()) { + expect(file.metadata).to.be.undefined + expect(file.metadataUrl).to.contain(servers[1].url) + expect(file.metadataUrl).to.contain(videoOnOrigin.uuid) + + const res3 = await getVideoFileMetadataUrl(file.metadataUrl) + const metadata: FfprobeData = res3.body + expect(metadata).to.have.nested.property('format.size') + expect(metadata.format.size).to.equal(videoOnOriginDetails.files[index].metadata.format.size) + } + } + }) + after(async function () { await cleanupTests(servers) }) diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 39a06b0d7..0d36a38a2 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -95,6 +95,14 @@ function getVideo (url: string, id: number | string, expectedStatus = 200) { .expect(expectedStatus) } +function getVideoFileMetadataUrl (url: string) { + return request(url) + .get('/') + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + function viewVideo (url: string, id: number | string, expectedStatus = 204, xForwardedFor?: string) { const path = '/api/v1/videos/' + id + '/views' @@ -643,6 +651,7 @@ export { getAccountVideos, getVideoChannelVideos, getVideo, + getVideoFileMetadataUrl, getVideoWithToken, getVideosList, getVideosListPagination, diff --git a/shared/models/activitypub/objects/common-objects.ts b/shared/models/activitypub/objects/common-objects.ts index e94d05429..bb3ffe678 100644 --- a/shared/models/activitypub/objects/common-objects.ts +++ b/shared/models/activitypub/objects/common-objects.ts @@ -28,6 +28,15 @@ export type ActivityPlaylistSegmentHashesObject = { href: string } +export type ActivityVideoFileMetadataObject = { + type: 'Link' + rel: [ 'metadata', any ] + mediaType: 'application/json' + height: number + href: string + fps: number +} + export type ActivityPlaylistInfohashesObject = { type: 'Infohash' name: string @@ -80,6 +89,7 @@ export type ActivityTagObject = | ActivityMentionObject | ActivityBitTorrentUrlObject | ActivityMagnetUrlObject + | ActivityVideoFileMetadataObject export type ActivityUrlObject = ActivityVideoUrlObject @@ -87,6 +97,7 @@ export type ActivityUrlObject = | ActivityBitTorrentUrlObject | ActivityMagnetUrlObject | ActivityHtmlUrlObject + | ActivityVideoFileMetadataObject export interface ActivityPubAttributedTo { type: 'Group' | 'Person' diff --git a/shared/models/videos/video-file-metadata.ts b/shared/models/videos/video-file-metadata.ts new file mode 100644 index 000000000..15683cacf --- /dev/null +++ b/shared/models/videos/video-file-metadata.ts @@ -0,0 +1,18 @@ +import { FfprobeData } from "fluent-ffmpeg" +import { DeepOmit } from "@server/models/utils" + +export type VideoFileMetadataModel = DeepOmit + +export class VideoFileMetadata implements VideoFileMetadataModel { + streams: { [x: string]: any, [x: number]: any }[] + format: { [x: string]: any, [x: number]: any } + chapters: any[] + + constructor (hash: Partial) { + this.chapters = hash.chapters + this.format = hash.format + this.streams = hash.streams + + delete this.format.filename + } +} diff --git a/shared/models/videos/video-file.model.ts b/shared/models/videos/video-file.model.ts index 04da0627e..6cc2d5aee 100644 --- a/shared/models/videos/video-file.model.ts +++ b/shared/models/videos/video-file.model.ts @@ -1,4 +1,5 @@ import { VideoConstant, VideoResolution } from '@shared/models' +import { FfprobeData } from 'fluent-ffmpeg' export interface VideoFile { magnetUri: string @@ -9,4 +10,6 @@ export interface VideoFile { fileUrl: string fileDownloadUrl: string fps: number + metadata?: FfprobeData + metadataUrl?: string }