Add video file metadata to download modal, via ffprobe (#2411)

* Add video file metadata via ffprobe

* Federate video file metadata

* Add tests for file metadata generation

* Complete tests for videoFile metadata federation

* Lint migration and video-file for metadata

* Objectify metadata from getter in ffmpeg-utils

* Add metadataUrl to all videoFiles

* Simplify metadata API middleware

* Load playlist in videoFile when requesting metadata

client/src/app/shared/video/modals/video-download.component.html

@@ -20,7 +20,7 @@
     <div class="form-group">
       <div class="input-group input-group-sm">
         <div class="input-group-prepend peertube-select-container">
-          <select *ngIf="type === 'video'" [(ngModel)]="resolutionId">
+          <select *ngIf="type === 'video'" [(ngModel)]="resolutionId" (ngModelChange)="onResolutionIdChange()">
             <option *ngFor="let file of getVideoFiles()" [value]="file.resolution.id">{{ file.resolution.label }}</option>
           </select>
 
@@ -38,6 +38,42 @@
       </div>
     </div>
 
+    <ngb-tabset *ngIf="type === 'video' && videoFile?.metadata">
+      <ngb-tab>
+        <ng-template ngbTabTitle i18n>Format</ng-template>
+        <ng-template ngbTabContent>
+          <div class="file-metadata">
+            <div class="metadata-attribute metadata-attribute-tags" *ngFor="let item of videoFileMetadataFormat | keyvalue">
+              <span i18n class="metadata-attribute-label">{{ item.value.label }}</span>
+              <span class="metadata-attribute-value">{{ item.value.value }}</span>
+            </div>
+          </div>
+        </ng-template>
+      </ngb-tab>
+      <ngb-tab [disabled]="videoFileMetadataVideoStream === undefined">
+        <ng-template ngbTabTitle i18n>Video stream</ng-template>
+        <ng-template ngbTabContent>
+          <div class="file-metadata">
+            <div class="metadata-attribute metadata-attribute-tags" *ngFor="let item of videoFileMetadataVideoStream | keyvalue">
+              <span i18n class="metadata-attribute-label">{{ item.value.label }}</span>
+              <span class="metadata-attribute-value">{{ item.value.value }}</span>
+            </div>
+          </div>
+        </ng-template>
+      </ngb-tab>
+      <ngb-tab [disabled]="videoFileMetadataAudioStream === undefined">
+        <ng-template ngbTabTitle i18n>Audio stream</ng-template>
+        <ng-template ngbTabContent>
+          <div class="file-metadata">
+            <div class="metadata-attribute metadata-attribute-tags" *ngFor="let item of videoFileMetadataAudioStream | keyvalue">
+              <span i18n class="metadata-attribute-label">{{ item.value.label }}</span>
+              <span class="metadata-attribute-value">{{ item.value.value }}</span>
+            </div>
+          </div>
+        </ng-template>
+      </ngb-tab>
+    </ngb-tabset>
+
     <div class="download-type" *ngIf="type === 'video'">
       <div class="peertube-radio-container">
         <input type="radio" name="download" id="download-direct" [(ngModel)]="downloadType" value="direct">

client/src/app/shared/video/modals/video-download.component.scss

@@ -27,3 +27,38 @@
     margin-right: 30px;
   }
 }
+
+.file-metadata {
+  padding: 1rem;
+}
+
+.file-metadata .metadata-attribute {
+  font-size: 13px;
+  display: block;
+  margin-bottom: 12px;
+
+  .metadata-attribute-label {
+    min-width: 142px;
+    padding-right: 5px;
+    display: inline-block;
+    color: $grey-foreground-color;
+    font-weight: $font-bold;
+  }
+
+  a.metadata-attribute-value {
+    @include disable-default-a-behaviour;
+    color: var(--mainForegroundColor);
+
+    &:hover {
+      opacity: 0.9;
+    }
+  }
+
+  &.metadata-attribute-tags {
+    .metadata-attribute-value:not(:nth-child(2)) {
+      &::before {
+        content: ', '
+      }
+    }
+  }
+}

client/src/app/shared/video/modals/video-download.component.ts

@@ -3,9 +3,15 @@ import { VideoDetails } from '../../../shared/video/video-details.model'
 import { NgbActiveModal, NgbModal } from '@ng-bootstrap/ng-bootstrap'
 import { I18n } from '@ngx-translate/i18n-polyfill'
 import { AuthService, Notifier } from '@app/core'
-import { VideoPrivacy, VideoCaption } from '@shared/models'
+import { VideoPrivacy, VideoCaption, VideoFile } from '@shared/models'
+import { FfprobeFormat, FfprobeStream } from 'fluent-ffmpeg'
+import { mapValues, pick } from 'lodash-es'
+import { NumberFormatterPipe } from '@app/shared/angular/number-formatter.pipe'
+import { BytesPipe } from 'ngx-pipes'
+import { VideoService } from '../video.service'
 
 type DownloadType = 'video' | 'subtitles'
+type FileMetadata = { [key: string]: { label: string, value: string }}
 
 @Component({
   selector: 'my-video-download',
@@ -20,17 +26,28 @@ export class VideoDownloadComponent {
   subtitleLanguageId: string
 
   video: VideoDetails
+  videoFile: VideoFile
+  videoFileMetadataFormat: FileMetadata
+  videoFileMetadataVideoStream: FileMetadata | undefined
+  videoFileMetadataAudioStream: FileMetadata | undefined
   videoCaptions: VideoCaption[]
   activeModal: NgbActiveModal
 
   type: DownloadType = 'video'
 
+  private bytesPipe: BytesPipe
+  private numbersPipe: NumberFormatterPipe
+
   constructor (
     private notifier: Notifier,
     private modalService: NgbModal,
+    private videoService: VideoService,
     private auth: AuthService,
     private i18n: I18n
-  ) { }
+  ) {
+    this.bytesPipe = new BytesPipe()
+    this.numbersPipe = new NumberFormatterPipe()
+  }
 
   get typeText () {
     return this.type === 'video'
@@ -51,6 +68,7 @@ export class VideoDownloadComponent {
     this.activeModal = this.modalService.open(this.modal, { centered: true })
 
     this.resolutionId = this.getVideoFiles()[0].resolution.id
+    this.onResolutionIdChange()
     if (this.videoCaptions) this.subtitleLanguageId = this.videoCaptions[0].language.id
   }
 
@@ -67,10 +85,27 @@ export class VideoDownloadComponent {
   getLink () {
     return this.type === 'subtitles' && this.videoCaptions
       ? this.getSubtitlesLink()
-      : this.getVideoLink()
+      : this.getVideoFileLink()
   }
 
-  getVideoLink () {
+  async onResolutionIdChange () {
+    this.videoFile = this.getVideoFile()
+    if (this.videoFile.metadata || !this.videoFile.metadataUrl) return
+
+    await this.hydrateMetadataFromMetadataUrl(this.videoFile)
+
+    this.videoFileMetadataFormat = this.videoFile
+      ? this.getMetadataFormat(this.videoFile.metadata.format)
+      : undefined
+    this.videoFileMetadataVideoStream = this.videoFile
+      ? this.getMetadataStream(this.videoFile.metadata.streams, 'video')
+      : undefined
+    this.videoFileMetadataAudioStream = this.videoFile
+      ? this.getMetadataStream(this.videoFile.metadata.streams, 'audio')
+      : undefined
+  }
+
+  getVideoFile () {
     // HTML select send us a string, so convert it to a number
     this.resolutionId = parseInt(this.resolutionId.toString(), 10)
 
@@ -79,6 +114,12 @@ export class VideoDownloadComponent {
       console.error('Could not find file with resolution %d.', this.resolutionId)
       return
     }
+    return file
+  }
+
+  getVideoFileLink () {
+    const file = this.videoFile
+    if (!file) return
 
     const suffix = this.video.privacy.id === VideoPrivacy.PRIVATE || this.video.privacy.id === VideoPrivacy.INTERNAL
       ? '?access_token=' + this.auth.getAccessToken()
@@ -104,4 +145,64 @@ export class VideoDownloadComponent {
   switchToType (type: DownloadType) {
     this.type = type
   }
+
+  getMetadataFormat (format: FfprobeFormat) {
+    const keyToTranslateFunction = {
+      'encoder': (value: string) => ({ label: this.i18n('Encoder'), value }),
+      'format_long_name': (value: string) => ({ label: this.i18n('Format name'), value }),
+      'size': (value: number) => ({ label: this.i18n('Size'), value: this.bytesPipe.transform(value, 2) }),
+      'bit_rate': (value: number) => ({
+        label: this.i18n('Bitrate'),
+        value: `${this.numbersPipe.transform(value)}bps`
+      })
+    }
+
+    // flattening format
+    const sanitizedFormat = Object.assign(format, format.tags)
+    delete sanitizedFormat.tags
+
+    return mapValues(
+      pick(sanitizedFormat, Object.keys(keyToTranslateFunction)),
+      (val, key) => keyToTranslateFunction[key](val)
+    )
+  }
+
+  getMetadataStream (streams: FfprobeStream[], type: 'video' | 'audio') {
+    const stream = streams.find(s => s.codec_type === type)
+    if (!stream) return undefined
+
+    let keyToTranslateFunction = {
+      'codec_long_name': (value: string) => ({ label: this.i18n('Codec'), value }),
+      'profile': (value: string) => ({ label: this.i18n('Profile'), value }),
+      'bit_rate': (value: number) => ({
+        label: this.i18n('Bitrate'),
+        value: `${this.numbersPipe.transform(value)}bps`
+      })
+    }
+
+    if (type === 'video') {
+      keyToTranslateFunction = Object.assign(keyToTranslateFunction, {
+        'width': (value: number) => ({ label: this.i18n('Resolution'), value: `${value}x${stream.height}` }),
+        'display_aspect_ratio': (value: string) => ({ label: this.i18n('Aspect ratio'), value }),
+        'avg_frame_rate': (value: string) => ({ label: this.i18n('Average frame rate'), value }),
+        'pix_fmt': (value: string) => ({ label: this.i18n('Pixel format'), value })
+      })
+    } else {
+      keyToTranslateFunction = Object.assign(keyToTranslateFunction, {
+        'sample_rate': (value: number) => ({ label: this.i18n('Sample rate'), value }),
+        'channel_layout': (value: number) => ({ label: this.i18n('Channel Layout'), value })
+      })
+    }
+
+    return mapValues(
+      pick(stream, Object.keys(keyToTranslateFunction)),
+      (val, key) => keyToTranslateFunction[key](val)
+    )
+  }
+
+  private hydrateMetadataFromMetadataUrl (file: VideoFile) {
+    const observable = this.videoService.getVideoFileMetadata(file.metadataUrl)
+    observable.subscribe(res => file.metadata = res)
+    return observable.toPromise()
+  }
 }

client/src/app/shared/video/video.service.ts

@@ -32,6 +32,7 @@ import { UserSubscriptionService } from '@app/shared/user-subscription/user-subs
 import { VideoChannel } from '@app/shared/video-channel/video-channel.model'
 import { I18n } from '@ngx-translate/i18n-polyfill'
 import { NSFWPolicyType } from '@shared/models/videos/nsfw-policy.type'
+import { FfprobeData } from 'fluent-ffmpeg'
 
 export interface VideosProvider {
   getVideos (parameters: {
@@ -291,6 +292,14 @@ export class VideoService implements VideosProvider {
     return this.buildBaseFeedUrls(params)
   }
 
+  getVideoFileMetadata (metadataUrl: string) {
+    return this.authHttp
+               .get<FfprobeData>(metadataUrl)
+               .pipe(
+                 catchError(err => this.restExtractor.handleError(err))
+               )
+  }
+
   removeVideo (id: number) {
     return this.authHttp
                .delete(VideoService.BASE_VIDEO_URL + id)

client/src/sass/bootstrap.scss

@@ -109,6 +109,11 @@ $icon-font-path: '~@neos21/bootstrap3-glyphicons/assets/fonts/';
       margin: 0;
       padding: 0;
       opacity: .5;
+
+      &[iconName="cross"] {
+        @include icon(16px);
+        top: -3px;
+      }
     }
   }
 
@@ -153,7 +158,7 @@ $icon-font-path: '~@neos21/bootstrap3-glyphicons/assets/fonts/';
   }
 }
 
-ngb-tabset.bootstrap {
+ngb-tabset {
 
   .nav-link {
     &, & a {

server/controllers/api/videos/index.ts

@@ -1,7 +1,7 @@
 import * as express from 'express'
 import { extname } from 'path'
 import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../shared'
-import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
+import { getVideoFileFPS, getVideoFileResolution, getMetadataFromFile } from '../../../helpers/ffmpeg-utils'
 import { logger } from '../../../helpers/logger'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
@@ -37,7 +37,8 @@ import {
   videosGetValidator,
   videosRemoveValidator,
   videosSortValidator,
-  videosUpdateValidator
+  videosUpdateValidator,
+  videoFileMetadataGetValidator
 } from '../../../middlewares'
 import { TagModel } from '../../../models/video/tag'
 import { VideoModel } from '../../../models/video/video'
@@ -66,6 +67,7 @@ import { Hooks } from '../../../lib/plugins/hooks'
 import { MVideoDetails, MVideoFullLight } from '@server/typings/models'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getVideoFilePath } from '@server/lib/video-paths'
+import toInt from 'validator/lib/toInt'
 
 const auditLogger = auditLoggerFactory('videos')
 const videosRouter = express.Router()
@@ -128,6 +130,10 @@ videosRouter.get('/:id/description',
   asyncMiddleware(videosGetValidator),
   asyncMiddleware(getVideoDescription)
 )
+videosRouter.get('/:id/metadata/:videoFileId',
+  asyncMiddleware(videoFileMetadataGetValidator),
+  asyncMiddleware(getVideoFileMetadata)
+)
 videosRouter.get('/:id',
   optionalAuthenticate,
   asyncMiddleware(videosCustomGetValidator('only-video-with-rights')),
@@ -206,7 +212,8 @@ async function addVideo (req: express.Request, res: express.Response) {
   const videoFile = new VideoFileModel({
     extname: extname(videoPhysicalFile.filename),
     size: videoPhysicalFile.size,
-    videoStreamingPlaylistId: null
+    videoStreamingPlaylistId: null,
+    metadata: await getMetadataFromFile<any>(videoPhysicalFile.path)
   })
 
   if (videoFile.isAudio()) {
@@ -493,6 +500,11 @@ async function getVideoDescription (req: express.Request, res: express.Response)
   return res.json({ description })
 }
 
+async function getVideoFileMetadata (req: express.Request, res: express.Response) {
+  const videoFile = await VideoFileModel.loadWithMetadata(toInt(req.params.videoFileId))
+  return res.json(videoFile.metadata)
+}
+
 async function listVideos (req: express.Request, res: express.Response) {
   const countVideos = getCountVideos(req)
 

server/helpers/ffmpeg-utils.ts

@@ -7,6 +7,7 @@ import { logger } from './logger'
 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
 import { readFile, remove, writeFile } from 'fs-extra'
 import { CONFIG } from '../initializers/config'
+import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
 
 /**
  * A toolbox to play with audio
@@ -169,24 +170,26 @@ async function getVideoFileFPS (path: string) {
   return 0
 }
 
-async function getVideoFileBitrate (path: string) {
+async function getMetadataFromFile<T> (path: string, cb = metadata => metadata) {
-  return new Promise<number>((res, rej) => {
+  return new Promise<T>((res, rej) => {
     ffmpeg.ffprobe(path, (err, metadata) => {
       if (err) return rej(err)
 
-      return res(metadata.format.bit_rate)
+      return res(cb(new VideoFileMetadata(metadata)))
     })
   })
 }
 
-function getDurationFromVideoFile (path: string) {
+async function getVideoFileBitrate (path: string) {
-  return new Promise<number>((res, rej) => {
+  return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
-    ffmpeg.ffprobe(path, (err, metadata) => {
+}
-      if (err) return rej(err)
 
-      return res(Math.floor(metadata.format.duration))
+function getDurationFromVideoFile (path: string) {
-    })
+  return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
-  })
+}
+
+function getVideoStreamFromFile (path: string) {
+  return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
 }
 
 async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
@@ -341,6 +344,7 @@ export {
   getAudioStreamCodec,
   getVideoStreamSize,
   getVideoFileResolution,
+  getMetadataFromFile,
   getDurationFromVideoFile,
   generateImageFromVideoFile,
   TranscodeOptions,
@@ -450,17 +454,6 @@ async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
   await writeFile(options.outputPath, newContent)
 }
 
-function getVideoStreamFromFile (path: string) {
-  return new Promise<any>((res, rej) => {
-    ffmpeg.ffprobe(path, (err, metadata) => {
-      if (err) return rej(err)
-
-      const videoStream = metadata.streams.find(s => s.codec_type === 'video')
-      return res(videoStream || null)
-    })
-  })
-}
-
 /**
  * A slightly customised version of the 'veryfast' x264 preset
  *

server/helpers/middlewares/videos.ts

@@ -12,6 +12,7 @@ import {
   MVideoThumbnail,
   MVideoWithRights
 } from '@server/typings/models'
+import { VideoFileModel } from '@server/models/video/video-file'
 
 async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') {
   const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined
@@ -51,6 +52,18 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi
   return true
 }
 
+async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) {
+  if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) {
+    res.status(404)
+       .json({ error: 'VideoFile matching Video not found' })
+       .end()
+
+    return false
+  }
+
+  return true
+}
+
 async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) {
   if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) {
     const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId)
@@ -107,5 +120,6 @@ function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right:
 export {
   doesVideoChannelOfAccountExist,
   doesVideoExist,
+  doesVideoFileOfVideoExist,
   checkUserCanManageVideo
 }

server/initializers/constants.ts

@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 480
+const LAST_MIGRATION_VERSION = 485
 
 // ---------------------------------------------------------------------------
 

server/initializers/migrations/0485-video-file-metadata.ts

@@ -0,0 +1,30 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<void> {
+
+  const metadata = {
+    type: Sequelize.JSONB,
+    allowNull: true
+  }
+  await utils.queryInterface.addColumn('videoFile', 'metadata', metadata)
+
+  const metadataUrl = {
+    type: Sequelize.STRING,
+    allowNull: true
+  }
+  await utils.queryInterface.addColumn('videoFile', 'metadataUrl', metadataUrl)
+
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/lib/activitypub/videos.ts

@@ -10,7 +10,8 @@ import {
   ActivityTagObject,
   ActivityUrlObject,
   ActivityVideoUrlObject,
-  VideoState
+  VideoState,
+  ActivityVideoFileMetadataObject
 } from '../../../shared/index'
 import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
 import { VideoPrivacy } from '../../../shared/models/videos'
@@ -526,6 +527,10 @@ function isAPHashTagObject (url: any): url is ActivityHashTagObject {
   return url && url.type === 'Hashtag'
 }
 
+function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject {
+  return url && url.type === 'Link' && url.mediaType === 'application/json' && url.hasAttribute('rel') && url.rel.includes('metadata')
+}
+
 async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
   logger.debug('Adding remote video %s.', videoObject.id)
 
@@ -694,6 +699,14 @@ function videoFileActivityUrlToDBAttributes (
       throw new Error('Cannot parse magnet URI ' + magnet.href)
     }
 
+    // Fetch associated metadata url, if any
+    const metadata = urls.filter(isAPVideoFileMetadataObject)
+                          .find(u =>
+                            u.height === fileUrl.height &&
+                            u.fps === fileUrl.fps &&
+                            u.rel.includes(fileUrl.mediaType)
+                          )
+
     const mediaType = fileUrl.mediaType
     const attribute = {
       extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType],
@@ -701,6 +714,7 @@ function videoFileActivityUrlToDBAttributes (
       resolution: fileUrl.height,
       size: fileUrl.size,
       fps: fileUrl.fps || -1,
+      metadataUrl: metadata?.href,
 
       // This is a video file owned by a video or by a streaming playlist
       videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,

server/lib/video-transcoding.ts

@@ -2,6 +2,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSER
 import { basename, extname as extnameUtil, join } from 'path'
 import {
   canDoQuickTranscode,
+  getMetadataFromFile,
   getDurationFromVideoFile,
   getVideoFileFPS,
   transcode,
@@ -19,6 +20,7 @@ import { CONFIG } from '../initializers/config'
 import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoWithAllFiles, MVideoWithFile } from '@server/typings/models'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { generateVideoStreamingPlaylistName, getVideoFilename, getVideoFilePath } from './video-paths'
+import { extractVideo } from './videos'
 
 /**
  * Optimize the original video file and replace it. The resolution is not changed.
@@ -202,6 +204,7 @@ async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoReso
 
   newVideoFile.size = stats.size
   newVideoFile.fps = await getVideoFileFPS(videoFilePath)
+  newVideoFile.metadata = await getMetadataFromFile(videoFilePath)
 
   await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile)
 
@@ -230,11 +233,16 @@ export {
 async function onVideoFileTranscoding (video: MVideoWithFile, videoFile: MVideoFile, transcodingPath: string, outputPath: string) {
   const stats = await stat(transcodingPath)
   const fps = await getVideoFileFPS(transcodingPath)
+  const metadata = await getMetadataFromFile(transcodingPath)
 
   await move(transcodingPath, outputPath)
 
+  const extractedVideo = extractVideo(video)
+
   videoFile.size = stats.size
   videoFile.fps = fps
+  videoFile.metadata = metadata
+  videoFile.metadataUrl = extractedVideo.getVideoFileMetadataUrl(videoFile, extractedVideo.getBaseUrls().baseUrlHttp)
 
   await createTorrentAndSetInfoHash(video, videoFile)
 

server/middlewares/validators/videos/videos.ts

@@ -42,7 +42,12 @@ import { getServerActor } from '../../../helpers/utils'
 import { CONFIG } from '../../../initializers/config'
 import { isLocalVideoAccepted } from '../../../lib/moderation'
 import { Hooks } from '../../../lib/plugins/hooks'
-import { checkUserCanManageVideo, doesVideoChannelOfAccountExist, doesVideoExist } from '../../../helpers/middlewares'
+import {
+  checkUserCanManageVideo,
+  doesVideoChannelOfAccountExist,
+  doesVideoExist,
+  doesVideoFileOfVideoExist
+} from '../../../helpers/middlewares'
 import { MVideoFullLight } from '@server/typings/models'
 import { getVideoWithAttributes } from '../../../helpers/video'
 
@@ -198,6 +203,20 @@ const videosCustomGetValidator = (
 const videosGetValidator = videosCustomGetValidator('all')
 const videosDownloadValidator = videosCustomGetValidator('all', true)
 
+const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
+  param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
+  param('videoFileId').custom(isIdValid).not().isEmpty().withMessage('Should have a valid videoFileId'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoFileMetadataGet parameters', { parameters: req.params })
+
+    if (areValidationErrors(req, res)) return
+    if (!await doesVideoFileOfVideoExist(+req.params.videoFileId, req.params.id, res)) return
+
+    return next()
+  }
+])
+
 const videosRemoveValidator = [
   param('id').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid id'),
 
@@ -411,6 +430,7 @@ export {
   videosAddValidator,
   videosUpdateValidator,
   videosGetValidator,
+  videoFileMetadataGetValidator,
   videosDownloadValidator,
   checkVideoFollowConstraints,
   videosCustomGetValidator,

server/models/redundancy/video-redundancy.ts

@@ -528,7 +528,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
       include: [
         {
           required: false,
-          model: VideoFileModel.unscoped(),
+          model: VideoFileModel,
           include: [
             {
               model: VideoRedundancyModel.unscoped(),
@@ -547,7 +547,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
               where: redundancyWhere
             },
             {
-              model: VideoFileModel.unscoped(),
+              model: VideoFileModel,
               required: false
             }
           ]
@@ -699,7 +699,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
 
     return {
       attributes: [],
-      model: VideoFileModel.unscoped(),
+      model: VideoFileModel,
       required: true,
       where: {
         id: {

server/models/utils.ts

@@ -3,6 +3,23 @@ import validator from 'validator'
 import { Col } from 'sequelize/types/lib/utils'
 import { literal, OrderItem } from 'sequelize'
 
+type Primitive = string | Function | number | boolean | Symbol | undefined | null
+type DeepOmitHelper<T, K extends keyof T> = {
+  [P in K]: // extra level of indirection needed to trigger homomorhic behavior
+  T[P] extends infer TP // distribute over unions
+    ? TP extends Primitive
+      ? TP // leave primitives and functions alone
+      : TP extends any[]
+        ? DeepOmitArray<TP, K> // Array special handling
+        : DeepOmit<TP, K>
+    : never
+}
+type DeepOmit<T, K> = T extends Primitive ? T : DeepOmitHelper<T, Exclude<keyof T, K>>
+
+type DeepOmitArray<T extends any[], K> = {
+  [P in keyof T]: DeepOmit<T[P], K>
+}
+
 type SortType = { sortModel: string, sortValue: string }
 
 // Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
@@ -193,6 +210,7 @@ function buildDirectionAndField (value: string) {
 // ---------------------------------------------------------------------------
 
 export {
+  DeepOmit,
   buildBlockedAccountSQL,
   buildLocalActorIdsIn,
   SortType,

server/models/video/video-file.ts

@@ -10,7 +10,9 @@ import {
   Is,
   Model,
   Table,
-  UpdatedAt
+  UpdatedAt,
+  Scopes,
+  DefaultScope
 } from 'sequelize-typescript'
 import {
   isVideoFileExtnameValid,
@@ -29,6 +31,60 @@ import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '.
 import { MStreamingPlaylistVideo, MVideo } from '@server/typings/models'
 import * as memoizee from 'memoizee'
 
+export enum ScopeNames {
+  WITH_VIDEO = 'WITH_VIDEO',
+  WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST',
+  WITH_METADATA = 'WITH_METADATA'
+}
+
+const METADATA_FIELDS = [ 'metadata', 'metadataUrl' ]
+
+@DefaultScope(() => ({
+  attributes: {
+    exclude: [ METADATA_FIELDS[0] ]
+  }
+}))
+@Scopes(() => ({
+  [ScopeNames.WITH_VIDEO]: {
+    include: [
+      {
+        model: VideoModel.unscoped(),
+        required: true
+      }
+    ]
+  },
+  [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (videoIdOrUUID: string | number) => {
+    const where = (typeof videoIdOrUUID === 'number')
+      ? { id: videoIdOrUUID }
+      : { uuid: videoIdOrUUID }
+
+    return {
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: false,
+          where
+        },
+        {
+          model: VideoStreamingPlaylistModel.unscoped(),
+          required: false,
+          include: [
+            {
+              model: VideoModel.unscoped(),
+              required: true,
+              where
+            }
+          ]
+        }
+      ]
+    }
+  },
+  [ScopeNames.WITH_METADATA]: {
+    attributes: {
+      include: METADATA_FIELDS
+    }
+  }
+}))
 @Table({
   tableName: 'videoFile',
   indexes: [
@@ -106,6 +162,14 @@ export class VideoFileModel extends Model<VideoFileModel> {
   @Column
   fps: number
 
+  @AllowNull(true)
+  @Column(DataType.JSONB)
+  metadata: any
+
+  @AllowNull(true)
+  @Column
+  metadataUrl: string
+
   @ForeignKey(() => VideoModel)
   @Column
   videoId: number
@@ -157,17 +221,29 @@ export class VideoFileModel extends Model<VideoFileModel> {
               .then(results => results.length === 1)
   }
 
-  static loadWithVideo (id: number) {
+  static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
-    const options = {
+    const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
-      include: [
+    return (videoFile?.Video.id === videoIdOrUUID) ||
-        {
+           (videoFile?.Video.uuid === videoIdOrUUID) ||
-          model: VideoModel.unscoped(),
+           (videoFile?.VideoStreamingPlaylist?.Video?.id === videoIdOrUUID) ||
-          required: true
+           (videoFile?.VideoStreamingPlaylist?.Video?.uuid === videoIdOrUUID)
-        }
+  }
-      ]
-    }
 
-    return VideoFileModel.findByPk(id, options)
+  static loadWithMetadata (id: number) {
+    return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
+  }
+
+  static loadWithVideo (id: number) {
+    return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id)
+  }
+
+  static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
+    return VideoFileModel.scope({
+      method: [
+        ScopeNames.WITH_VIDEO_OR_PLAYLIST,
+        videoIdOrUUID
+      ]
+    }).findByPk(id)
   }
 
   static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {

server/models/video/video-format-utils.ts

@@ -23,6 +23,7 @@ import {
 import { MVideoFileRedundanciesOpt } from '../../typings/models/video/video-file'
 import { VideoFile } from '@shared/models/videos/video-file.model'
 import { generateMagnetUri } from '@server/helpers/webtorrent'
+import { extractVideo } from '@server/lib/videos'
 
 export type VideoFormattingJSONOptions = {
   completeDescription?: boolean
@@ -193,7 +194,8 @@ function videoFilesModelToFormattedJSON (
         torrentUrl: model.getTorrentUrl(videoFile, baseUrlHttp),
         torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp),
         fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp),
-        fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp)
+        fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp),
+        metadataUrl: videoFile.metadataUrl // only send the metadataUrl and not the metadata over the wire
       } as VideoFile
     })
     .sort((a, b) => {
@@ -220,6 +222,15 @@ function addVideoFilesInAPAcc (
       fps: file.fps
     })
 
+    acc.push({
+      type: 'Link',
+      rel: [ 'metadata', MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] ],
+      mediaType: 'application/json' as 'application/json',
+      href: extractVideo(model).getVideoFileMetadataUrl(file, baseUrlHttp),
+      height: file.resolution,
+      fps: file.fps
+    })
+
     acc.push({
       type: 'Link',
       mediaType: 'application/x-bittorrent' as 'application/x-bittorrent',

server/models/video/video.ts

@@ -216,7 +216,7 @@ export type AvailableForListIDsOptions = {
 
     if (options.withFiles === true) {
       query.include.push({
-        model: VideoFileModel.unscoped(),
+        model: VideoFileModel,
         required: true
       })
     }
@@ -337,7 +337,7 @@ export type AvailableForListIDsOptions = {
     return {
       include: [
         {
-          model: VideoFileModel.unscoped(),
+          model: VideoFileModel,
           separate: true, // We may have multiple files, having multiple redundancies so let's separate this join
           required: false,
           include: subInclude
@@ -348,7 +348,7 @@ export type AvailableForListIDsOptions = {
   [ScopeNames.WITH_STREAMING_PLAYLISTS]: (withRedundancies = false) => {
     const subInclude: IncludeOptions[] = [
       {
-        model: VideoFileModel.unscoped(),
+        model: VideoFileModel,
         required: false
       }
     ]
@@ -1847,6 +1847,13 @@ export class VideoModel extends Model<VideoModel> {
     return baseUrlHttp + STATIC_PATHS.WEBSEED + getVideoFilename(this, videoFile)
   }
 
+  getVideoFileMetadataUrl (videoFile: MVideoFile, baseUrlHttp: string) {
+    const path = '/api/v1/videos/'
+    return videoFile.metadata
+      ? baseUrlHttp + path + this.uuid + '/metadata/' + videoFile.id
+      : videoFile.metadataUrl
+  }
+
   getVideoRedundancyUrl (videoFile: MVideoFile, baseUrlHttp: string) {
     return baseUrlHttp + STATIC_PATHS.REDUNDANCY + getVideoFilename(this, videoFile)
   }

server/tests/api/videos/video-transcoder.ts

@@ -4,7 +4,14 @@ import * as chai from 'chai'
 import 'mocha'
 import { omit } from 'lodash'
 import { getMaxBitrate, VideoDetails, VideoResolution, VideoState } from '../../../../shared/models/videos'
-import { audio, canDoQuickTranscode, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
+import {
+  audio,
+  canDoQuickTranscode,
+  getVideoFileBitrate,
+  getVideoFileFPS,
+  getVideoFileResolution,
+  getMetadataFromFile
+} from '../../../helpers/ffmpeg-utils'
 import {
   buildAbsoluteFixturePath,
   cleanupTests,
@@ -14,6 +21,7 @@ import {
   generateVideoWithFramerate,
   getMyVideos,
   getVideo,
+  getVideoFileMetadataUrl,
   getVideosList,
   makeGetRequest,
   root,
@@ -25,6 +33,7 @@ import {
 } from '../../../../shared/extra-utils'
 import { join } from 'path'
 import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants'
+import { FfprobeData } from 'fluent-ffmpeg'
 
 const expect = chai.expect
 
@@ -458,6 +467,68 @@ describe('Test video transcoding', function () {
     }
   })
 
+  it('Should provide valid ffprobe data', async function () {
+    this.timeout(160000)
+
+    const videoAttributes = {
+      name: 'my super name for server 1',
+      description: 'my super description for server 1',
+      fixture: 'video_short.webm'
+    }
+    await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
+
+    await waitJobs(servers)
+
+    const res = await getVideosList(servers[1].url)
+
+    const videoOnOrigin = res.body.data.find(v => v.name === videoAttributes.name)
+    const res2 = await getVideo(servers[1].url, videoOnOrigin.id)
+    const videoOnOriginDetails: VideoDetails = res2.body
+
+    {
+      const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', videoOnOrigin.uuid + '-240.mp4')
+      const metadata = await getMetadataFromFile(path)
+      for (const p of [
+        // expected format properties
+        'format.encoder',
+        'format.format_long_name',
+        'format.size',
+        'format.bit_rate',
+        // expected stream properties
+        'stream[0].codec_long_name',
+        'stream[0].profile',
+        'stream[0].width',
+        'stream[0].height',
+        'stream[0].display_aspect_ratio',
+        'stream[0].avg_frame_rate',
+        'stream[0].pix_fmt'
+      ]) {
+        expect(metadata).to.have.nested.property(p)
+      }
+      expect(metadata).to.not.have.nested.property('format.filename')
+    }
+
+    for (const server of servers) {
+      const res = await getVideosList(server.url)
+
+      const video = res.body.data.find(v => v.name === videoAttributes.name)
+      const res2 = await getVideo(server.url, video.id)
+      const videoDetails = res2.body
+
+      const videoFiles = videoDetails.files
+      for (const [ index, file ] of videoFiles.entries()) {
+        expect(file.metadata).to.be.undefined
+        expect(file.metadataUrl).to.contain(servers[1].url)
+        expect(file.metadataUrl).to.contain(videoOnOrigin.uuid)
+
+        const res3 = await getVideoFileMetadataUrl(file.metadataUrl)
+        const metadata: FfprobeData = res3.body
+        expect(metadata).to.have.nested.property('format.size')
+        expect(metadata.format.size).to.equal(videoOnOriginDetails.files[index].metadata.format.size)
+      }
+    }
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })

shared/extra-utils/videos/videos.ts

@@ -95,6 +95,14 @@ function getVideo (url: string, id: number | string, expectedStatus = 200) {
           .expect(expectedStatus)
 }
 
+function getVideoFileMetadataUrl (url: string) {
+  return request(url)
+    .get('/')
+    .set('Accept', 'application/json')
+    .expect(200)
+    .expect('Content-Type', /json/)
+}
+
 function viewVideo (url: string, id: number | string, expectedStatus = 204, xForwardedFor?: string) {
   const path = '/api/v1/videos/' + id + '/views'
 
@@ -643,6 +651,7 @@ export {
   getAccountVideos,
   getVideoChannelVideos,
   getVideo,
+  getVideoFileMetadataUrl,
   getVideoWithToken,
   getVideosList,
   getVideosListPagination,

shared/models/activitypub/objects/common-objects.ts

@@ -28,6 +28,15 @@ export type ActivityPlaylistSegmentHashesObject = {
   href: string
 }
 
+export type ActivityVideoFileMetadataObject = {
+  type: 'Link'
+  rel: [ 'metadata', any ]
+  mediaType: 'application/json'
+  height: number
+  href: string
+  fps: number
+}
+
 export type ActivityPlaylistInfohashesObject = {
   type: 'Infohash'
   name: string
@@ -80,6 +89,7 @@ export type ActivityTagObject =
   | ActivityMentionObject
   | ActivityBitTorrentUrlObject
   | ActivityMagnetUrlObject
+  | ActivityVideoFileMetadataObject
 
 export type ActivityUrlObject =
   ActivityVideoUrlObject
@@ -87,6 +97,7 @@ export type ActivityUrlObject =
   | ActivityBitTorrentUrlObject
   | ActivityMagnetUrlObject
   | ActivityHtmlUrlObject
+  | ActivityVideoFileMetadataObject
 
 export interface ActivityPubAttributedTo {
   type: 'Group' | 'Person'

shared/models/videos/video-file-metadata.ts

@@ -0,0 +1,18 @@
+import { FfprobeData } from "fluent-ffmpeg"
+import { DeepOmit } from "@server/models/utils"
+
+export type VideoFileMetadataModel = DeepOmit<FfprobeData, 'filename'>
+
+export class VideoFileMetadata implements VideoFileMetadataModel {
+  streams: { [x: string]: any, [x: number]: any }[]
+  format: { [x: string]: any, [x: number]: any }
+  chapters: any[]
+
+  constructor (hash: Partial<VideoFileMetadataModel>) {
+    this.chapters = hash.chapters
+    this.format = hash.format
+    this.streams = hash.streams
+
+    delete this.format.filename
+  }
+}

shared/models/videos/video-file.model.ts

@@ -1,4 +1,5 @@
 import { VideoConstant, VideoResolution } from '@shared/models'
+import { FfprobeData } from 'fluent-ffmpeg'
 
 export interface VideoFile {
   magnetUri: string
@@ -9,4 +10,6 @@ export interface VideoFile {
   fileUrl: string
   fileDownloadUrl: string
   fps: number
+  metadata?: FfprobeData
+  metadataUrl?: string
 }