Optimize view endpoint

scripts/simulate-many-viewers.ts

@@ -46,10 +46,12 @@ async function prepare () {
     }
   }
 
+  const env = { PRODUCTION_CONSTANTS: 'true' }
+
   servers = await Promise.all([
-    createSingleServer(1, config, { nodeArgs: [ '--inspect' ] }),
+    createSingleServer(1, config, { env, nodeArgs: [ '--inspect' ] }),
-    createSingleServer(2, config),
+    createSingleServer(2, config, { env }),
-    createSingleServer(3, config)
+    createSingleServer(3, config, { env })
   ])
 
   await setAccessTokensToServers(servers)
@@ -81,7 +83,7 @@ async function runViewers () {
 
   await Bluebird.map(viewers, viewer => {
     return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor })
-  }, { concurrency: 100 })
+  }, { concurrency: 500 })
 
   console.log('Finished to run views in %d seconds.', (new Date().getTime() - before) / 1000)
 

server/controllers/api/videos/view.ts

@@ -26,7 +26,7 @@ export {
 // ---------------------------------------------------------------------------
 
 async function viewVideo (req: express.Request, res: express.Response) {
-  const video = res.locals.onlyVideo
+  const video = res.locals.onlyImmutableVideo
 
   const body = req.body as VideoView
 

server/initializers/constants.ts

@@ -734,12 +734,14 @@ const VIDEO_LIVE = {
 const MEMOIZE_TTL = {
   OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours
   INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours
+  VIDEO_DURATION: 1000 * 10, // 10 seconds
   LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute
   LIVE_CHECK_SOCKET_HEALTH: 1000 * 60 // 1 minute
 }
 
 const MEMOIZE_LENGTH = {
-  INFO_HASH_EXISTS: 200
+  INFO_HASH_EXISTS: 200,
+  VIDEO_DURATION: 200
 }
 
 const QUEUE_CONCURRENCY = {
@@ -812,7 +814,7 @@ const STATS_TIMESERIE = {
 // ---------------------------------------------------------------------------
 
 // Special constants for a test instance
-if (isTestInstance() === true) {
+if (isTestInstance() === true && process.env.PRODUCTION_CONSTANTS !== 'true') {
   PRIVATE_RSA_KEY_SIZE = 1024
 
   ACTOR_FOLLOW_SCORE.BASE = 20

server/lib/video.ts

@@ -1,6 +1,6 @@
 import { UploadFiles } from 'express'
 import { Transaction } from 'sequelize/types'
-import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
+import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants'
 import { TagModel } from '@server/models/video/tag'
 import { VideoModel } from '@server/models/video/video'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@@ -10,6 +10,7 @@ import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingP
 import { CreateJobOptions, JobQueue } from './job-queue/job-queue'
 import { updateVideoMiniatureFromExisting } from './thumbnail'
 import { CONFIG } from '@server/initializers/config'
+import memoizee from 'memoizee'
 
 function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
   return {
@@ -150,6 +151,24 @@ async function addMoveToObjectStorageJob (options: {
 
 // ---------------------------------------------------------------------------
 
+async function getVideoDuration (videoId: number | string) {
+  const video = await VideoModel.load(videoId)
+
+  const duration = video.isLive
+    ? undefined
+    : video.duration
+
+  return { duration, isLive: video.isLive }
+}
+
+const getCachedVideoDuration = memoizee(getVideoDuration, {
+  promise: true,
+  max: MEMOIZE_LENGTH.VIDEO_DURATION,
+  maxAge: MEMOIZE_TTL.VIDEO_DURATION
+})
+
+// ---------------------------------------------------------------------------
+
 export {
   buildLocalVideoFromReq,
   buildVideoThumbnailsFromReq,
@@ -157,5 +176,6 @@ export {
   addOptimizeOrMergeAudioJob,
   addTranscodingJob,
   addMoveToObjectStorageJob,
-  getTranscodingJobPriority
+  getTranscodingJobPriority,
+  getCachedVideoDuration
 }

server/lib/views/shared/video-viewer-counters.ts

@@ -5,7 +5,7 @@ import { sendView } from '@server/lib/activitypub/send/send-view'
 import { PeerTubeSocket } from '@server/lib/peertube-socket'
 import { getServerActor } from '@server/models/application/application'
 import { VideoModel } from '@server/models/video/video'
-import { MVideo } from '@server/types/models'
+import { MVideo, MVideoImmutable } from '@server/types/models'
 import { buildUUID, sha256 } from '@shared/extra-utils'
 
 const lTags = loggerTagsFactory('views')
@@ -33,7 +33,7 @@ export class VideoViewerCounters {
   // ---------------------------------------------------------------------------
 
   async addLocalViewer (options: {
-    video: MVideo
+    video: MVideoImmutable
     ip: string
   }) {
     const { video, ip } = options
@@ -86,7 +86,7 @@ export class VideoViewerCounters {
   // ---------------------------------------------------------------------------
 
   private async addViewerToVideo (options: {
-    video: MVideo
+    video: MVideoImmutable
     viewerId: string
     viewerExpires?: Date
   }) {
@@ -162,7 +162,7 @@ export class VideoViewerCounters {
     return sha256(this.salt + '-' + ip + '-' + videoUUID)
   }
 
-  private async federateViewerIfNeeded (video: MVideo, viewer: Viewer) {
+  private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) {
     // Federate the viewer if it's been a "long" time we did not
     const now = new Date().getTime()
     const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER / 2)

server/lib/views/shared/video-viewer-stats.ts

@@ -10,7 +10,7 @@ import { Redis } from '@server/lib/redis'
 import { VideoModel } from '@server/models/video/video'
 import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
 import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
-import { MVideo } from '@server/types/models'
+import { MVideo, MVideoImmutable } from '@server/types/models'
 import { VideoViewEvent } from '@shared/models'
 
 const lTags = loggerTagsFactory('views')
@@ -41,7 +41,7 @@ export class VideoViewerStats {
   // ---------------------------------------------------------------------------
 
   async addLocalViewer (options: {
-    video: MVideo
+    video: MVideoImmutable
     currentTime: number
     ip: string
     viewEvent?: VideoViewEvent
@@ -64,7 +64,7 @@ export class VideoViewerStats {
   // ---------------------------------------------------------------------------
 
   private async updateLocalViewerStats (options: {
-    video: MVideo
+    video: MVideoImmutable
     ip: string
     currentTime: number
     viewEvent?: VideoViewEvent

server/lib/views/shared/video-views.ts

@@ -1,7 +1,8 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { sendView } from '@server/lib/activitypub/send/send-view'
+import { getCachedVideoDuration } from '@server/lib/video'
 import { getServerActor } from '@server/models/application/application'
-import { MVideo } from '@server/types/models'
+import { MVideo, MVideoImmutable } from '@server/types/models'
 import { buildUUID } from '@shared/extra-utils'
 import { Redis } from '../../redis'
 
@@ -10,7 +11,7 @@ const lTags = loggerTagsFactory('views')
 export class VideoViews {
 
   async addLocalView (options: {
-    video: MVideo
+    video: MVideoImmutable
     ip: string
     watchTime: number
   }) {
@@ -18,7 +19,7 @@ export class VideoViews {
 
     logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) })
 
-    if (!this.hasEnoughWatchTime(video, watchTime)) return false
+    if (!await this.hasEnoughWatchTime(video, watchTime)) return false
 
     const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid)
     if (viewExists) return false
@@ -46,7 +47,7 @@ export class VideoViews {
 
   // ---------------------------------------------------------------------------
 
-  private async addView (video: MVideo) {
+  private async addView (video: MVideoImmutable) {
     const promises: Promise<any>[] = []
 
     if (video.isOwned()) {
@@ -58,10 +59,12 @@ export class VideoViews {
     await Promise.all(promises)
   }
 
-  private hasEnoughWatchTime (video: MVideo, watchTime: number) {
+  private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) {
-    if (video.isLive || video.duration >= 30) return watchTime >= 30
+    const { duration, isLive } = await getCachedVideoDuration(video.id)
+
+    if (isLive || duration >= 30) return watchTime >= 30
 
     // Check more than 50% of the video is watched
-    return video.duration / watchTime < 2
+    return duration / watchTime < 2
   }
 }

server/lib/views/video-views-manager.ts

@@ -1,5 +1,5 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { MVideo } from '@server/types/models'
+import { MVideo, MVideoImmutable } from '@server/types/models'
 import { VideoViewEvent } from '@shared/models'
 import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared'
 
@@ -41,7 +41,7 @@ export class VideoViewsManager {
   }
 
   async processLocalView (options: {
-    video: MVideo
+    video: MVideoImmutable
     currentTime: number
     ip: string | null
     viewEvent?: VideoViewEvent

server/middlewares/validators/videos/video-view.ts

@@ -6,6 +6,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
 import { exists, isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc'
 import { logger } from '../../../helpers/logger'
 import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
+import { getCachedVideoDuration } from '@server/lib/video'
 
 const getVideoLocalViewerValidator = [
   param('localViewerId')
@@ -42,20 +43,18 @@ const videoViewValidator = [
     logger.debug('Checking videoView parameters', { parameters: req.body })
 
     if (areValidationErrors(req, res)) return
-    if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return
+    if (!await doesVideoExist(req.params.videoId, res, 'only-immutable-attributes')) return
 
-    const video = res.locals.onlyVideo
+    const video = res.locals.onlyImmutableVideo
-    const videoDuration = video.isLive
+    const { duration } = await getCachedVideoDuration(video.id)
-      ? undefined
-      : video.duration
 
     if (!exists(req.body.currentTime)) { // TODO: remove in a few versions, introduced in 4.2
-      req.body.currentTime = Math.min(videoDuration ?? 0, 30)
+      req.body.currentTime = Math.min(duration ?? 0, 30)
     }
 
     const currentTime: number = req.body.currentTime
 
-    if (!isVideoTimeValid(currentTime, videoDuration)) {
+    if (!isVideoTimeValid(currentTime, duration)) {
       return res.fail({
         status: HttpStatusCode.BAD_REQUEST_400,
         message: 'Current time is invalid'