forks/peertube
1
0
Fork 0
Code Releases Activity

Support video views/viewers stats in server

Browse source 
* Add "currentTime" and "event" body params to view endpoint
 * Merge watching and view endpoints
 * Introduce WatchAction AP activity
 * Add tables to store viewer information of local videos
 * Add endpoints to fetch video views/viewers stats of local videos
 * Refactor views/viewers handlers
 * Support "views" and "viewers" counters for both VOD and live videos
This commit is contained in:
Chocobozzz 2022-03-24 13:36:47 +01:00 • committed by Chocobozzz
parent 69d48ee30c
commit b211106695
108 changed files with 2834 additions and 655 deletions
Download patch file Download diff file Expand all files Collapse all files

7
config/default.yaml
View file

@ -261,6 +261,13 @@ views:
ip_view_expiration: '1 hour'
# Used to get country location of views of local videos
geo_ip:
enabled: true
country:
database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb'
plugins:
# The website PeerTube will ask for available PeerTube plugins and themes
# This is an unmoderated plugin index, so only install plugins/themes you trust

7
config/production.yaml.example
View file

@ -257,6 +257,13 @@ views:
ip_view_expiration: '1 hour'
# Used to get country location of views of local videos
geo_ip:
enabled: true
country:
database_url: 'https://dbip.mirror.framasoft.org/files/dbip-country-lite-latest.mmdb'
plugins:
# The website PeerTube will ask for available PeerTube plugins and themes
# This is an unmoderated plugin index, so only install plugins/themes you trust

3
config/test.yaml
View file

@ -168,5 +168,8 @@ views:
local_buffer_update_interval: '5 seconds'
ip_view_expiration: '1 second'
geo_ip:
enabled: false
video_studio:
enabled: true

1
package.json
View file

@ -121,6 +121,7 @@
"magnet-uri": "^6.1.0",
"markdown-it": "^12.0.4",
"markdown-it-emoji": "^2.0.0",
"maxmind": "^4.3.6",
"memoizee": "^0.4.14",
"morgan": "^1.5.3",
"multer": "^1.4.4",

8
scripts/benchmark.ts
View file

@ -153,21 +153,23 @@ async function run () {
}
},
{
title: 'API - watching',
title: 'API - views with token',
method: 'PUT',
headers: {
...buildAuthorizationHeader(),
...buildJSONHeader()
},
body: JSON.stringify({ currentTime: 2 }),
path: '/api/v1/videos/' + video.uuid + '/watching',
path: '/api/v1/videos/' + video.uuid + '/views',
expecter: (body, status) => {
return status === 204
}
},
{
title: 'API - views',
title: 'API - views without token',
method: 'POST',
headers: buildJSONHeader(),
body: JSON.stringify({ currentTime: 2 }),
path: '/api/v1/videos/' + video.uuid + '/views',
expecter: (body, status) => {
return status === 204

3
scripts/ci.sh
View file

@ -84,8 +84,9 @@ elif [ "$1" = "api-3" ]; then
npm run build:server
videosFiles=$(findTestFiles ./dist/server/tests/api/videos)
viewsFiles=$(findTestFiles ./dist/server/tests/api/views)
MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $videosFiles
MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $viewsFiles $videosFiles
elif [ "$1" = "api-4" ]; then
npm run build:server

6
server.ts
View file

@ -112,6 +112,7 @@ import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-hi
import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler'
import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler'
import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
import { PeerTubeSocket } from './server/lib/peertube-socket'
import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@ -123,7 +124,7 @@ import { LiveManager } from './server/lib/live'
import { HttpStatusCode } from './shared/models/http/http-error-codes'
import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache'
import { ServerConfigManager } from '@server/lib/server-config-manager'
import { VideoViews } from '@server/lib/video-views'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { isTestInstance } from './server/helpers/core-utils'
// ----------- Command line -----------
@ -295,10 +296,11 @@ async function startApplication () {
AutoFollowIndexInstances.Instance.enable()
RemoveDanglingResumableUploadsScheduler.Instance.enable()
VideoViewsBufferScheduler.Instance.enable()
GeoIPUpdateScheduler.Instance.enable()
Redis.Instance.init()
PeerTubeSocket.Instance.init(server)
VideoViews.Instance.init()
VideoViewsManager.Instance.init()
updateStreamingPlaylistsInfohashesIfNeeded()
.catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))

14
server/controllers/activitypub/client.ts
View file

@ -27,7 +27,7 @@ import {
videosShareValidator
} from '../../middlewares'
import { cacheRoute } from '../../middlewares/cache/cache'
import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators'
import { getAccountVideoRateValidatorFactory, getVideoLocalViewerValidator, videoCommentGetValidator } from '../../middlewares/validators'
import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists'
import { AccountModel } from '../../models/account/account'
@ -175,6 +175,12 @@ activityPubClientRouter.get('/video-playlists/:playlistId/videos/:playlistElemen
videoPlaylistElementController
)
activityPubClientRouter.get('/videos/local-viewer/:localViewerId',
executeIfActivityPub,
asyncMiddleware(getVideoLocalViewerValidator),
getVideoLocalViewerController
)
// ---------------------------------------------------------------------------
export {
@ -399,6 +405,12 @@ function videoPlaylistElementController (req: express.Request, res: express.Resp
return activityPubResponse(activityPubContextify(json, 'Playlist'), res)
}
function getVideoLocalViewerController (req: express.Request, res: express.Response) {
const localViewer = res.locals.localViewerFull
return activityPubResponse(activityPubContextify(localViewer.toActivityPubObject(), 'WatchAction'), res)
}
// ---------------------------------------------------------------------------
function actorFollowing (req: express.Request, actor: MActorId) {

10
server/controllers/api/server/debug.ts
View file

@ -1,6 +1,8 @@
import express from 'express'
import { InboxManager } from '@server/lib/activitypub/inbox-manager'
import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { Debug, SendDebugCommand } from '@shared/models'
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { UserRight } from '../../../../shared/models/users'
@ -38,9 +40,13 @@ function getDebug (req: express.Request, res: express.Response) {
async function runCommand (req: express.Request, res: express.Response) {
const body: SendDebugCommand = req.body
if (body.command === 'remove-dandling-resumable-uploads') {
await RemoveDanglingResumableUploadsScheduler.Instance.execute()
const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
'process-video-viewers': () => VideoViewsManager.Instance.processViewers()
}
await processors[body.command]()
return res.status(HttpStatusCode.NO_CONTENT_204).end()
}

31
server/controllers/api/videos/index.ts
View file

@ -1,7 +1,6 @@
import express from 'express'
import { pickCommonVideoQuery } from '@server/helpers/query'
import { doJSONRequest } from '@server/helpers/requests'
import { VideoViews } from '@server/lib/video-views'
import { openapiOperationDoc } from '@server/middlewares/doc'
import { getServerActor } from '@server/models/application/application'
import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils'
@ -13,7 +12,6 @@ import { logger } from '../../../helpers/logger'
import { getFormattedObjects } from '../../../helpers/utils'
import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants'
import { sequelizeTypescript } from '../../../initializers/database'
import { sendView } from '../../../lib/activitypub/send/send-view'
import { JobQueue } from '../../../lib/job-queue'
import { Hooks } from '../../../lib/plugins/hooks'
import {
@ -35,28 +33,30 @@ import { VideoModel } from '../../../models/video/video'
import { blacklistRouter } from './blacklist'
import { videoCaptionsRouter } from './captions'
import { videoCommentRouter } from './comment'
import { studioRouter } from './studio'
import { filesRouter } from './files'
import { videoImportsRouter } from './import'
import { liveRouter } from './live'
import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { statsRouter } from './stats'
import { studioRouter } from './studio'
import { transcodingRouter } from './transcoding'
import { updateRouter } from './update'
import { uploadRouter } from './upload'
import { watchingRouter } from './watching'
import { viewRouter } from './view'
const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()
videosRouter.use('/', blacklistRouter)
videosRouter.use('/', statsRouter)
videosRouter.use('/', rateVideoRouter)
videosRouter.use('/', videoCommentRouter)
videosRouter.use('/', studioRouter)
videosRouter.use('/', videoCaptionsRouter)
videosRouter.use('/', videoImportsRouter)
videosRouter.use('/', ownershipVideoRouter)
videosRouter.use('/', watchingRouter)
videosRouter.use('/', viewRouter)
videosRouter.use('/', liveRouter)
videosRouter.use('/', uploadRouter)
videosRouter.use('/', updateRouter)
@ -103,11 +103,6 @@ videosRouter.get('/:id',
asyncMiddleware(checkVideoFollowConstraints),
getVideo
)
videosRouter.post('/:id/views',
openapiOperationDoc({ operationId: 'addView' }),
asyncMiddleware(videosCustomGetValidator('only-video')),
asyncMiddleware(viewVideo)
)
videosRouter.delete('/:id',
openapiOperationDoc({ operationId: 'delVideo' }),
@ -150,22 +145,6 @@ function getVideo (_req: express.Request, res: express.Response) {
return res.json(video.toFormattedDetailsJSON())
}
async function viewVideo (req: express.Request, res: express.Response) {
const video = res.locals.onlyVideo
const ip = req.ip
const success = await VideoViews.Instance.processView({ video, ip })
if (success) {
const serverActor = await getServerActor()
await sendView(serverActor, video, undefined)
Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res })
}
return res.status(HttpStatusCode.NO_CONTENT_204).end()
}
async function getVideoDescription (req: express.Request, res: express.Response) {
const videoInstance = res.locals.videoAll

66
server/controllers/api/videos/stats.ts Normal file
View file

@ -0,0 +1,66 @@
import express from 'express'
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
import { VideoStatsTimeserieMetric } from '@shared/models'
import {
asyncMiddleware,
authenticate,
videoOverallStatsValidator,
videoRetentionStatsValidator,
videoTimeserieStatsValidator
} from '../../../middlewares'
const statsRouter = express.Router()
statsRouter.get('/:videoId/stats/overall',
authenticate,
asyncMiddleware(videoOverallStatsValidator),
asyncMiddleware(getOverallStats)
)
statsRouter.get('/:videoId/stats/timeseries/:metric',
authenticate,
asyncMiddleware(videoTimeserieStatsValidator),
asyncMiddleware(getTimeserieStats)
)
statsRouter.get('/:videoId/stats/retention',
authenticate,
asyncMiddleware(videoRetentionStatsValidator),
asyncMiddleware(getRetentionStats)
)
// ---------------------------------------------------------------------------
export {
statsRouter
}
// ---------------------------------------------------------------------------
async function getOverallStats (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const stats = await LocalVideoViewerModel.getOverallStats(video)
return res.json(stats)
}
async function getRetentionStats (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const stats = await LocalVideoViewerModel.getRetentionStats(video)
return res.json(stats)
}
async function getTimeserieStats (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const metric = req.params.metric as VideoStatsTimeserieMetric
const stats = await LocalVideoViewerModel.getTimeserieStats({
video,
metric
})
return res.json(stats)
}

68
server/controllers/api/videos/view.ts Normal file
View file

@ -0,0 +1,68 @@
import express from 'express'
import { sendView } from '@server/lib/activitypub/send/send-view'
import { Hooks } from '@server/lib/plugins/hooks'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { getServerActor } from '@server/models/application/application'
import { MVideoId } from '@server/types/models'
import { HttpStatusCode, VideoView } from '@shared/models'
import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares'
import { UserVideoHistoryModel } from '../../../models/user/user-video-history'
const viewRouter = express.Router()
viewRouter.all(
[ '/:videoId/views', '/:videoId/watching' ],
openapiOperationDoc({ operationId: 'addView' }),
methodsValidator([ 'PUT', 'POST' ]),
optionalAuthenticate,
asyncMiddleware(videoViewValidator),
asyncMiddleware(viewVideo)
)
// ---------------------------------------------------------------------------
export {
viewRouter
}
// ---------------------------------------------------------------------------
async function viewVideo (req: express.Request, res: express.Response) {
const video = res.locals.onlyVideo
const body = req.body as VideoView
const ip = req.ip
const { successView, successViewer } = await VideoViewsManager.Instance.processLocalView({
video,
ip,
currentTime: body.currentTime,
viewEvent: body.viewEvent
})
if (successView) {
await sendView({ byActor: await getServerActor(), video, type: 'view' })
Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res })
}
if (successViewer) {
await sendView({ byActor: await getServerActor(), video, type: 'viewer' })
}
await updateUserHistoryIfNeeded(body, video, res)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
}
async function updateUserHistoryIfNeeded (body: VideoView, video: MVideoId, res: express.Response) {
const user = res.locals.oauth?.token.User
if (!user) return
if (user.videosHistoryEnabled !== true) return
await UserVideoHistoryModel.upsert({
videoId: video.id,
userId: user.id,
currentTime: body.currentTime
})
}

44
server/controllers/api/videos/watching.ts
View file

@ -1,44 +0,0 @@
import express from 'express'
import { HttpStatusCode, UserWatchingVideo } from '@shared/models'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
authenticate,
openapiOperationDoc,
videoWatchingValidator
} from '../../../middlewares'
import { UserVideoHistoryModel } from '../../../models/user/user-video-history'
const watchingRouter = express.Router()
watchingRouter.put('/:videoId/watching',
openapiOperationDoc({ operationId: 'setProgress' }),
authenticate,
asyncMiddleware(videoWatchingValidator),
asyncRetryTransactionMiddleware(userWatchVideo)
)
// ---------------------------------------------------------------------------
export {
watchingRouter
}
// ---------------------------------------------------------------------------
async function userWatchVideo (req: express.Request, res: express.Response) {
const user = res.locals.oauth.token.User
const body: UserWatchingVideo = req.body
const { id: videoId } = res.locals.videoId
await UserVideoHistoryModel.upsert({
videoId,
userId: user.id,
currentTime: body.currentTime
})
return res.type('json')
.status(HttpStatusCode.NO_CONTENT_204)
.end()
}

2
server/helpers/custom-validators/activitypub/activity.ts
View file

@ -8,6 +8,7 @@ import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './mis
import { isPlaylistObjectValid } from './playlist'
import { sanitizeAndCheckVideoCommentObject } from './video-comments'
import { sanitizeAndCheckVideoTorrentObject } from './videos'
import { isWatchActionObjectValid } from './watch-action'
function isRootActivityValid (activity: any) {
return isCollection(activity) || isActivity(activity)
@ -82,6 +83,7 @@ function isCreateActivityValid (activity: any) {
isDislikeActivityValid(activity.object) ||
isFlagActivityValid(activity.object) ||
isPlaylistObjectValid(activity.object) ||
isWatchActionObjectValid(activity.object) ||
isCacheFileObjectValid(activity.object) ||
sanitizeAndCheckVideoCommentObject(activity.object) ||

11
server/helpers/custom-validators/activitypub/misc.ts
View file

@ -57,10 +57,19 @@ function setValidAttributedTo (obj: any) {
return true
}
function isActivityPubVideoDurationValid (value: string) {
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
return exists(value) &&
typeof value === 'string' &&
value.startsWith('PT') &&
value.endsWith('S')
}
export {
isUrlValid,
isActivityPubUrlValid,
isBaseActivityValid,
setValidAttributedTo,
isObjectValid
isObjectValid,
isActivityPubVideoDurationValid
}

14
server/helpers/custom-validators/activitypub/videos.ts
View file

@ -4,7 +4,7 @@ import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@s
import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
import { peertubeTruncate } from '../../core-utils'
import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
import { isLiveLatencyModeValid } from '../video-lives'
import {
isVideoDurationValid,
@ -14,22 +14,13 @@ import {
isVideoTruncatedDescriptionValid,
isVideoViewsValid
} from '../videos'
import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc'
function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
return isBaseActivityValid(activity, 'Update') &&
sanitizeAndCheckVideoTorrentObject(activity.object)
}
function isActivityPubVideoDurationValid (value: string) {
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
return exists(value) &&
typeof value === 'string' &&
value.startsWith('PT') &&
value.endsWith('S') &&
isVideoDurationValid(value.replace(/[^0-9]+/g, ''))
}
function sanitizeAndCheckVideoTorrentObject (video: any) {
if (!video || video.type !== 'Video') return false
@ -71,6 +62,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
return isActivityPubUrlValid(video.id) &&
isVideoNameValid(video.name) &&
isActivityPubVideoDurationValid(video.duration) &&
isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) &&
isUUIDValid(video.uuid) &&
(!video.category || isRemoteNumberIdentifierValid(video.category)) &&
(!video.licence || isRemoteNumberIdentifierValid(video.licence)) &&

37
server/helpers/custom-validators/activitypub/watch-action.ts Normal file
View file

@ -0,0 +1,37 @@
import { WatchActionObject } from '@shared/models'
import { exists, isDateValid, isUUIDValid } from '../misc'
import { isVideoTimeValid } from '../video-view'
import { isActivityPubVideoDurationValid, isObjectValid } from './misc'
function isWatchActionObjectValid (action: WatchActionObject) {
return exists(action) &&
action.type === 'WatchAction' &&
isObjectValid(action.id) &&
isActivityPubVideoDurationValid(action.duration) &&
isDateValid(action.startTime) &&
isDateValid(action.endTime) &&
isLocationValid(action.location) &&
isUUIDValid(action.uuid) &&
isObjectValid(action.object) &&
isWatchSectionsValid(action.watchSections)
}
// ---------------------------------------------------------------------------
export {
isWatchActionObjectValid
}
// ---------------------------------------------------------------------------
function isLocationValid (location: any) {
if (!location) return true
return typeof location === 'object' && typeof location.addressCountry === 'string'
}
function isWatchSectionsValid (sections: WatchActionObject['watchSections']) {
return Array.isArray(sections) && sections.every(s => {
return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp)
})
}

16
server/helpers/custom-validators/video-stats.ts Normal file
View file

@ -0,0 +1,16 @@
import { VideoStatsTimeserieMetric } from '@shared/models'
const validMetrics = new Set<VideoStatsTimeserieMetric>([
'viewers',
'aggregateWatchTime'
])
function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) {
return validMetrics.has(value)
}
// ---------------------------------------------------------------------------
export {
isValidStatTimeserieMetric
}

12
server/helpers/custom-validators/video-view.ts Normal file
View file

@ -0,0 +1,12 @@
import { exists } from './misc'
function isVideoTimeValid (value: number, videoDuration?: number) {
if (value < 0) return false
if (exists(videoDuration) && value > videoDuration) return false
return true
}
export {
isVideoTimeValid
}

78
server/helpers/geo-ip.ts Normal file
View file

@ -0,0 +1,78 @@
import { pathExists, writeFile } from 'fs-extra'
import maxmind, { CountryResponse, Reader } from 'maxmind'
import { join } from 'path'
import { CONFIG } from '@server/initializers/config'
import { logger, loggerTagsFactory } from './logger'
import { isBinaryResponse, peertubeGot } from './requests'
const lTags = loggerTagsFactory('geo-ip')
const mmbdFilename = 'dbip-country-lite-latest.mmdb'
const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename)
export class GeoIP {
private static instance: GeoIP
private reader: Reader<CountryResponse>
private constructor () {
}
async safeCountryISOLookup (ip: string): Promise<string> {
if (CONFIG.GEO_IP.ENABLED === false) return null
await this.initReaderIfNeeded()
try {
const result = this.reader.get(ip)
if (!result) return null
return result.country.iso_code
} catch (err) {
logger.error('Cannot get country from IP.', { err })
return null
}
}
async updateDatabase () {
if (CONFIG.GEO_IP.ENABLED === false) return
const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL
logger.info('Updating GeoIP database from %s.', url, lTags())
const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' }
try {
const gotResult = await peertubeGot(url, gotOptions)
if (!isBinaryResponse(gotResult)) {
throw new Error('Not a binary response')
}
await writeFile(mmdbPath, gotResult.body)
// Reini reader
this.reader = undefined
logger.info('GeoIP database updated %s.', mmdbPath, lTags())
} catch (err) {
logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
}
}
private async initReaderIfNeeded () {
if (!this.reader) {
if (!await pathExists(mmdbPath)) {
await this.updateDatabase()
}
this.reader = await maxmind.open(mmdbPath)
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

1
server/initializers/checker-before-init.ts
View file

@ -44,6 +44,7 @@ function checkMissedConfig () {
'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration',
'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
'theme.default',
'geo_ip.enabled', 'geo_ip.country.database_url',
'remote_redundancy.videos.accept_from',
'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions',
'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url',

6
server/initializers/config.ts
View file

@ -215,6 +215,12 @@ const CONFIG = {
IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration'))
}
},
GEO_IP: {
ENABLED: config.get<boolean>('geo_ip.enabled'),
COUNTRY: {
DATABASE_URL: config.get<string>('geo_ip.country.database_url')
}
},
PLUGINS: {
INDEX: {
ENABLED: config.get<boolean>('plugins.index.enabled'),

17
server/initializers/constants.ts
View file

@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 700
const LAST_MIGRATION_VERSION = 705
// ---------------------------------------------------------------------------
@ -228,6 +228,7 @@ const SCHEDULER_INTERVALS_MS = {
REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
UPDATE_VIDEOS: 60000, // 1 minute
YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day
GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day
VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL,
CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL,
CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day
@ -366,9 +367,12 @@ const CONSTRAINTS_FIELDS = {
const VIEW_LIFETIME = {
VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION,
VIEWER: 60000 * 5 // 5 minutes
VIEWER: 60000 * 5, // 5 minutes
VIEWER_STATS: 60000 * 60 // 1 hour
}
const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 10
let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour
const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = {
@ -800,6 +804,12 @@ const SEARCH_INDEX = {
// ---------------------------------------------------------------------------
const STATS_TIMESERIE = {
MAX_DAYS: 30
}
// ---------------------------------------------------------------------------
// Special constants for a test instance
if (isTestInstance() === true) {
PRIVATE_RSA_KEY_SIZE = 1024
@ -836,6 +846,7 @@ if (isTestInstance() === true) {
REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
VIEW_LIFETIME.VIEWER = 1000 * 5 // 5 second
VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second
CONTACT_FORM_LIFETIME = 1000 // 1 second
JOB_ATTEMPTS['email'] = 1
@ -907,6 +918,7 @@ export {
LAST_MIGRATION_VERSION,
OAUTH_LIFETIME,
CUSTOM_HTML_TAG_COMMENTS,
STATS_TIMESERIE,
BROADCAST_CONCURRENCY,
AUDIT_LOG_FILENAME,
PAGINATION,
@ -949,6 +961,7 @@ export {
ABUSE_STATES,
LRU_CACHE,
REQUEST_TIMEOUTS,
MAX_LOCAL_VIEWER_WATCH_SECTIONS,
USER_PASSWORD_RESET_LIFETIME,
USER_PASSWORD_CREATE_LIFETIME,
MEMOIZE_TTL,

10
server/initializers/database.ts
View file

@ -1,10 +1,14 @@
import { QueryTypes, Transaction } from 'sequelize'
import { Sequelize as SequelizeTypescript } from 'sequelize-typescript'
import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
import { TrackerModel } from '@server/models/server/tracker'
import { VideoTrackerModel } from '@server/models/server/video-tracker'
import { UserModel } from '@server/models/user/user'
import { UserNotificationModel } from '@server/models/user/user-notification'
import { UserVideoHistoryModel } from '@server/models/user/user-video-history'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
import { isTestInstance } from '../helpers/core-utils'
import { logger } from '../helpers/logger'
import { AbuseModel } from '../models/abuse/abuse'
@ -42,10 +46,8 @@ import { VideoPlaylistElementModel } from '../models/video/video-playlist-elemen
import { VideoShareModel } from '../models/video/video-share'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
import { VideoTagModel } from '../models/video/video-tag'
import { VideoViewModel } from '../models/video/video-view'
import { VideoViewModel } from '../models/view/video-view'
import { CONFIG } from './config'
import { ActorCustomPageModel } from '@server/models/account/actor-custom-page'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
@ -140,6 +142,8 @@ async function initDatabaseModels (silent: boolean) {
VideoStreamingPlaylistModel,
VideoPlaylistModel,
VideoPlaylistElementModel,
LocalVideoViewerModel,
LocalVideoViewerWatchSectionModel,
ThumbnailModel,
TrackerModel,
VideoTrackerModel,

52
server/initializers/migrations/0705-local-video-viewers.ts Normal file
View file

@ -0,0 +1,52 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {
const { transaction } = utils
{
const query = `
CREATE TABLE IF NOT EXISTS "localVideoViewer" (
"id" serial,
"startDate" timestamp with time zone NOT NULL,
"endDate" timestamp with time zone NOT NULL,
"watchTime" integer NOT NULL,
"country" varchar(255),
"uuid" uuid NOT NULL,
"url" varchar(255) NOT NULL,
"videoId" integer NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" timestamp with time zone NOT NULL,
PRIMARY KEY ("id")
);
`
await utils.sequelize.query(query, { transaction })
}
{
const query = `
CREATE TABLE IF NOT EXISTS "localVideoViewerWatchSection" (
"id" serial,
"watchStart" integer NOT NULL,
"watchEnd" integer NOT NULL,
"localVideoViewerId" integer NOT NULL REFERENCES "localVideoViewer" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" timestamp with time zone NOT NULL,
PRIMARY KEY ("id")
);
`
await utils.sequelize.query(query, { transaction })
}
}
function down () {
throw new Error('Not implemented.')
}
export {
up,
down
}

15
server/lib/activitypub/activity.ts
View file

@ -4,6 +4,17 @@ function getAPId (object: string | { id: string }) {
return object.id
}
export {
getAPId
function getActivityStreamDuration (duration: number) {
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
return 'PT' + duration + 'S'
}
function getDurationFromActivityStream (duration: string) {
return parseInt(duration.replace(/[^\d]+/, ''))
}
export {
getAPId,
getActivityStreamDuration,
getDurationFromActivityStream
}

23
server/lib/activitypub/context.ts
View file

@ -15,7 +15,7 @@ export {
type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) }
const contextStore = {
const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = {
Video: buildContext({
Hashtag: 'as:Hashtag',
uuid: 'sc:identifier',
@ -109,7 +109,8 @@ const contextStore = {
stopTimestamp: {
'@type': 'sc:Number',
'@id': 'pt:stopTimestamp'
}
},
uuid: 'sc:identifier'
}),
CacheFile: buildContext({
@ -128,6 +129,24 @@ const contextStore = {
}
}),
WatchAction: buildContext({
WatchAction: 'sc:WatchAction',
startTimestamp: {
'@type': 'sc:Number',
'@id': 'pt:startTimestamp'
},
stopTimestamp: {
'@type': 'sc:Number',
'@id': 'pt:stopTimestamp'
},
watchSection: {
'@type': 'sc:Number',
'@id': 'pt:stopTimestamp'
},
uuid: 'sc:identifier'
}),
Collection: buildContext(),
Follow: buildContext(),
Reject: buildContext(),
Accept: buildContext(),

42
server/lib/activitypub/local-video-viewer.ts Normal file
View file

@ -0,0 +1,42 @@
import { Transaction } from 'sequelize'
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
import { MVideo } from '@server/types/models'
import { WatchActionObject } from '@shared/models'
import { getDurationFromActivityStream } from './activity'
async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) {
const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id)
if (stats) await stats.destroy({ transaction: t })
const localVideoViewer = await LocalVideoViewerModel.create({
url: watchAction.id,
uuid: watchAction.uuid,
watchTime: getDurationFromActivityStream(watchAction.duration),
startDate: new Date(watchAction.startTime),
endDate: new Date(watchAction.endTime),
country: watchAction.location
? watchAction.location.addressCountry
: null,
videoId: video.id
})
await LocalVideoViewerWatchSectionModel.bulkCreateSections({
localVideoViewerId: localVideoViewer.id,
watchSections: watchAction.watchSections.map(s => ({
start: s.startTimestamp,
end: s.endTimestamp
}))
})
}
// ---------------------------------------------------------------------------
export {
createOrUpdateLocalVideoViewer
}

21
server/lib/activitypub/process/process-create.ts
View file

@ -1,6 +1,7 @@
import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
import { isRedundancyAccepted } from '@server/lib/redundancy'
import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject } from '@shared/models'
import { VideoModel } from '@server/models/video/video'
import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject, WatchActionObject } from '@shared/models'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { logger } from '../../../helpers/logger'
import { sequelizeTypescript } from '../../../initializers/database'
@ -8,6 +9,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models'
import { Notifier } from '../../notifier'
import { createOrUpdateCacheFile } from '../cache-file'
import { createOrUpdateLocalVideoViewer } from '../local-video-viewer'
import { createOrUpdateVideoPlaylist } from '../playlists'
import { forwardVideoRelatedActivity } from '../send/shared/send-utils'
import { resolveThread } from '../video-comments'
@ -32,6 +34,10 @@ async function processCreateActivity (options: APProcessorOptions<ActivityCreate
return retryTransactionWrapper(processCreateVideoComment, activity, byActor, notify)
}
if (activityType === 'WatchAction') {
return retryTransactionWrapper(processCreateWatchAction, activity)
}
if (activityType === 'CacheFile') {
return retryTransactionWrapper(processCreateCacheFile, activity, byActor)
}
@ -81,6 +87,19 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor
}
}
async function processCreateWatchAction (activity: ActivityCreate) {
const watchAction = activity.object as WatchActionObject
if (watchAction.actionStatus !== 'CompletedActionStatus') return
const video = await VideoModel.loadByUrl(watchAction.object)
if (video.remote) return
await sequelizeTypescript.transaction(async t => {
return createOrUpdateLocalVideoViewer(watchAction, video, t)
})
}
async function processCreateVideoComment (activity: ActivityCreate, byActor: MActorSignature, notify: boolean) {
const commentObject = activity.object as VideoCommentObject
const byAccount = byActor.Account

4
server/lib/activitypub/process/process-view.ts
View file

@ -1,4 +1,4 @@
import { VideoViews } from '@server/lib/video-views'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { ActivityView } from '../../../../shared/models/activitypub'
import { APProcessorOptions } from '../../../types/activitypub-processor.model'
import { MActorSignature } from '../../../types/models'
@ -32,7 +32,7 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu
? new Date(activity.expires)
: undefined
await VideoViews.Instance.processView({ video, ip: null, viewerExpires })
await VideoViewsManager.Instance.processRemoteView({ video, viewerExpires })
if (video.isOwned()) {
// Forward the view but don't resend the activity to the sender

17
server/lib/activitypub/send/send-create.ts
View file

@ -6,6 +6,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment'
import {
MActorLight,
MCommentOwnerVideo,
MLocalVideoViewerWithWatchSections,
MVideoAccountLight,
MVideoAP,
MVideoPlaylistFull,
@ -19,6 +20,7 @@ import {
getActorsInvolvedInVideo,
getAudienceFromFollowersOf,
getVideoCommentAudience,
sendVideoActivityToOrigin,
sendVideoRelatedActivity,
unicastTo
} from './shared'
@ -61,6 +63,18 @@ async function sendCreateCacheFile (
})
}
async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) {
logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid))
const byActor = await getServerActor()
const activityBuilder = (audience: ActivityAudience) => {
return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience)
}
return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' })
}
async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) {
if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
@ -175,7 +189,8 @@ export {
buildCreateActivity,
sendCreateVideoComment,
sendCreateVideoPlaylist,
sendCreateCacheFile
sendCreateCacheFile,
sendCreateWatchAction
}
// ---------------------------------------------------------------------------

63
server/lib/activitypub/send/send-view.ts
View file

@ -1,38 +1,31 @@
import { Transaction } from 'sequelize'
import { VideoViews } from '@server/lib/video-views'
import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models'
import { ActivityAudience, ActivityView } from '@shared/models'
import { logger } from '../../../helpers/logger'
import { ActorModel } from '../../../models/actor/actor'
import { audiencify, getAudience } from '../audience'
import { getLocalVideoViewActivityPubUrl } from '../url'
import { sendVideoRelatedActivity } from './shared/send-utils'
async function sendView (byActor: ActorModel, video: MVideoImmutable, t: Transaction) {
logger.info('Creating job to send view of %s.', video.url)
type ViewType = 'view' | 'viewer'
async function sendView (options: {
byActor: MActorLight
type: ViewType
video: MVideoImmutable
transaction?: Transaction
}) {
const { byActor, type, video, transaction } = options
logger.info('Creating job to send %s of %s.', type, video.url)
const activityBuilder = (audience: ActivityAudience) => {
const url = getLocalVideoViewActivityPubUrl(byActor, video)
return buildViewActivity(url, byActor, video, audience)
return buildViewActivity({ url, byActor, video, audience, type })
}
return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t, contextType: 'View' })
}
function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityView {
if (!audience) audience = getAudience(byActor)
return audiencify(
{
id: url,
type: 'View' as 'View',
actor: byActor.url,
object: video.url,
expires: new Date(VideoViews.Instance.buildViewerExpireTime()).toISOString()
},
audience
)
return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View' })
}
// ---------------------------------------------------------------------------
@ -40,3 +33,29 @@ function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoU
export {
sendView
}
// ---------------------------------------------------------------------------
function buildViewActivity (options: {
url: string
byActor: MActorAudience
video: MVideoUrl
type: ViewType
audience?: ActivityAudience
}): ActivityView {
const { url, byActor, type, video, audience = getAudience(byActor) } = options
return audiencify(
{
id: url,
type: 'View' as 'View',
actor: byActor.url,
object: video.url,
expires: type === 'viewer'
? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString()
: undefined
},
audience
)
}

6
server/lib/activitypub/url.ts
View file

@ -7,6 +7,7 @@ import {
MActorId,
MActorUrl,
MCommentId,
MLocalVideoViewer,
MVideoId,
MVideoPlaylistElement,
MVideoUrl,
@ -59,6 +60,10 @@ function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId) {
return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString()
}
function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) {
return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid
}
function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) {
return byActor.url + '/likes/' + video.id
}
@ -167,6 +172,7 @@ export {
getLocalVideoCommentsActivityPubUrl,
getLocalVideoLikesActivityPubUrl,
getLocalVideoDislikesActivityPubUrl,
getLocalVideoViewerActivityPubUrl,
getAbuseTargetUrl,
checkUrlsSameHost,

4
server/lib/activitypub/videos/shared/object-to-model-attributes.ts
View file

@ -24,6 +24,7 @@ import {
VideoPrivacy,
VideoStreamingPlaylistType
} from '@shared/models'
import { getDurationFromActivityStream } from '../../activity'
function getThumbnailFromIcons (videoObject: VideoObject) {
let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@ -170,7 +171,6 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi
? VideoPrivacy.PUBLIC
: VideoPrivacy.UNLISTED
const duration = videoObject.duration.replace(/[^\d]+/, '')
const language = videoObject.language?.identifier
const category = videoObject.category
@ -200,7 +200,7 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi
isLive: videoObject.isLiveBroadcast,
state: videoObject.state,
channelId: videoChannel.id,
duration: parseInt(duration, 10),
duration: getDurationFromActivityStream(videoObject.duration),
createdAt: new Date(videoObject.published),
publishedAt: new Date(videoObject.published),

2
server/lib/client-html.ts
View file

@ -23,11 +23,11 @@ import {
WEBSERVER
} from '../initializers/constants'
import { AccountModel } from '../models/account/account'
import { getActivityStreamDuration } from '../models/video/formatter/video-format-utils'
import { VideoModel } from '../models/video/video'
import { VideoChannelModel } from '../models/video/video-channel'
import { VideoPlaylistModel } from '../models/video/video-playlist'
import { MAccountActor, MChannelActor } from '../types/models'
import { getActivityStreamDuration } from './activitypub/activity'
import { getBiggestActorImage } from './actor-image'
import { ServerConfigManager } from './server-config-manager'

2
server/lib/job-queue/handlers/video-views-stats.ts
View file

@ -1,7 +1,7 @@
import { VideoViewModel } from '@server/models/view/video-view'
import { isTestInstance } from '../../../helpers/core-utils'
import { logger } from '../../../helpers/logger'
import { VideoModel } from '../../../models/video/video'
import { VideoViewModel } from '../../../models/video/video-view'
import { Redis } from '../../redis'
async function processVideosViewsStats () {

68
server/lib/redis.ts
View file

@ -249,6 +249,45 @@ class Redis {
])
}
/* ************ Video viewers stats ************ */
getLocalVideoViewer (options: {
key?: string
// Or
ip?: string
videoId?: number
}) {
if (options.key) return this.getObject(options.key)
const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId)
return this.getObject(viewerKey)
}
setLocalVideoViewer (ip: string, videoId: number, object: any) {
const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId)
return Promise.all([
this.addToSet(setKey, viewerKey),
this.setObject(viewerKey, object)
])
}
listLocalVideoViewerKeys () {
const { setKey } = this.generateLocalVideoViewerKeys()
return this.getSet(setKey)
}
deleteLocalVideoViewersKeys (key: string) {
const { setKey } = this.generateLocalVideoViewerKeys()
return Promise.all([
this.deleteFromSet(setKey, key),
this.deleteKey(key)
])
}
/* ************ Resumable uploads final responses ************ */
setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) {
@ -290,10 +329,18 @@ class Redis {
/* ************ Keys generation ************ */
private generateLocalVideoViewsKeys (videoId?: Number) {
private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string }
private generateLocalVideoViewsKeys (): { setKey: string }
private generateLocalVideoViewsKeys (videoId?: number) {
return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` }
}
private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string }
private generateLocalVideoViewerKeys (): { setKey: string }
private generateLocalVideoViewerKeys (ip?: string, videoId?: number) {
return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` }
}
private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) {
const hour = exists(options.hour)
? options.hour
@ -352,8 +399,23 @@ class Redis {
return this.client.del(this.prefix + key)
}
private async setValue (key: string, value: string, expirationMilliseconds: number) {
const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds })
private async getObject (key: string) {
const value = await this.getValue(key)
if (!value) return null
return JSON.parse(value)
}
private setObject (key: string, value: { [ id: string ]: number | string }) {
return this.setValue(key, JSON.stringify(value))
}
private async setValue (key: string, value: string, expirationMilliseconds?: number) {
const options = expirationMilliseconds
? { PX: expirationMilliseconds }
: {}
const result = await this.client.set(this.prefix + key, value, options)
if (result !== 'OK') throw new Error('Redis set result is not OK.')
}

22
server/lib/schedulers/geo-ip-update-scheduler.ts Normal file
View file

@ -0,0 +1,22 @@
import { GeoIP } from '@server/helpers/geo-ip'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { AbstractScheduler } from './abstract-scheduler'
export class GeoIPUpdateScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE
private constructor () {
super()
}
protected internalExecute () {
return GeoIP.Instance.updateDatabase()
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

6
server/lib/schedulers/remove-old-views-scheduler.ts
View file

@ -1,8 +1,8 @@
import { VideoViewModel } from '@server/models/view/video-view'
import { logger } from '../../helpers/logger'
import { AbstractScheduler } from './abstract-scheduler'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { CONFIG } from '../../initializers/config'
import { VideoViewModel } from '../../models/video/video-view'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { AbstractScheduler } from './abstract-scheduler'
export class RemoveOldViewsScheduler extends AbstractScheduler {

4
server/lib/schedulers/video-views-buffer-scheduler.ts
View file

@ -21,8 +21,6 @@ export class VideoViewsBufferScheduler extends AbstractScheduler {
const videoIds = await Redis.Instance.listLocalVideosViewed()
if (videoIds.length === 0) return
logger.info('Processing local video views buffer.', { videoIds, ...lTags() })
for (const videoId of videoIds) {
try {
const views = await Redis.Instance.getLocalVideoViews(videoId)
@ -34,6 +32,8 @@ export class VideoViewsBufferScheduler extends AbstractScheduler {
continue
}
logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid))
// If this is a remote video, the origin instance will send us an update
await VideoModel.incrementViews(videoId, views)

131
server/lib/video-views.ts
View file

@ -1,131 +0,0 @@
import { isTestInstance } from '@server/helpers/core-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { VIEW_LIFETIME } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
import { MVideo } from '@server/types/models'
import { PeerTubeSocket } from './peertube-socket'
import { Redis } from './redis'
const lTags = loggerTagsFactory('views')
export class VideoViews {
// Values are Date().getTime()
private readonly viewersPerVideo = new Map<number, number[]>()
private static instance: VideoViews
private constructor () {
}
init () {
setInterval(() => this.cleanViewers(), VIEW_LIFETIME.VIEWER)
}
async processView (options: {
video: MVideo
ip: string | null
viewerExpires?: Date
}) {
const { video, ip, viewerExpires } = options
logger.debug('Processing view for %s and ip %s.', video.url, ip, lTags())
let success = await this.addView(video, ip)
if (video.isLive) {
const successViewer = await this.addViewer(video, ip, viewerExpires)
success ||= successViewer
}
return success
}
getViewers (video: MVideo) {
const viewers = this.viewersPerVideo.get(video.id)
if (!viewers) return 0
return viewers.length
}
buildViewerExpireTime () {
return new Date().getTime() + VIEW_LIFETIME.VIEWER
}
private async addView (video: MVideo, ip: string | null) {
const promises: Promise<any>[] = []
if (ip !== null) {
const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid)
if (viewExists) return false
promises.push(Redis.Instance.setIPVideoView(ip, video.uuid))
}
if (video.isOwned()) {
promises.push(Redis.Instance.addLocalVideoView(video.id))
}
promises.push(Redis.Instance.addVideoViewStats(video.id))
await Promise.all(promises)
return true
}
private async addViewer (video: MVideo, ip: string | null, viewerExpires?: Date) {
if (ip !== null) {
const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid)
if (viewExists) return false
await Redis.Instance.setIPVideoViewer(ip, video.uuid)
}
let watchers = this.viewersPerVideo.get(video.id)
if (!watchers) {
watchers = []
this.viewersPerVideo.set(video.id, watchers)
}
const expiration = viewerExpires
? viewerExpires.getTime()
: this.buildViewerExpireTime()
watchers.push(expiration)
await this.notifyClients(video.id, watchers.length)
return true
}
private async cleanViewers () {
if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags())
for (const videoId of this.viewersPerVideo.keys()) {
const notBefore = new Date().getTime()
const viewers = this.viewersPerVideo.get(videoId)
// Only keep not expired viewers
const newViewers = viewers.filter(w => w > notBefore)
if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
else this.viewersPerVideo.set(videoId, newViewers)
await this.notifyClients(videoId, newViewers.length)
}
}
private async notifyClients (videoId: string | number, viewersLength: number) {
const video = await VideoModel.loadImmutableAttributes(videoId)
if (!video) return
PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
logger.debug('Live video views update for %s is %d.', video.url, viewersLength, lTags())
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

2
server/lib/views/shared/index.ts Normal file
View file

@ -0,0 +1,2 @@
export * from './video-viewers'
export * from './video-views'

276
server/lib/views/shared/video-viewers.ts Normal file
View file

@ -0,0 +1,276 @@
import { Transaction } from 'sequelize/types'
import { isTestInstance } from '@server/helpers/core-utils'
import { GeoIP } from '@server/helpers/geo-ip'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants'
import { sequelizeTypescript } from '@server/initializers/database'
import { sendCreateWatchAction } from '@server/lib/activitypub/send'
import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url'
import { PeerTubeSocket } from '@server/lib/peertube-socket'
import { Redis } from '@server/lib/redis'
import { VideoModel } from '@server/models/video/video'
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
import { MVideo } from '@server/types/models'
import { VideoViewEvent } from '@shared/models'
const lTags = loggerTagsFactory('views')
type LocalViewerStats = {
firstUpdated: number // Date.getTime()
lastUpdated: number // Date.getTime()
watchSections: {
start: number
end: number
}[]
watchTime: number
country: string
videoId: number
}
export class VideoViewers {
// Values are Date().getTime()
private readonly viewersPerVideo = new Map<number, number[]>()
private processingViewerCounters = false
private processingViewerStats = false
constructor () {
setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER)
setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS)
}
// ---------------------------------------------------------------------------
getViewers (video: MVideo) {
const viewers = this.viewersPerVideo.get(video.id)
if (!viewers) return 0
return viewers.length
}
buildViewerExpireTime () {
return new Date().getTime() + VIEW_LIFETIME.VIEWER
}
async getWatchTime (videoId: number, ip: string) {
const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId })
return stats?.watchTime || 0
}
async addLocalViewer (options: {
video: MVideo
currentTime: number
ip: string
viewEvent?: VideoViewEvent
}) {
const { video, ip, viewEvent, currentTime } = options
logger.debug('Adding local viewer to video %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) })
await this.updateLocalViewerStats({ video, viewEvent, currentTime, ip })
const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid)
if (viewExists) return false
await Redis.Instance.setIPVideoViewer(ip, video.uuid)
return this.addViewerToVideo({ video })
}
async addRemoteViewer (options: {
video: MVideo
viewerExpires: Date
}) {
const { video, viewerExpires } = options
logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
return this.addViewerToVideo({ video, viewerExpires })
}
private async addViewerToVideo (options: {
video: MVideo
viewerExpires?: Date
}) {
const { video, viewerExpires } = options
let watchers = this.viewersPerVideo.get(video.id)
if (!watchers) {
watchers = []
this.viewersPerVideo.set(video.id, watchers)
}
const expiration = viewerExpires
? viewerExpires.getTime()
: this.buildViewerExpireTime()
watchers.push(expiration)
await this.notifyClients(video.id, watchers.length)
return true
}
private async updateLocalViewerStats (options: {
video: MVideo
ip: string
currentTime: number
viewEvent?: VideoViewEvent
}) {
const { video, ip, viewEvent, currentTime } = options
const nowMs = new Date().getTime()
let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id })
if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) {
logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) })
return
}
if (!stats) {
const country = await GeoIP.Instance.safeCountryISOLookup(ip)
stats = {
firstUpdated: nowMs,
lastUpdated: nowMs,
watchSections: [],
watchTime: 0,
country,
videoId: video.id
}
}
stats.lastUpdated = nowMs
if (viewEvent === 'seek' || stats.watchSections.length === 0) {
stats.watchSections.push({
start: currentTime,
end: currentTime
})
} else {
const lastSection = stats.watchSections[stats.watchSections.length - 1]
lastSection.end = currentTime
}
stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections)
logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) })
await Redis.Instance.setLocalVideoViewer(ip, video.id, stats)
}
private async cleanViewerCounters () {
if (this.processingViewerCounters) return
this.processingViewerCounters = true
if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags())
try {
for (const videoId of this.viewersPerVideo.keys()) {
const notBefore = new Date().getTime()
const viewers = this.viewersPerVideo.get(videoId)
// Only keep not expired viewers
const newViewers = viewers.filter(w => w > notBefore)
if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
else this.viewersPerVideo.set(videoId, newViewers)
await this.notifyClients(videoId, newViewers.length)
}
} catch (err) {
logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
}
this.processingViewerCounters = false
}
private async notifyClients (videoId: string | number, viewersLength: number) {
const video = await VideoModel.loadImmutableAttributes(videoId)
if (!video) return
PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags())
}
async processViewerStats () {
if (this.processingViewerStats) return
this.processingViewerStats = true
if (!isTestInstance()) logger.info('Processing viewers.', lTags())
const now = new Date().getTime()
try {
const allKeys = await Redis.Instance.listLocalVideoViewerKeys()
for (const key of allKeys) {
const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key })
if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) {
continue
}
try {
await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.load(stats.videoId, t)
const statsModel = await this.saveViewerStats(video, stats, t)
if (video.remote) {
await sendCreateWatchAction(statsModel, t)
}
})
await Redis.Instance.deleteLocalVideoViewersKeys(key)
} catch (err) {
logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() })
}
}
} catch (err) {
logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() })
}
this.processingViewerStats = false
}
private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) {
const statsModel = new LocalVideoViewerModel({
startDate: new Date(stats.firstUpdated),
endDate: new Date(stats.lastUpdated),
watchTime: stats.watchTime,
country: stats.country,
videoId: video.id
})
statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel)
statsModel.Video = video as VideoModel
await statsModel.save({ transaction })
statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({
localVideoViewerId: statsModel.id,
watchSections: stats.watchSections,
transaction
})
return statsModel
}
private buildWatchTimeFromSections (sections: { start: number, end: number }[]) {
return sections.reduce((p, current) => p + (current.end - current.start), 0)
}
}

60
server/lib/views/shared/video-views.ts Normal file
View file

@ -0,0 +1,60 @@
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { MVideo } from '@server/types/models'
import { Redis } from '../../redis'
const lTags = loggerTagsFactory('views')
export class VideoViews {
async addLocalView (options: {
video: MVideo
ip: string
watchTime: number
}) {
const { video, ip, watchTime } = options
logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) })
if (!this.hasEnoughWatchTime(video, watchTime)) return false
const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid)
if (viewExists) return false
await Redis.Instance.setIPVideoView(ip, video.uuid)
await this.addView(video)
return true
}
async addRemoteView (options: {
video: MVideo
}) {
const { video } = options
logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) })
await this.addView(video)
return true
}
private async addView (video: MVideo) {
const promises: Promise<any>[] = []
if (video.isOwned()) {
promises.push(Redis.Instance.addLocalVideoView(video.id))
}
promises.push(Redis.Instance.addVideoViewStats(video.id))
await Promise.all(promises)
}
private hasEnoughWatchTime (video: MVideo, watchTime: number) {
if (video.isLive || video.duration >= 30) return watchTime >= 30
// Check more than 50% of the video is watched
return video.duration / watchTime < 2
}
}

70
server/lib/views/video-views-manager.ts Normal file
View file

@ -0,0 +1,70 @@
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { MVideo } from '@server/types/models'
import { VideoViewEvent } from '@shared/models'
import { VideoViewers, VideoViews } from './shared'
const lTags = loggerTagsFactory('views')
export class VideoViewsManager {
private static instance: VideoViewsManager
private videoViewers: VideoViewers
private videoViews: VideoViews
private constructor () {
}
init () {
this.videoViewers = new VideoViewers()
this.videoViews = new VideoViews()
}
async processLocalView (options: {
video: MVideo
currentTime: number
ip: string | null
viewEvent?: VideoViewEvent
}) {
const { video, ip, viewEvent, currentTime } = options
logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags())
const successViewer = await this.videoViewers.addLocalViewer({ video, ip, viewEvent, currentTime })
// Do it after added local viewer to fetch updated information
const watchTime = await this.videoViewers.getWatchTime(video.id, ip)
const successView = await this.videoViews.addLocalView({ video, watchTime, ip })
return { successView, successViewer }
}
async processRemoteView (options: {
video: MVideo
viewerExpires?: Date
}) {
const { video, viewerExpires } = options
logger.debug('Processing remote view for %s.', video.url, { viewerExpires, ...lTags() })
if (viewerExpires) await this.videoViewers.addRemoteViewer({ video, viewerExpires })
else await this.videoViews.addRemoteView({ video })
}
getViewers (video: MVideo) {
return this.videoViewers.getViewers(video)
}
buildViewerExpireTime () {
return this.videoViewers.buildViewerExpireTime()
}
processViewers () {
return this.videoViewers.processViewerStats()
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

4
server/middlewares/cache/shared/api-cache.ts vendored
View file

@ -6,8 +6,8 @@ import { OutgoingHttpHeaders } from 'http'
import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils'
import { logger } from '@server/helpers/logger'
import { Redis } from '@server/lib/redis'
import { HttpStatusCode } from '@shared/models'
import { asyncMiddleware } from '@server/middlewares'
import { HttpStatusCode } from '@shared/models'
export interface APICacheOptions {
headerBlacklist?: string[]
@ -152,7 +152,7 @@ export class ApiCache {
end: res.end,
cacheable: true,
content: undefined,
headers: {}
headers: undefined
}
// Patch express

15
server/middlewares/validators/express.ts Normal file
View file

@ -0,0 +1,15 @@
import * as express from 'express'
const methodsValidator = (methods: string[]) => {
return (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (methods.includes(req.method) !== true) {
return res.sendStatus(405)
}
return next()
}
}
export {
methodsValidator
}

25
server/middlewares/validators/index.ts
View file

@ -1,17 +1,26 @@
export * from './activitypub'
export * from './videos'
export * from './abuse'
export * from './account'
export * from './actor-image'
export * from './blocklist'
export * from './oembed'
export * from './activitypub'
export * from './pagination'
export * from './follows'
export * from './bulk'
export * from './config'
export * from './express'
export * from './feeds'
export * from './sort'
export * from './users'
export * from './user-subscriptions'
export * from './videos'
export * from './follows'
export * from './jobs'
export * from './logs'
export * from './oembed'
export * from './pagination'
export * from './plugins'
export * from './redundancy'
export * from './search'
export * from './server'
export * from './sort'
export * from './themes'
export * from './user-history'
export * from './user-notifications'
export * from './user-subscriptions'
export * from './users'
export * from './webfinger'

3
server/middlewares/validators/videos/index.ts
View file

@ -6,9 +6,10 @@ export * from './video-files'
export * from './video-imports'
export * from './video-live'
export * from './video-ownership-changes'
export * from './video-watch'
export * from './video-view'
export * from './video-rates'
export * from './video-shares'
export * from './video-stats'
export * from './video-studio'
export * from './video-transcoding'
export * from './videos'

73
server/middlewares/validators/videos/video-stats.ts Normal file
View file

@ -0,0 +1,73 @@
import express from 'express'
import { param } from 'express-validator'
import { isValidStatTimeserieMetric } from '@server/helpers/custom-validators/video-stats'
import { HttpStatusCode, UserRight } from '@shared/models'
import { logger } from '../../../helpers/logger'
import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared'
const videoOverallStatsValidator = [
isValidVideoIdParam('videoId'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoOverallStatsValidator parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!await commonStatsCheck(req, res)) return
return next()
}
]
const videoRetentionStatsValidator = [
isValidVideoIdParam('videoId'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoRetentionStatsValidator parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!await commonStatsCheck(req, res)) return
if (res.locals.videoAll.isLive) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Cannot get retention stats of live video'
})
}
return next()
}
]
const videoTimeserieStatsValidator = [
isValidVideoIdParam('videoId'),
param('metric')
.custom(isValidStatTimeserieMetric)
.withMessage('Should have a valid timeserie metric'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoTimeserieStatsValidator parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!await commonStatsCheck(req, res)) return
return next()
}
]
// ---------------------------------------------------------------------------
export {
videoOverallStatsValidator,
videoTimeserieStatsValidator,
videoRetentionStatsValidator
}
// ---------------------------------------------------------------------------
async function commonStatsCheck (req: express.Request, res: express.Response) {
if (!await doesVideoExist(req.params.videoId, res, 'all')) return false
if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return false
return true
}

74
server/middlewares/validators/videos/video-view.ts Normal file
View file

@ -0,0 +1,74 @@
import express from 'express'
import { body, param } from 'express-validator'
import { isVideoTimeValid } from '@server/helpers/custom-validators/video-view'
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { exists, isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc'
import { logger } from '../../../helpers/logger'
import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
const getVideoLocalViewerValidator = [
param('localViewerId')
.custom(isIdValid).withMessage('Should have a valid local viewer id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking getVideoLocalViewerValidator parameters', { parameters: req.params })
if (areValidationErrors(req, res)) return
const localViewer = await LocalVideoViewerModel.loadFullById(+req.params.localViewerId)
if (!localViewer) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
message: 'Local viewer not found'
})
}
res.locals.localViewerFull = localViewer
return next()
}
]
const videoViewValidator = [
isValidVideoIdParam('videoId'),
body('currentTime')
.optional() // TODO: remove optional in a few versions, introduced in 4.2
.customSanitizer(toIntOrNull)
.custom(isIntOrNull).withMessage('Should have correct current time'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoView parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return
const video = res.locals.onlyVideo
const videoDuration = video.isLive
? undefined
: video.duration
if (!exists(req.body.currentTime)) { // TODO: remove in a few versions, introduced in 4.2
req.body.currentTime = Math.min(videoDuration ?? 0, 30)
}
const currentTime: number = req.body.currentTime
if (!isVideoTimeValid(currentTime, videoDuration)) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Current time is invalid'
})
}
return next()
}
]
// ---------------------------------------------------------------------------
export {
videoViewValidator,
getVideoLocalViewerValidator
}

38
server/middlewares/validators/videos/video-watch.ts
View file

@ -1,38 +0,0 @@
import express from 'express'
import { body } from 'express-validator'
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { toIntOrNull } from '../../../helpers/custom-validators/misc'
import { logger } from '../../../helpers/logger'
import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
const videoWatchingValidator = [
isValidVideoIdParam('videoId'),
body('currentTime')
.customSanitizer(toIntOrNull)
.isInt().withMessage('Should have correct current time'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoWatching parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.videoId, res, 'id')) return
const user = res.locals.oauth.token.User
if (user.videosHistoryEnabled === false) {
logger.warn('Cannot set videos to watch by user %d: videos history is disabled.', user.id)
return res.fail({
status: HttpStatusCode.CONFLICT_409,
message: 'Video history is disabled'
})
}
return next()
}
]
// ---------------------------------------------------------------------------
export {
videoWatchingValidator
}

31
server/models/video/formatter/video-format-utils.ts
View file

@ -1,11 +1,19 @@
import { generateMagnetUri } from '@server/helpers/webtorrent'
import { getActivityStreamDuration } from '@server/lib/activitypub/activity'
import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
import { VideoViews } from '@server/lib/video-views'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { uuidToShort } from '@shared/extra-utils'
import { VideoFile, VideosCommonQueryAfterSanitize } from '@shared/models'
import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects'
import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos'
import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model'
import {
ActivityTagObject,
ActivityUrlObject,
Video,
VideoDetails,
VideoFile,
VideoInclude,
VideoObject,
VideosCommonQueryAfterSanitize,
VideoStreamingPlaylist
} from '@shared/models'
import { isArray } from '../../../helpers/custom-validators/misc'
import {
MIMETYPES,
@ -97,7 +105,10 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm
isLocal: video.isOwned(),
duration: video.duration,
views: video.views,
viewers: VideoViewsManager.Instance.getViewers(video),
likes: video.likes,
dislikes: video.dislikes,
thumbnailPath: video.getMiniatureStaticPath(),
@ -121,10 +132,6 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm
pluginData: (video as any).pluginData
}
if (video.isLive) {
videoObject.viewers = VideoViews.Instance.getViewers(video)
}
const add = options.additionalAttributes
if (add?.state === true) {
videoObject.state = {
@ -459,11 +466,6 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
}
}
function getActivityStreamDuration (duration: number) {
// https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
return 'PT' + duration + 'S'
}
function getCategoryLabel (id: number) {
return VIDEO_CATEGORIES[id] || 'Misc'
}
@ -489,7 +491,6 @@ export {
videoModelToFormattedDetailsJSON,
videoFilesModelToFormattedJSON,
videoModelToActivityPubObject,
getActivityStreamDuration,
guessAdditionalAttributesFromQuery,

2
server/models/video/video.ts
View file

@ -106,6 +106,7 @@ import { setAsUpdated } from '../shared'
import { UserModel } from '../user/user'
import { UserVideoHistoryModel } from '../user/user-video-history'
import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils'
import { VideoViewModel } from '../view/video-view'
import {
videoFilesModelToFormattedJSON,
VideoFormattingJSONOptions,
@ -135,7 +136,6 @@ import { VideoPlaylistElementModel } from './video-playlist-element'
import { VideoShareModel } from './video-share'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
import { VideoTagModel } from './video-tag'
import { VideoViewModel } from './video-view'
export enum ScopeNames {
FOR_API = 'FOR_API',

63
server/models/view/local-video-viewer-watch-section.ts Normal file
View file

@ -0,0 +1,63 @@
import { Transaction } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript'
import { MLocalVideoViewerWatchSection } from '@server/types/models'
import { AttributesOnly } from '@shared/typescript-utils'
import { LocalVideoViewerModel } from './local-video-viewer'
@Table({
tableName: 'localVideoViewerWatchSection',
updatedAt: false,
indexes: [
{
fields: [ 'localVideoViewerId' ]
}
]
})
export class LocalVideoViewerWatchSectionModel extends Model<Partial<AttributesOnly<LocalVideoViewerWatchSectionModel>>> {
@CreatedAt
createdAt: Date
@AllowNull(false)
@Column
watchStart: number
@AllowNull(false)
@Column
watchEnd: number
@ForeignKey(() => LocalVideoViewerModel)
@Column
localVideoViewerId: number
@BelongsTo(() => LocalVideoViewerModel, {
foreignKey: {
allowNull: false
},
onDelete: 'CASCADE'
})
LocalVideoViewer: LocalVideoViewerModel
static async bulkCreateSections (options: {
localVideoViewerId: number
watchSections: {
start: number
end: number
}[]
transaction?: Transaction
}) {
const { localVideoViewerId, watchSections, transaction } = options
const models: MLocalVideoViewerWatchSection[] = []
for (const section of watchSections) {
const model = await this.create({
watchStart: section.start,
watchEnd: section.end,
localVideoViewerId
}, { transaction })
models.push(model)
}
return models
}
}

274
server/models/view/local-video-viewer.ts Normal file
View file

@ -0,0 +1,274 @@
import { QueryTypes } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IsUUID, Model, Table } from 'sequelize-typescript'
import { STATS_TIMESERIE } from '@server/initializers/constants'
import { getActivityStreamDuration } from '@server/lib/activitypub/activity'
import { MLocalVideoViewer, MLocalVideoViewerWithWatchSections, MVideo } from '@server/types/models'
import { VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric, WatchActionObject } from '@shared/models'
import { AttributesOnly } from '@shared/typescript-utils'
import { VideoModel } from '../video/video'
import { LocalVideoViewerWatchSectionModel } from './local-video-viewer-watch-section'
@Table({
tableName: 'localVideoViewer',
updatedAt: false,
indexes: [
{
fields: [ 'videoId' ]
}
]
})
export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVideoViewerModel>>> {
@CreatedAt
createdAt: Date
@AllowNull(false)
@Column(DataType.DATE)
startDate: Date
@AllowNull(false)
@Column(DataType.DATE)
endDate: Date
@AllowNull(false)
@Column
watchTime: number
@AllowNull(true)
@Column
country: string
@AllowNull(false)
@Default(DataType.UUIDV4)
@IsUUID(4)
@Column(DataType.UUID)
uuid: string
@AllowNull(false)
@Column
url: string
@ForeignKey(() => VideoModel)
@Column
videoId: number
@BelongsTo(() => VideoModel, {
foreignKey: {
allowNull: false
},
onDelete: 'CASCADE'
})
Video: VideoModel
@HasMany(() => LocalVideoViewerWatchSectionModel, {
foreignKey: {
allowNull: false
},
onDelete: 'cascade'
})
WatchSections: LocalVideoViewerWatchSectionModel[]
static loadByUrl (url: string): Promise<MLocalVideoViewer> {
return this.findOne({
where: {
url
}
})
}
static loadFullById (id: number): Promise<MLocalVideoViewerWithWatchSections> {
return this.findOne({
include: [
{
model: VideoModel.unscoped(),
required: true
},
{
model: LocalVideoViewerWatchSectionModel.unscoped(),
required: true
}
],
where: {
id
}
})
}
static async getOverallStats (video: MVideo): Promise<VideoStatsOverall> {
const options = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
replacements: { videoId: video.id }
}
const watchTimeQuery = `SELECT ` +
`SUM("localVideoViewer"."watchTime") AS "totalWatchTime", ` +
`AVG("localVideoViewer"."watchTime") AS "averageWatchTime" ` +
`FROM "localVideoViewer" ` +
`INNER JOIN "video" ON "video"."id" = "localVideoViewer"."videoId" ` +
`WHERE "videoId" = :videoId`
const watchTimePromise = LocalVideoViewerModel.sequelize.query<any>(watchTimeQuery, options)
const watchPeakQuery = `WITH "watchPeakValues" AS (
SELECT "startDate" AS "dateBreakpoint", 1 AS "inc"
FROM "localVideoViewer"
WHERE "videoId" = :videoId
UNION ALL
SELECT "endDate" AS "dateBreakpoint", -1 AS "inc"
FROM "localVideoViewer"
WHERE "videoId" = :videoId
)
SELECT "dateBreakpoint", "concurrent"
FROM (
SELECT "dateBreakpoint", SUM(SUM("inc")) OVER (ORDER BY "dateBreakpoint") AS "concurrent"
FROM "watchPeakValues"
GROUP BY "dateBreakpoint"
) tmp
ORDER BY "concurrent" DESC
FETCH FIRST 1 ROW ONLY`
const watchPeakPromise = LocalVideoViewerModel.sequelize.query<any>(watchPeakQuery, options)
const commentsQuery = `SELECT COUNT(*) AS comments FROM "videoComment" WHERE "videoId" = :videoId`
const commentsPromise = LocalVideoViewerModel.sequelize.query<any>(commentsQuery, options)
const countriesQuery = `SELECT country, COUNT(country) as viewers ` +
`FROM "localVideoViewer" ` +
`WHERE "videoId" = :videoId AND country IS NOT NULL ` +
`GROUP BY country ` +
`ORDER BY viewers DESC`
const countriesPromise = LocalVideoViewerModel.sequelize.query<any>(countriesQuery, options)
const [ rowsWatchTime, rowsWatchPeak, rowsComment, rowsCountries ] = await Promise.all([
watchTimePromise,
watchPeakPromise,
commentsPromise,
countriesPromise
])
return {
totalWatchTime: rowsWatchTime.length !== 0
? Math.round(rowsWatchTime[0].totalWatchTime) || 0
: 0,
averageWatchTime: rowsWatchTime.length !== 0
? Math.round(rowsWatchTime[0].averageWatchTime) || 0
: 0,
viewersPeak: rowsWatchPeak.length !== 0
? parseInt(rowsWatchPeak[0].concurrent) || 0
: 0,
viewersPeakDate: rowsWatchPeak.length !== 0
? rowsWatchPeak[0].dateBreakpoint || null
: null,
views: video.views,
likes: video.likes,
dislikes: video.dislikes,
comments: rowsComment.length !== 0
? parseInt(rowsComment[0].comments) || 0
: 0,
countries: rowsCountries.map(r => ({
isoCode: r.country,
viewers: r.viewers
}))
}
}
static async getRetentionStats (video: MVideo): Promise<VideoStatsRetention> {
const step = Math.max(Math.round(video.duration / 100), 1)
const query = `WITH "total" AS (SELECT COUNT(*) AS viewers FROM "localVideoViewer" WHERE "videoId" = :videoId) ` +
`SELECT serie AS "second", ` +
`(COUNT("localVideoViewer".id)::float / (SELECT GREATEST("total"."viewers", 1) FROM "total")) AS "retention" ` +
`FROM generate_series(0, ${video.duration}, ${step}) serie ` +
`LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` +
`AND EXISTS (` +
`SELECT 1 FROM "localVideoViewerWatchSection" ` +
`WHERE "localVideoViewer"."id" = "localVideoViewerWatchSection"."localVideoViewerId" ` +
`AND serie >= "localVideoViewerWatchSection"."watchStart" ` +
`AND serie <= "localVideoViewerWatchSection"."watchEnd"` +
`)` +
`GROUP BY serie ` +
`ORDER BY serie ASC`
const queryOptions = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
replacements: { videoId: video.id }
}
const rows = await LocalVideoViewerModel.sequelize.query<any>(query, queryOptions)
return {
data: rows.map(r => ({
second: r.second,
retentionPercent: parseFloat(r.retention) * 100
}))
}
}
static async getTimeserieStats (options: {
video: MVideo
metric: VideoStatsTimeserieMetric
}): Promise<VideoStatsTimeserie> {
const { video, metric } = options
const selectMetrics: { [ id in VideoStatsTimeserieMetric ]: string } = {
viewers: 'COUNT("localVideoViewer"."id")',
aggregateWatchTime: 'SUM("localVideoViewer"."watchTime")'
}
const query = `WITH days AS ( ` +
`SELECT (current_date::timestamp - (serie || ' days')::interval)::timestamptz AS day
FROM generate_series(0, ${STATS_TIMESERIE.MAX_DAYS - 1}) serie` +
`) ` +
`SELECT days.day AS date, COALESCE(${selectMetrics[metric]}, 0) AS value ` +
`FROM days ` +
`LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` +
`AND date_trunc('day', "localVideoViewer"."startDate") = date_trunc('day', days.day) ` +
`GROUP BY day ` +
`ORDER BY day `
const queryOptions = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
replacements: { videoId: video.id }
}
const rows = await LocalVideoViewerModel.sequelize.query<any>(query, queryOptions)
return {
data: rows.map(r => ({
date: r.date,
value: parseInt(r.value)
}))
}
}
toActivityPubObject (this: MLocalVideoViewerWithWatchSections): WatchActionObject {
const location = this.country
? {
location: {
addressCountry: this.country
}
}
: {}
return {
id: this.url,
type: 'WatchAction',
duration: getActivityStreamDuration(this.watchTime),
startTime: this.startDate.toISOString(),
endTime: this.endDate.toISOString(),
object: this.Video.url,
uuid: this.uuid,
actionStatus: 'CompletedActionStatus',
watchSections: this.WatchSections.map(w => ({
startTimestamp: w.watchStart,
endTimestamp: w.watchEnd
})),
...location
}
}
}

2
server/models/video/video-view.ts → server/models/view/video-view.ts
View file

@ -1,7 +1,7 @@
import { literal, Op } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table } from 'sequelize-typescript'
import { AttributesOnly } from '@shared/typescript-utils'
import { VideoModel } from './video'
import { VideoModel } from '../video/video'
@Table({
tableName: 'videoView',

20
server/tests/api/activitypub/client.ts
View file

@ -2,6 +2,8 @@
import 'mocha'
import * as chai from 'chai'
import { processViewersStats } from '@server/tests/shared'
import { HttpStatusCode, VideoPlaylistPrivacy, WatchActionObject } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
@ -11,7 +13,6 @@ import {
setAccessTokensToServers,
setDefaultVideoChannel
} from '@shared/server-commands'
import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
const expect = chai.expect
@ -115,6 +116,23 @@ describe('Test activitypub', function () {
expect(res.header.location).to.equal('http://localhost:' + servers[0].port + '/videos/watch/' + video.uuid)
})
it('Should return the watch action', async function () {
this.timeout(50000)
await servers[0].views.simulateViewer({ id: video.uuid, currentTimes: [ 0, 2 ] })
await processViewersStats(servers)
const res = await makeActivityPubGetRequest(servers[0].url, '/videos/local-viewer/1', HttpStatusCode.OK_200)
const object: WatchActionObject = res.body
expect(object.type).to.equal('WatchAction')
expect(object.duration).to.equal('PT2S')
expect(object.actionStatus).to.equal('CompletedActionStatus')
expect(object.watchSections).to.have.lengthOf(1)
expect(object.watchSections[0].startTimestamp).to.equal(0)
expect(object.watchSections[0].endTimestamp).to.equal(2)
})
after(async function () {
await cleanupTests(servers)
})

1
server/tests/api/check-params/index.ts
View file

@ -33,3 +33,4 @@ import './videos-common-filters'
import './video-files'
import './videos-history'
import './videos-overviews'
import './views'

46
server/tests/api/check-params/videos-history.ts
View file

@ -17,7 +17,7 @@ import {
describe('Test videos history API validator', function () {
const myHistoryPath = '/api/v1/users/me/history/videos'
const myHistoryRemove = myHistoryPath + '/remove'
let watchingPath: string
let viewPath: string
let server: PeerTubeServer
let videoId: number
@ -31,51 +31,15 @@ describe('Test videos history API validator', function () {
await setAccessTokensToServers([ server ])
const { id, uuid } = await server.videos.upload()
watchingPath = '/api/v1/videos/' + uuid + '/watching'
viewPath = '/api/v1/videos/' + uuid + '/views'
videoId = id
})
describe('When notifying a user is watching a video', function () {
it('Should fail with an unauthenticated user', async function () {
it('Should fail with a bad token', async function () {
const fields = { currentTime: 5 }
await makePutBodyRequest({ url: server.url, path: watchingPath, fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should fail with an incorrect video id', async function () {
const fields = { currentTime: 5 }
const path = '/api/v1/videos/blabla/watching'
await makePutBodyRequest({
url: server.url,
path,
fields,
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail with an unknown video', async function () {
const fields = { currentTime: 5 }
const path = '/api/v1/videos/d91fff41-c24d-4508-8e13-3bd5902c3b02/watching'
await makePutBodyRequest({
url: server.url,
path,
fields,
token: server.accessToken,
expectedStatus: HttpStatusCode.NOT_FOUND_404
})
})
it('Should fail with a bad current time', async function () {
const fields = { currentTime: 'hello' }
await makePutBodyRequest({
url: server.url,
path: watchingPath,
fields,
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
await makePutBodyRequest({ url: server.url, path: viewPath, fields, token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should succeed with the correct parameters', async function () {
@ -83,7 +47,7 @@ describe('Test videos history API validator', function () {
await makePutBodyRequest({
url: server.url,
path: watchingPath,
path: viewPath,
fields,
token: server.accessToken,
expectedStatus: HttpStatusCode.NO_CONTENT_204

157
server/tests/api/check-params/views.ts Normal file
View file

@ -0,0 +1,157 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel
} from '@shared/server-commands'
describe('Test videos views', function () {
let servers: PeerTubeServer[]
let liveVideoId: string
let videoId: string
let remoteVideoId: string
let userAccessToken: string
before(async function () {
this.timeout(30000)
servers = await createMultipleServers(2)
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await servers[0].config.enableLive({ allowReplay: false, transcoding: false });
({ uuid: videoId } = await servers[0].videos.quickUpload({ name: 'video' }));
({ uuid: remoteVideoId } = await servers[1].videos.quickUpload({ name: 'video' }));
({ uuid: liveVideoId } = await servers[0].live.create({
fields: {
name: 'live',
privacy: VideoPrivacy.PUBLIC,
channelId: servers[0].store.channel.id
}
}))
userAccessToken = await servers[0].users.generateUserAndToken('user')
await doubleFollow(servers[0], servers[1])
})
describe('When viewing a video', async function () {
// TODO: implement it when we'll remove backward compatibility in REST API
it('Should fail without current time')
it('Should fail with an invalid current time', async function () {
await servers[0].views.view({ id: videoId, currentTime: -1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].views.view({ id: videoId, currentTime: 10, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should succeed with correct parameters', async function () {
await servers[0].views.view({ id: videoId, currentTime: 1 })
})
})
describe('When getting overall stats', function () {
it('Should fail with a remote video', async function () {
await servers[0].videoStats.getOverallStats({ videoId: remoteVideoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail without token', async function () {
await servers[0].videoStats.getOverallStats({ videoId: videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should fail with another token', async function () {
await servers[0].videoStats.getOverallStats({
videoId: videoId,
token: userAccessToken,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should succeed with the correct parameters', async function () {
await servers[0].videoStats.getOverallStats({ videoId })
})
})
describe('When getting timeserie stats', function () {
it('Should fail with a remote video', async function () {
await servers[0].videoStats.getTimeserieStats({
videoId: remoteVideoId,
metric: 'viewers',
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail without token', async function () {
await servers[0].videoStats.getTimeserieStats({
videoId: videoId,
token: null,
metric: 'viewers',
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail with another token', async function () {
await servers[0].videoStats.getTimeserieStats({
videoId: videoId,
token: userAccessToken,
metric: 'viewers',
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail with an invalid metric', async function () {
await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'hello' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should succeed with the correct parameters', async function () {
await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' })
})
})
describe('When getting retention stats', function () {
it('Should fail with a remote video', async function () {
await servers[0].videoStats.getRetentionStats({
videoId: remoteVideoId,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail without token', async function () {
await servers[0].videoStats.getRetentionStats({
videoId: videoId,
token: null,
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail with another token', async function () {
await servers[0].videoStats.getRetentionStats({
videoId: videoId,
token: userAccessToken,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail on live video', async function () {
await servers[0].videoStats.getRetentionStats({ videoId: liveVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should succeed with the correct parameters', async function () {
await servers[0].videoStats.getRetentionStats({ videoId })
})
})
after(async function () {
await cleanupTests(servers)
})
})

1
server/tests/api/live/index.ts
View file

@ -3,5 +3,4 @@ import './live-socket-messages'
import './live-permanent'
import './live-rtmps'
import './live-save-replay'
import './live-views'
import './live'

4
server/tests/api/live/live-socket-messages.ts
View file

@ -140,8 +140,8 @@ describe('Test live', function () {
expect(localLastVideoViews).to.equal(0)
expect(remoteLastVideoViews).to.equal(0)
await servers[0].videos.view({ id: liveVideoUUID })
await servers[1].videos.view({ id: liveVideoUUID })
await servers[0].views.simulateView({ id: liveVideoUUID })
await servers[1].views.simulateView({ id: liveVideoUUID })
await waitJobs(servers)

132
server/tests/api/live/live-views.ts
View file

@ -1,132 +0,0 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { wait } from '@shared/core-utils'
import { VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
stopFfmpeg,
waitJobs,
waitUntilLivePublishedOnAllServers
} from '@shared/server-commands'
const expect = chai.expect
describe('Live views', function () {
let servers: PeerTubeServer[] = []
before(async function () {
this.timeout(120000)
servers = await createMultipleServers(2)
// Get the access tokens
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await servers[0].config.updateCustomSubConfig({
newConfig: {
live: {
enabled: true,
allowReplay: true,
transcoding: {
enabled: false
}
}
}
})
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
})
let liveVideoId: string
let command: FfmpegCommand
async function countViewers (expectedViewers: number) {
for (const server of servers) {
const video = await server.videos.get({ id: liveVideoId })
expect(video.viewers).to.equal(expectedViewers)
}
}
async function countViews (expectedViews: number) {
for (const server of servers) {
const video = await server.videos.get({ id: liveVideoId })
expect(video.views).to.equal(expectedViews)
}
}
before(async function () {
this.timeout(30000)
const liveAttributes = {
name: 'live video',
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC
}
const live = await servers[0].live.create({ fields: liveAttributes })
liveVideoId = live.uuid
command = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId })
await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
await waitJobs(servers)
})
it('Should display no views and viewers for a live', async function () {
await countViews(0)
await countViewers(0)
})
it('Should view a live twice and display 1 view/viewer', async function () {
this.timeout(30000)
await servers[0].videos.view({ id: liveVideoId })
await servers[0].videos.view({ id: liveVideoId })
await waitJobs(servers)
await countViewers(1)
await wait(7000)
await countViews(1)
})
it('Should wait and display 0 viewers while still have 1 view', async function () {
this.timeout(30000)
await wait(12000)
await waitJobs(servers)
await countViews(1)
await countViewers(0)
})
it('Should view a live on a remote and on local and display 2 viewers and 3 views', async function () {
this.timeout(30000)
await servers[0].videos.view({ id: liveVideoId })
await servers[1].videos.view({ id: liveVideoId })
await servers[1].videos.view({ id: liveVideoId })
await waitJobs(servers)
await countViewers(2)
await wait(7000)
await waitJobs(servers)
await countViews(3)
})
after(async function () {
await stopFfmpeg(command)
await cleanupTests(servers)
})
})

10
server/tests/api/redundancy/redundancy.ts
View file

@ -87,7 +87,7 @@ async function createServers (strategy: VideoRedundancyStrategy | null, addition
const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
video1Server2 = await servers[1].videos.get({ id })
await servers[1].videos.view({ id })
await servers[1].views.simulateView({ id })
}
await waitJobs(servers)
@ -447,8 +447,8 @@ describe('Test videos redundancy', function () {
it('Should view 2 times the first video to have > min_views config', async function () {
this.timeout(80000)
await servers[0].videos.view({ id: video1Server2.uuid })
await servers[2].videos.view({ id: video1Server2.uuid })
await servers[0].views.simulateView({ id: video1Server2.uuid })
await servers[2].views.simulateView({ id: video1Server2.uuid })
await wait(10000)
await waitJobs(servers)
@ -516,8 +516,8 @@ describe('Test videos redundancy', function () {
it('Should have 1 redundancy on the first video', async function () {
this.timeout(160000)
await servers[0].videos.view({ id: video1Server2.uuid })
await servers[2].videos.view({ id: video1Server2.uuid })
await servers[0].views.simulateView({ id: video1Server2.uuid })
await servers[2].views.simulateView({ id: video1Server2.uuid })
await wait(10000)
await waitJobs(servers)

16
server/tests/api/server/reverse-proxy.ts
View file

@ -41,8 +41,8 @@ describe('Test application behind a reverse proxy', function () {
it('Should view a video only once with the same IP by default', async function () {
this.timeout(20000)
await server.videos.view({ id: videoId })
await server.videos.view({ id: videoId })
await server.views.simulateView({ id: videoId })
await server.views.simulateView({ id: videoId })
// Wait the repeatable job
await wait(8000)
@ -54,8 +54,8 @@ describe('Test application behind a reverse proxy', function () {
it('Should view a video 2 times with the X-Forwarded-For header set', async function () {
this.timeout(20000)
await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' })
await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' })
await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' })
await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' })
// Wait the repeatable job
await wait(8000)
@ -67,8 +67,8 @@ describe('Test application behind a reverse proxy', function () {
it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () {
this.timeout(20000)
await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' })
await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' })
await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' })
await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' })
// Wait the repeatable job
await wait(8000)
@ -80,8 +80,8 @@ describe('Test application behind a reverse proxy', function () {
it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () {
this.timeout(20000)
await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' })
await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' })
await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' })
await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' })
// Wait the repeatable job
await wait(8000)

2
server/tests/api/server/stats.ts
View file

@ -38,7 +38,7 @@ describe('Test stats (excluding redundancy)', function () {
await servers[0].comments.createThread({ videoId: uuid, text: 'comment' })
await servers[0].videos.view({ id: uuid })
await servers[0].views.simulateView({ id: uuid })
// Wait the video views repeatable job
await wait(8000)

1
server/tests/api/videos/index.ts
View file

@ -16,4 +16,3 @@ import './video-schedule-update'
import './videos-common-filters'
import './videos-history'
import './videos-overview'
import './videos-views-cleaner'

40
server/tests/api/videos/multiple-servers.ts
View file

@ -504,21 +504,22 @@ describe('Test multiple servers', function () {
it('Should view multiple videos on owned servers', async function () {
this.timeout(30000)
await servers[2].videos.view({ id: localVideosServer3[0] })
await servers[2].views.simulateView({ id: localVideosServer3[0] })
await wait(1000)
await servers[2].videos.view({ id: localVideosServer3[0] })
await servers[2].videos.view({ id: localVideosServer3[1] })
await servers[2].views.simulateView({ id: localVideosServer3[0] })
await servers[2].views.simulateView({ id: localVideosServer3[1] })
await wait(1000)
await servers[2].videos.view({ id: localVideosServer3[0] })
await servers[2].videos.view({ id: localVideosServer3[0] })
await servers[2].views.simulateView({ id: localVideosServer3[0] })
await servers[2].views.simulateView({ id: localVideosServer3[0] })
await waitJobs(servers)
// Wait the repeatable job
await wait(6000)
for (const server of servers) {
await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
}
await waitJobs(servers)
@ -537,23 +538,24 @@ describe('Test multiple servers', function () {
this.timeout(45000)
const tasks: Promise<any>[] = []
tasks.push(servers[0].videos.view({ id: remoteVideosServer1[0] }))
tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] }))
tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] }))
tasks.push(servers[2].videos.view({ id: remoteVideosServer3[0] }))
tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] }))
tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] }))
tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] }))
tasks.push(servers[2].videos.view({ id: localVideosServer3[1] }))
tasks.push(servers[2].videos.view({ id: localVideosServer3[1] }))
tasks.push(servers[2].videos.view({ id: localVideosServer3[1] }))
tasks.push(servers[0].views.simulateView({ id: remoteVideosServer1[0] }))
tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] }))
tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] }))
tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[0] }))
tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] }))
tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] }))
tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] }))
tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] }))
tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] }))
tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] }))
await Promise.all(tasks)
await waitJobs(servers)
// Wait the repeatable job
await wait(16000)
for (const server of servers) {
await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
}
await waitJobs(servers)

17
server/tests/api/videos/single-server.ts
View file

@ -179,22 +179,21 @@ describe('Test a single server', function () {
it('Should have the views updated', async function () {
this.timeout(20000)
await server.videos.view({ id: videoId })
await server.videos.view({ id: videoId })
await server.videos.view({ id: videoId })
await server.views.simulateView({ id: videoId })
await server.views.simulateView({ id: videoId })
await server.views.simulateView({ id: videoId })
await wait(1500)
await server.videos.view({ id: videoId })
await server.videos.view({ id: videoId })
await server.views.simulateView({ id: videoId })
await server.views.simulateView({ id: videoId })
await wait(1500)
await server.videos.view({ id: videoId })
await server.videos.view({ id: videoId })
await server.views.simulateView({ id: videoId })
await server.views.simulateView({ id: videoId })
// Wait the repeatable job
await wait(8000)
await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
const video = await server.videos.get({ id: videoId })
expect(video.views).to.equal(3)

4
server/tests/api/videos/video-channels.ts
View file

@ -466,8 +466,8 @@ describe('Test video channels', function () {
{
// video has been posted on channel servers[0].store.videoChannel.id since last update
await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' })
await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' })
await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' })
await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' })
// Wait the repeatable job
await wait(8000)

73
server/tests/api/videos/videos-history.ts
View file

@ -3,15 +3,8 @@
import 'mocha'
import * as chai from 'chai'
import { wait } from '@shared/core-utils'
import { HttpStatusCode, Video } from '@shared/models'
import {
cleanupTests,
createSingleServer,
HistoryCommand,
killallServers,
PeerTubeServer,
setAccessTokensToServers
} from '@shared/server-commands'
import { Video } from '@shared/models'
import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
const expect = chai.expect
@ -23,7 +16,6 @@ describe('Test videos history', function () {
let video3UUID: string
let video3WatchedDate: Date
let userAccessToken: string
let command: HistoryCommand
before(async function () {
this.timeout(30000)
@ -32,30 +24,26 @@ describe('Test videos history', function () {
await setAccessTokensToServers([ server ])
command = server.history
// 10 seconds long
const fixture = 'video_59fps.mp4'
{
const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1' } })
const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1', fixture } })
video1UUID = uuid
video1Id = id
}
{
const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
const { uuid } = await server.videos.upload({ attributes: { name: 'video 2', fixture } })
video2UUID = uuid
}
{
const { uuid } = await server.videos.upload({ attributes: { name: 'video 3' } })
const { uuid } = await server.videos.upload({ attributes: { name: 'video 3', fixture } })
video3UUID = uuid
}
const user = {
username: 'user_1',
password: 'super password'
}
await server.users.create({ username: user.username, password: user.password })
userAccessToken = await server.login.getAccessToken(user)
userAccessToken = await server.users.generateUserAndToken('user_1')
})
it('Should get videos, without watching history', async function () {
@ -70,8 +58,8 @@ describe('Test videos history', function () {
})
it('Should watch the first and second video', async function () {
await command.watchVideo({ videoId: video2UUID, currentTime: 8 })
await command.watchVideo({ videoId: video1UUID, currentTime: 3 })
await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 3 })
})
it('Should return the correct history when listing, searching and getting videos', async function () {
@ -124,9 +112,9 @@ describe('Test videos history', function () {
it('Should have these videos when listing my history', async function () {
video3WatchedDate = new Date()
await command.watchVideo({ videoId: video3UUID, currentTime: 2 })
await server.views.view({ id: video3UUID, token: server.accessToken, currentTime: 2 })
const body = await command.list()
const body = await server.history.list()
expect(body.total).to.equal(3)
@ -137,14 +125,14 @@ describe('Test videos history', function () {
})
it('Should not have videos history on another user', async function () {
const body = await command.list({ token: userAccessToken })
const body = await server.history.list({ token: userAccessToken })
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
})
it('Should be able to search through videos in my history', async function () {
const body = await command.list({ search: '2' })
const body = await server.history.list({ search: '2' })
expect(body.total).to.equal(1)
const videos = body.data
@ -152,11 +140,11 @@ describe('Test videos history', function () {
})
it('Should clear my history', async function () {
await command.removeAll({ beforeDate: video3WatchedDate.toISOString() })
await server.history.removeAll({ beforeDate: video3WatchedDate.toISOString() })
})
it('Should have my history cleared', async function () {
const body = await command.list()
const body = await server.history.list()
expect(body.total).to.equal(1)
const videos = body.data
@ -168,7 +156,10 @@ describe('Test videos history', function () {
videosHistoryEnabled: false
})
await command.watchVideo({ videoId: video2UUID, currentTime: 8, expectedStatus: HttpStatusCode.CONFLICT_409 })
await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
const { data } = await server.history.list()
expect(data[0].name).to.not.equal('video 2')
})
it('Should re-enable videos history', async function () {
@ -176,14 +167,10 @@ describe('Test videos history', function () {
videosHistoryEnabled: true
})
await command.watchVideo({ videoId: video1UUID, currentTime: 8 })
await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
const body = await command.list()
expect(body.total).to.equal(2)
const videos = body.data
expect(videos[0].name).to.equal('video 1')
expect(videos[1].name).to.equal('video 3')
const { data } = await server.history.list()
expect(data[0].name).to.equal('video 2')
})
it('Should not clean old history', async function () {
@ -197,7 +184,7 @@ describe('Test videos history', function () {
// Should still have history
const body = await command.list()
const body = await server.history.list()
expect(body.total).to.equal(2)
})
@ -210,25 +197,25 @@ describe('Test videos history', function () {
await wait(6000)
const body = await command.list()
const body = await server.history.list()
expect(body.total).to.equal(0)
})
it('Should delete a specific history element', async function () {
{
await command.watchVideo({ videoId: video1UUID, currentTime: 4 })
await command.watchVideo({ videoId: video2UUID, currentTime: 8 })
await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 4 })
await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
}
{
const body = await command.list()
const body = await server.history.list()
expect(body.total).to.equal(2)
}
{
await command.removeElement({ videoId: video1Id })
await server.history.removeElement({ videoId: video1Id })
const body = await command.list()
const body = await server.history.list()
expect(body.total).to.equal(1)
expect(body.data[0].uuid).to.equal(video2UUID)
}

5
server/tests/api/views/index.ts Normal file
View file

@ -0,0 +1,5 @@
export * from './video-views-counter'
export * from './video-views-overall-stats'
export * from './video-views-retention-stats'
export * from './video-views-timeserie-stats'
export * from './videos-views-cleaner'

155
server/tests/api/views/video-views-counter.ts Normal file
View file

@ -0,0 +1,155 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@server/tests/shared'
import { wait } from '@shared/core-utils'
import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands'
const expect = chai.expect
describe('Test video views/viewers counters', function () {
let servers: PeerTubeServer[]
async function checkCounter (field: 'views' | 'viewers', id: string, expected: number) {
for (const server of servers) {
const video = await server.videos.get({ id })
const messageSuffix = video.isLive
? 'live video'
: 'vod video'
expect(video[field]).to.equal(expected, `${field} not valid on server ${server.serverNumber} for ${messageSuffix} ${video.uuid}`)
}
}
before(async function () {
this.timeout(120000)
servers = await prepareViewsServers()
})
describe('Test views counter on VOD', function () {
let videoUUID: string
before(async function () {
this.timeout(30000)
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
videoUUID = uuid
await waitJobs(servers)
})
it('Should not view a video if watch time is below the threshold', async function () {
await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 2 ] })
await processViewsBuffer(servers)
await checkCounter('views', videoUUID, 0)
})
it('Should view a video if watch time is above the threshold', async function () {
await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] })
await processViewsBuffer(servers)
await checkCounter('views', videoUUID, 1)
})
it('Should not view again this video with the same IP', async function () {
await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] })
await processViewsBuffer(servers)
await checkCounter('views', videoUUID, 1)
})
it('Should view the video from server 2 and send the event', async function () {
await servers[1].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] })
await waitJobs(servers)
await processViewsBuffer(servers)
await checkCounter('views', videoUUID, 2)
})
})
describe('Test views and viewers counters on live and VOD', function () {
let liveVideoId: string
let vodVideoId: string
let command: FfmpegCommand
before(async function () {
this.timeout(60000);
({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true }))
})
it('Should display no views and viewers', async function () {
await checkCounter('views', liveVideoId, 0)
await checkCounter('viewers', liveVideoId, 0)
await checkCounter('views', vodVideoId, 0)
await checkCounter('viewers', vodVideoId, 0)
})
it('Should view twice and display 1 view/viewer', async function () {
this.timeout(30000)
await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await waitJobs(servers)
await checkCounter('viewers', liveVideoId, 1)
await checkCounter('viewers', vodVideoId, 1)
await processViewsBuffer(servers)
await checkCounter('views', liveVideoId, 1)
await checkCounter('views', vodVideoId, 1)
})
it('Should wait and display 0 viewers but still have 1 view', async function () {
this.timeout(30000)
await wait(12000)
await waitJobs(servers)
await checkCounter('views', liveVideoId, 1)
await checkCounter('viewers', liveVideoId, 0)
await checkCounter('views', vodVideoId, 1)
await checkCounter('viewers', vodVideoId, 0)
})
it('Should view on a remote and on local and display 2 viewers and 3 views', async function () {
this.timeout(30000)
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] })
await waitJobs(servers)
await checkCounter('viewers', liveVideoId, 2)
await checkCounter('viewers', vodVideoId, 2)
await processViewsBuffer(servers)
await checkCounter('views', liveVideoId, 3)
await checkCounter('views', vodVideoId, 3)
})
after(async function () {
await stopFfmpeg(command)
})
})
after(async function () {
await cleanupTests(servers)
})
})

291
server/tests/api/views/video-views-overall-stats.ts Normal file
View file

@ -0,0 +1,291 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared'
import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands'
const expect = chai.expect
describe('Test views overall stats', function () {
let servers: PeerTubeServer[]
before(async function () {
this.timeout(120000)
servers = await prepareViewsServers()
})
describe('Test rates and comments of local videos on VOD', function () {
let vodVideoId: string
before(async function () {
this.timeout(60000);
({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true }))
})
it('Should have the appropriate likes', async function () {
this.timeout(60000)
await servers[0].videos.rate({ id: vodVideoId, rating: 'like' })
await servers[1].videos.rate({ id: vodVideoId, rating: 'like' })
await waitJobs(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId })
expect(stats.likes).to.equal(2)
expect(stats.dislikes).to.equal(0)
})
it('Should have the appropriate dislikes', async function () {
this.timeout(60000)
await servers[0].videos.rate({ id: vodVideoId, rating: 'dislike' })
await servers[1].videos.rate({ id: vodVideoId, rating: 'dislike' })
await waitJobs(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId })
expect(stats.likes).to.equal(0)
expect(stats.dislikes).to.equal(2)
})
it('Should have the appropriate comments', async function () {
this.timeout(60000)
await servers[0].comments.createThread({ videoId: vodVideoId, text: 'root' })
await servers[0].comments.addReplyToLastThread({ text: 'reply' })
await servers[1].comments.createThread({ videoId: vodVideoId, text: 'root' })
await waitJobs(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId })
expect(stats.comments).to.equal(3)
})
})
describe('Test watch time stats of local videos on live and VOD', function () {
let vodVideoId: string
let liveVideoId: string
let command: FfmpegCommand
before(async function () {
this.timeout(60000);
({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true }))
})
it('Should display overall stats of a video with no viewers', async function () {
for (const videoId of [ liveVideoId, vodVideoId ]) {
const stats = await servers[0].videoStats.getOverallStats({ videoId })
expect(stats.views).to.equal(0)
expect(stats.averageWatchTime).to.equal(0)
expect(stats.totalWatchTime).to.equal(0)
}
})
it('Should display overall stats with 1 viewer below the watch time limit', async function () {
this.timeout(60000)
for (const videoId of [ liveVideoId, vodVideoId ]) {
await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] })
}
await processViewersStats(servers)
for (const videoId of [ liveVideoId, vodVideoId ]) {
const stats = await servers[0].videoStats.getOverallStats({ videoId })
expect(stats.views).to.equal(0)
expect(stats.averageWatchTime).to.equal(1)
expect(stats.totalWatchTime).to.equal(1)
}
})
it('Should display overall stats with 2 viewers', async function () {
this.timeout(60000)
{
await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] })
await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35, 40 ] })
await processViewersStats(servers)
{
const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId })
expect(stats.views).to.equal(1)
expect(stats.averageWatchTime).to.equal(2)
expect(stats.totalWatchTime).to.equal(4)
}
{
const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId })
expect(stats.views).to.equal(1)
expect(stats.averageWatchTime).to.equal(21)
expect(stats.totalWatchTime).to.equal(41)
}
}
})
it('Should display overall stats with a remote viewer below the watch time limit', async function () {
this.timeout(60000)
for (const videoId of [ liveVideoId, vodVideoId ]) {
await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 2 ] })
}
await processViewersStats(servers)
{
const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId })
expect(stats.views).to.equal(1)
expect(stats.averageWatchTime).to.equal(2)
expect(stats.totalWatchTime).to.equal(6)
}
{
const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId })
expect(stats.views).to.equal(1)
expect(stats.averageWatchTime).to.equal(14)
expect(stats.totalWatchTime).to.equal(43)
}
})
it('Should display overall stats with a remote viewer above the watch time limit', async function () {
this.timeout(60000)
await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] })
await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 45 ] })
await processViewersStats(servers)
{
const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId })
expect(stats.views).to.equal(2)
expect(stats.averageWatchTime).to.equal(3)
expect(stats.totalWatchTime).to.equal(11)
}
{
const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId })
expect(stats.views).to.equal(2)
expect(stats.averageWatchTime).to.equal(22)
expect(stats.totalWatchTime).to.equal(88)
}
})
after(async function () {
await stopFfmpeg(command)
})
})
describe('Test watchers peak stats of local videos on VOD', function () {
let videoUUID: string
before(async function () {
this.timeout(60000);
({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true }))
})
it('Should not have watchers peak', async function () {
const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID })
expect(stats.viewersPeak).to.equal(0)
expect(stats.viewersPeakDate).to.be.null
})
it('Should have watcher peak with 1 watcher', async function () {
this.timeout(60000)
const before = new Date()
await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 0, 2 ] })
const after = new Date()
await processViewersStats(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID })
expect(stats.viewersPeak).to.equal(1)
expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after)
})
it('Should have watcher peak with 2 watchers', async function () {
this.timeout(60000)
const before = new Date()
await servers[0].views.view({ id: videoUUID, currentTime: 0 })
await servers[1].views.view({ id: videoUUID, currentTime: 0 })
await servers[0].views.view({ id: videoUUID, currentTime: 2 })
await servers[1].views.view({ id: videoUUID, currentTime: 2 })
const after = new Date()
await processViewersStats(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID })
expect(stats.viewersPeak).to.equal(2)
expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after)
})
})
describe('Test countries', function () {
it('Should not report countries if geoip is disabled', async function () {
this.timeout(60000)
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers)
await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 })
await processViewersStats(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
expect(stats.countries).to.have.lengthOf(0)
})
it('Should report countries if geoip is enabled', async function () {
this.timeout(60000)
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers)
await Promise.all([
servers[0].kill(),
servers[1].kill()
])
const config = { geo_ip: { enabled: true } }
await Promise.all([
servers[0].run(config),
servers[1].run(config)
])
await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 })
await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 })
await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 })
await processViewersStats(servers)
const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid })
expect(stats.countries).to.have.lengthOf(2)
expect(stats.countries[0].isoCode).to.equal('US')
expect(stats.countries[0].viewers).to.equal(2)
expect(stats.countries[1].isoCode).to.equal('FR')
expect(stats.countries[1].viewers).to.equal(1)
})
})
after(async function () {
await cleanupTests(servers)
})
})

56
server/tests/api/views/video-views-retention-stats.ts Normal file
View file

@ -0,0 +1,56 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared'
import { cleanupTests, PeerTubeServer } from '@shared/server-commands'
const expect = chai.expect
describe('Test views retention stats', function () {
let servers: PeerTubeServer[]
before(async function () {
this.timeout(120000)
servers = await prepareViewsServers()
})
describe('Test retention stats on VOD', function () {
let vodVideoId: string
before(async function () {
this.timeout(60000);
({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true }))
})
it('Should display empty retention', async function () {
const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId })
expect(data).to.have.lengthOf(6)
for (let i = 0; i < 6; i++) {
expect(data[i].second).to.equal(i)
expect(data[i].retentionPercent).to.equal(0)
}
})
it('Should display appropriate retention metrics', async function () {
await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] })
await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] })
await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 4 ] })
await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] })
await processViewersStats(servers)
const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId })
expect(data).to.have.lengthOf(6)
expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ])
})
})
after(async function () {
await cleanupTests(servers)
})
})

109
server/tests/api/views/video-views-timeserie-stats.ts Normal file
View file

@ -0,0 +1,109 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared'
import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models'
import { cleanupTests, PeerTubeServer, stopFfmpeg } from '@shared/server-commands'
const expect = chai.expect
describe('Test views timeserie stats', function () {
const availableMetrics: VideoStatsTimeserieMetric[] = [ 'viewers' ]
let servers: PeerTubeServer[]
before(async function () {
this.timeout(120000)
servers = await prepareViewsServers()
})
describe('Common metric tests', function () {
let vodVideoId: string
before(async function () {
this.timeout(60000);
({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true }))
})
it('Should display empty metric stats', async function () {
for (const metric of availableMetrics) {
const { data } = await servers[0].videoStats.getTimeserieStats({ videoId: vodVideoId, metric })
expect(data).to.have.lengthOf(30)
for (const d of data) {
expect(d.value).to.equal(0)
}
}
})
})
describe('Test viewer and watch time metrics on live and VOD', function () {
let vodVideoId: string
let liveVideoId: string
let command: FfmpegCommand
function expectTimeserieData (result: VideoStatsTimeserie, lastValue: number) {
const { data } = result
expect(data).to.have.lengthOf(30)
const last = data[data.length - 1]
const today = new Date().getDate()
expect(new Date(last.date).getDate()).to.equal(today)
expect(last.value).to.equal(lastValue)
for (let i = 0; i < data.length - 2; i++) {
expect(data[i].value).to.equal(0)
}
}
before(async function () {
this.timeout(60000);
({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true }))
})
it('Should display appropriate viewers metrics', async function () {
for (const videoId of [ vodVideoId, liveVideoId ]) {
await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 3 ] })
await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 5 ] })
}
await processViewersStats(servers)
for (const videoId of [ vodVideoId, liveVideoId ]) {
const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' })
expectTimeserieData(result, 2)
}
})
it('Should display appropriate watch time metrics', async function () {
for (const videoId of [ vodVideoId, liveVideoId ]) {
const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'aggregateWatchTime' })
expectTimeserieData(result, 8)
await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] })
}
await processViewersStats(servers)
for (const videoId of [ vodVideoId, liveVideoId ]) {
const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'aggregateWatchTime' })
expectTimeserieData(result, 9)
}
})
after(async function () {
await stopFfmpeg(command)
})
})
after(async function () {
await cleanupTests(servers)
})
})

8
server/tests/api/videos/videos-views-cleaner.ts → server/tests/api/views/videos-views-cleaner.ts
View file

@ -34,10 +34,10 @@ describe('Test video views cleaner', function () {
await waitJobs(servers)
await servers[0].videos.view({ id: videoIdServer1 })
await servers[1].videos.view({ id: videoIdServer1 })
await servers[0].videos.view({ id: videoIdServer2 })
await servers[1].videos.view({ id: videoIdServer2 })
await servers[0].views.simulateView({ id: videoIdServer1 })
await servers[1].views.simulateView({ id: videoIdServer1 })
await servers[0].views.simulateView({ id: videoIdServer2 })
await servers[1].views.simulateView({ id: videoIdServer2 })
await waitJobs(servers)
})

4
server/tests/plugins/action-hooks.ts
View file

@ -1,6 +1,7 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
@ -10,7 +11,6 @@ import {
setAccessTokensToServers,
setDefaultVideoChannel
} from '@shared/server-commands'
import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
describe('Test plugin action hooks', function () {
let servers: PeerTubeServer[]
@ -61,7 +61,7 @@ describe('Test plugin action hooks', function () {
})
it('Should run action:api.video.viewed', async function () {
await servers[0].videos.view({ id: videoUUID })
await servers[0].views.simulateView({ id: videoUUID })
await checkHook('action:api.video.viewed')
})

2
server/tests/plugins/plugin-helpers.ts
View file

@ -301,7 +301,7 @@ describe('Test plugin helpers', function () {
// Should not throw -> video exists
const video = await servers[0].videos.get({ id: videoUUID })
// Should delete the video
await servers[0].videos.view({ id: videoUUID })
await servers[0].views.simulateView({ id: videoUUID })
await servers[0].servers.waitUntilLog('Video deleted by plugin four.')

1
server/tests/shared/index.ts
View file

@ -13,3 +13,4 @@ export * from './streaming-playlists'
export * from './tests'
export * from './tracker'
export * from './videos'
export * from './views'

93
server/tests/shared/views.ts Normal file
View file

@ -0,0 +1,93 @@
import { FfmpegCommand } from 'fluent-ffmpeg'
import { wait } from '@shared/core-utils'
import { VideoCreateResult, VideoPrivacy } from '@shared/models'
import {
createMultipleServers,
doubleFollow,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
waitJobs,
waitUntilLivePublishedOnAllServers
} from '@shared/server-commands'
async function processViewersStats (servers: PeerTubeServer[]) {
await wait(6000)
for (const server of servers) {
await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
await server.debug.sendCommand({ body: { command: 'process-video-viewers' } })
}
await waitJobs(servers)
}
async function processViewsBuffer (servers: PeerTubeServer[]) {
for (const server of servers) {
await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
}
await waitJobs(servers)
}
async function prepareViewsServers () {
const servers = await createMultipleServers(2)
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await servers[0].config.updateCustomSubConfig({
newConfig: {
live: {
enabled: true,
allowReplay: true,
transcoding: {
enabled: false
}
}
}
})
await doubleFollow(servers[0], servers[1])
return servers
}
async function prepareViewsVideos (options: {
servers: PeerTubeServer[]
live: boolean
vod: boolean
}) {
const { servers } = options
const liveAttributes = {
name: 'live video',
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC
}
let ffmpegCommand: FfmpegCommand
let live: VideoCreateResult
let vod: VideoCreateResult
if (options.live) {
live = await servers[0].live.create({ fields: liveAttributes })
ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: live.uuid })
await waitUntilLivePublishedOnAllServers(servers, live.uuid)
}
if (options.vod) {
vod = await servers[0].videos.quickUpload({ name: 'video' })
}
await waitJobs(servers)
return { liveVideoId: live?.uuid, vodVideoId: vod?.uuid, ffmpegCommand }
}
export {
processViewersStats,
prepareViewsServers,
processViewsBuffer,
prepareViewsVideos
}

2
server/types/express.d.ts vendored
View file

@ -185,6 +185,8 @@ declare module 'express' {
externalAuth?: RegisterServerAuthExternalOptions
plugin?: MPlugin
localViewerFull?: MLocalVideoViewerWithWatchSections
}
}
}

2
server/types/models/video/index.ts
View file

@ -1,3 +1,5 @@
export * from './local-video-viewer-watch-section'
export * from './local-video-viewer'
export * from './schedule-video-update'
export * from './tag'
export * from './thumbnail'

5
server/types/models/video/local-video-viewer-watch-section.ts Normal file
View file

@ -0,0 +1,5 @@
import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
// ############################################################################
export type MLocalVideoViewerWatchSection = Omit<LocalVideoViewerWatchSectionModel, 'LocalVideoViewerModel'>

19
server/types/models/video/local-video-viewer.ts Normal file
View file

@ -0,0 +1,19 @@
import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
import { PickWith } from '@shared/typescript-utils'
import { MLocalVideoViewerWatchSection } from './local-video-viewer-watch-section'
import { MVideo } from './video'
type Use<K extends keyof LocalVideoViewerModel, M> = PickWith<LocalVideoViewerModel, K, M>
// ############################################################################
export type MLocalVideoViewer = Omit<LocalVideoViewerModel, 'Video'>
export type MLocalVideoViewerVideo =
MLocalVideoViewer &
Use<'Video', MVideo>
export type MLocalVideoViewerWithWatchSections =
MLocalVideoViewer &
Use<'Video', MVideo> &
Use<'WatchSections', MLocalVideoViewerWatchSection[]>

8
shared/models/activitypub/activity.ts
View file

@ -1,6 +1,6 @@
import { ActivityPubActor } from './activitypub-actor'
import { ActivityPubSignature } from './activitypub-signature'
import { ActivityFlagReasonObject, CacheFileObject, VideoObject } from './objects'
import { ActivityFlagReasonObject, CacheFileObject, VideoObject, WatchActionObject } from './objects'
import { AbuseObject } from './objects/abuse-object'
import { DislikeObject } from './objects/dislike-object'
import { APObject } from './objects/object.model'
@ -52,7 +52,7 @@ export interface BaseActivity {
export interface ActivityCreate extends BaseActivity {
type: 'Create'
object: VideoObject | AbuseObject | DislikeObject | VideoCommentObject | CacheFileObject | PlaylistObject
object: VideoObject | AbuseObject | DislikeObject | VideoCommentObject | CacheFileObject | PlaylistObject | WatchActionObject
}
export interface ActivityUpdate extends BaseActivity {
@ -99,7 +99,9 @@ export interface ActivityView extends BaseActivity {
type: 'View'
actor: string
object: APObject
expires: string
// If sending a "viewer" event
expires?: string
}
export interface ActivityDislike extends BaseActivity {

3
shared/models/activitypub/context.ts
View file

@ -12,4 +12,5 @@ export type ContextType =
'Rate' |
'Flag' |
'Actor' |
'Collection'
'Collection' |
'WatchAction'

1
shared/models/activitypub/objects/index.ts
View file

@ -8,3 +8,4 @@ export * from './playlist-object'
export * from './video-comment-object'
export * from './video-torrent-object'
export * from './view-object'
export * from './watch-action-object'

22
shared/models/activitypub/objects/watch-action-object.ts Normal file
View file

@ -0,0 +1,22 @@
export interface WatchActionObject {
id: string
type: 'WatchAction'
startTime: string
endTime: string
location?: {
addressCountry: string
}
uuid: string
object: string
actionStatus: 'CompletedActionStatus'
duration: string
watchSections: {
startTimestamp: number
endTimestamp: number
}[]
}

2
shared/models/server/debug.model.ts
View file

@ -4,5 +4,5 @@ export interface Debug {
}
export interface SendDebugCommand {
command: 'remove-dandling-resumable-uploads'
command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers'
}

1
shared/models/users/index.ts
View file

@ -12,5 +12,4 @@ export * from './user-scoped-token'
export * from './user-update-me.model'
export * from './user-update.model'
export * from './user-video-quota.model'
export * from './user-watching-video.model'
export * from './user.model'

3
shared/models/users/user-watching-video.model.ts
View file

@ -1,3 +0,0 @@
export interface UserWatchingVideo {
currentTime: number
}

2
shared/models/videos/index.ts
View file

@ -9,6 +9,7 @@ export * from './file'
export * from './import'
export * from './playlist'
export * from './rate'
export * from './stats'
export * from './transcoding'
export * from './nsfw-policy.type'
@ -32,5 +33,6 @@ export * from './video-streaming-playlist.model'
export * from './video-streaming-playlist.type'
export * from './video-update.model'
export * from './video-view.model'
export * from './video.model'
export * from './video-create-result.model'

4
shared/models/videos/stats/index.ts Normal file
View file

@ -0,0 +1,4 @@
export * from './video-stats-overall.model'
export * from './video-stats-retention.model'
export * from './video-stats-timeserie.model'
export * from './video-stats-timeserie-metric.type'

17
shared/models/videos/stats/video-stats-overall.model.ts Normal file
View file

@ -0,0 +1,17 @@
export interface VideoStatsOverall {
averageWatchTime: number
totalWatchTime: number
viewersPeak: number
viewersPeakDate: string
views: number
likes: number
dislikes: number
comments: number
countries: {
isoCode: string
viewers: number
}[]
}

6
shared/models/videos/stats/video-stats-retention.model.ts Normal file
View file

@ -0,0 +1,6 @@
export interface VideoStatsRetention {
data: {
second: number
retentionPercent: number
}[]
}

1
shared/models/videos/stats/video-stats-timeserie-metric.type.ts Normal file
View file

@ -0,0 +1 @@
export type VideoStatsTimeserieMetric = 'viewers' | 'aggregateWatchTime'

6
shared/models/videos/stats/video-stats-timeserie.model.ts Normal file
View file

@ -0,0 +1,6 @@
export interface VideoStatsTimeserie {
data: {
date: string
value: number
}[]
}

6
shared/models/videos/video-view.model.ts Normal file
View file

@ -0,0 +1,6 @@
export type VideoViewEvent = 'seek'
export interface VideoView {
currentTime: number
viewEvent?: VideoViewEvent
}

Some files were not shown because too many files have changed in this diff Show more