From adc94cf09c86112051f72055852efcc977e4a04a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 27 Jul 2022 16:19:25 +0200 Subject: [PATCH] Add live and viewers otel metrics --- .../lib/opentelemetry/metric-helpers/index.ts | 2 ++ .../metric-helpers/lives-observers-builder.ts | 21 +++++++++++++ .../viewers-observers-builder.ts | 24 ++++++++++++++ server/lib/opentelemetry/metrics.ts | 14 ++++++++- .../lib/views/shared/video-viewer-counters.ts | 31 ++++++++++++++++--- server/lib/views/video-views-manager.ts | 9 +++++- .../validators/videos/video-live.ts | 2 +- server/models/video/video.ts | 17 +++++----- 8 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts create mode 100644 server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts index ff0aff9fd..1b3813743 100644 --- a/server/lib/opentelemetry/metric-helpers/index.ts +++ b/server/lib/opentelemetry/metric-helpers/index.ts @@ -1,3 +1,5 @@ +export * from './lives-observers-builder' export * from './job-queue-observers-builder' export * from './nodejs-observers-builder' export * from './stats-observers-builder' +export * from './viewers-observers-builder' diff --git a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts new file mode 100644 index 000000000..e27bd8548 --- /dev/null +++ b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts @@ -0,0 +1,21 @@ +import { Meter } from '@opentelemetry/api-metrics' +import { VideoModel } from '@server/models/video/video' + +export class LivesObserversBuilder { + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.meter.createObservableGauge('peertube_running_lives_total', { + description: 'Total running lives on the instance' + }).addCallback(async observableResult => { + const local = await VideoModel.countLives({ remote: false, mode: 'published' }) + const remote = await VideoModel.countLives({ remote: true, mode: 'published' }) + + observableResult.observe(local, { liveOrigin: 'local' }) + observableResult.observe(remote, { liveOrigin: 'remote' }) + }) + } +} diff --git a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts new file mode 100644 index 000000000..634e5bbc9 --- /dev/null +++ b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts @@ -0,0 +1,24 @@ +import { Meter } from '@opentelemetry/api-metrics' +import { VideoScope, ViewerScope } from '@server/lib/views/shared' +import { VideoViewsManager } from '@server/lib/views/video-views-manager' + +export class ViewersObserversBuilder { + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.meter.createObservableGauge('peertube_viewers_total', { + description: 'Total viewers on the instance' + }).addCallback(observableResult => { + for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) { + for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) { + const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope }) + + observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope }) + } + } + }) + } +} diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts index 149f421be..ffe493670 100644 --- a/server/lib/opentelemetry/metrics.ts +++ b/server/lib/opentelemetry/metrics.ts @@ -4,7 +4,13 @@ import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' import { MeterProvider } from '@opentelemetry/sdk-metrics-base' import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' -import { JobQueueObserversBuilder, NodeJSObserversBuilder, StatsObserversBuilder } from './metric-helpers' +import { + JobQueueObserversBuilder, + LivesObserversBuilder, + NodeJSObserversBuilder, + StatsObserversBuilder, + ViewersObserversBuilder +} from './metric-helpers' class OpenTelemetryMetrics { @@ -53,6 +59,12 @@ class OpenTelemetryMetrics { const statsObserversBuilder = new StatsObserversBuilder(this.meter) statsObserversBuilder.buildObservers() + + const livesObserversBuilder = new LivesObserversBuilder(this.meter) + livesObserversBuilder.buildObservers() + + const viewersObserversBuilder = new ViewersObserversBuilder(this.meter) + viewersObserversBuilder.buildObservers() } private buildRequestObserver () { diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts index f851ce050..f5b83130e 100644 --- a/server/lib/views/shared/video-viewer-counters.ts +++ b/server/lib/views/shared/video-viewer-counters.ts @@ -10,9 +10,14 @@ import { buildUUID, sha256 } from '@shared/extra-utils' const lTags = loggerTagsFactory('views') +export type ViewerScope = 'local' | 'remote' +export type VideoScope = 'local' | 'remote' + type Viewer = { expires: number id: string + viewerScope: ViewerScope + videoScope: VideoScope lastFederation?: number } @@ -50,7 +55,7 @@ export class VideoViewerCounters { return false } - const newViewer = await this.addViewerToVideo({ viewerId, video }) + const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' }) await this.federateViewerIfNeeded(video, newViewer) return true @@ -65,13 +70,26 @@ export class VideoViewerCounters { logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) - await this.addViewerToVideo({ video, viewerExpires, viewerId }) + await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' }) return true } // --------------------------------------------------------------------------- + getTotalViewers (options: { + viewerScope: ViewerScope + videoScope: VideoScope + }) { + let total = 0 + + for (const viewers of this.viewersPerVideo.values()) { + total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length + } + + return total + } + getViewers (video: MVideo) { const viewers = this.viewersPerVideo.get(video.id) if (!viewers) return 0 @@ -88,9 +106,10 @@ export class VideoViewerCounters { private async addViewerToVideo (options: { video: MVideoImmutable viewerId: string + viewerScope: ViewerScope viewerExpires?: Date }) { - const { video, viewerExpires, viewerId } = options + const { video, viewerExpires, viewerId, viewerScope } = options let watchers = this.viewersPerVideo.get(video.id) @@ -103,7 +122,11 @@ export class VideoViewerCounters { ? viewerExpires.getTime() : this.buildViewerExpireTime() - const viewer = { id: viewerId, expires } + const videoScope: VideoScope = video.remote + ? 'remote' + : 'local' + + const viewer = { id: viewerId, expires, videoScope, viewerScope } watchers.push(viewer) this.idToViewer.set(viewerId, viewer) diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts index 86758e8d8..c088dad5e 100644 --- a/server/lib/views/video-views-manager.ts +++ b/server/lib/views/video-views-manager.ts @@ -1,7 +1,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' import { MVideo, MVideoImmutable } from '@server/types/models' import { VideoViewEvent } from '@shared/models' -import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared' +import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared' /** * If processing a local view: @@ -79,6 +79,13 @@ export class VideoViewsManager { return this.videoViewerCounters.getViewers(video) } + getTotalViewers (options: { + viewerScope: ViewerScope + videoScope: VideoScope + }) { + return this.videoViewerCounters.getTotalViewers(options) + } + buildViewerExpireTime () { return this.videoViewerCounters.buildViewerExpireTime() } diff --git a/server/middlewares/validators/videos/video-live.ts b/server/middlewares/validators/videos/video-live.ts index 59638d5e0..777b57e9a 100644 --- a/server/middlewares/validators/videos/video-live.ts +++ b/server/middlewares/validators/videos/video-live.ts @@ -119,7 +119,7 @@ const videoLiveAddValidator = getCommonVideoEditAttributes().concat([ if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req) if (CONFIG.LIVE.MAX_INSTANCE_LIVES !== -1) { - const totalInstanceLives = await VideoModel.countLocalLives() + const totalInstanceLives = await VideoModel.countLives({ remote: false, mode: 'not-ended' }) if (totalInstanceLives >= CONFIG.LIVE.MAX_INSTANCE_LIVES) { cleanUpReqFiles(req) diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 27e605be6..924f12a5e 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1209,18 +1209,21 @@ export class VideoModel extends Model>> { return VideoModel.getAvailableForApi(queryOptions) } - static countLocalLives () { - const options = { + static countLives (options: { + remote: boolean + mode: 'published' | 'not-ended' + }) { + const query = { where: { - remote: false, + remote: options.remote, isLive: true, - state: { - [Op.ne]: VideoState.LIVE_ENDED - } + state: options.mode === 'not-ended' + ? { [Op.ne]: VideoState.LIVE_ENDED } + : { [Op.eq]: VideoState.PUBLISHED } } } - return VideoModel.count(options) + return VideoModel.count(query) } static countVideosUploadedByUserSince (userId: number, since: Date) {