From 71e3e879c0616882ee82a0e44f8c2e5ee9698a3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 2 Dec 2022 14:47:21 +0100 Subject: [PATCH] Support reinjecting token in private m3u8 playlist --- .../src/suites-all/private-videos.e2e-spec.ts | 15 ++++ client/e2e/src/utils/urls.ts | 4 + .../manager-options/hls-options-builder.ts | 1 + .../p2p-media-loader-plugin.ts | 78 ++++++++++--------- .../p2p-media-loader/segment-validator.ts | 3 +- .../player/types/peertube-videojs-typings.ts | 1 + server/controllers/object-storage-proxy.ts | 20 ++++- server/controllers/shared/m3u8-playlist.ts | 14 ++++ server/controllers/static.ts | 39 ++++++++++ server/helpers/stream-replacer.ts | 58 ++++++++++++++ server/lib/hls.ts | 9 ++- server/middlewares/validators/static.ts | 11 ++- server/middlewares/validators/users.ts | 2 +- .../video-static-file-privacy.ts | 36 ++++++++- .../api/videos/video-static-file-privacy.ts | 61 ++++++++++++++- server/tests/shared/checks.ts | 7 ++ server/tests/shared/index.ts | 2 +- server/tests/shared/streaming-playlists.ts | 50 +++++++++++- .../{playlists.ts => video-playlists.ts} | 0 shared/core-utils/common/url.ts | 9 +++ .../videos/streaming-playlists-command.ts | 10 ++- support/doc/api/openapi.yaml | 9 ++- 22 files changed, 391 insertions(+), 48 deletions(-) create mode 100644 server/controllers/shared/m3u8-playlist.ts create mode 100644 server/helpers/stream-replacer.ts rename server/tests/shared/{playlists.ts => video-playlists.ts} (100%) diff --git a/client/e2e/src/suites-all/private-videos.e2e-spec.ts b/client/e2e/src/suites-all/private-videos.e2e-spec.ts index db3554659..a25208bb3 100644 --- a/client/e2e/src/suites-all/private-videos.e2e-spec.ts +++ b/client/e2e/src/suites-all/private-videos.e2e-spec.ts @@ -15,6 +15,7 @@ describe('Private videos all workflow', () => { let playerPage: PlayerPage const internalVideoName = 'Internal E2E test' + const internalHLSOnlyVideoName = 'Internal E2E test - HLS only' beforeEach(async () => { videoWatchPage = new VideoWatchPage(isMobileDevice(), isSafari()) @@ -44,6 +45,13 @@ describe('Private videos all workflow', () => { await checkCorrectlyPlay(playerPage) }) + it('Should play an internal HLS only video', async () => { + await go(FIXTURE_URLS.INTERNAL_HLS_ONLY_VIDEO) + + await videoWatchPage.waitWatchVideoName(internalHLSOnlyVideoName) + await checkCorrectlyPlay(playerPage) + }) + it('Should play an internal WebTorrent video in embed', async () => { await go(FIXTURE_URLS.INTERNAL_EMBED_WEBTORRENT_VIDEO) @@ -57,4 +65,11 @@ describe('Private videos all workflow', () => { await videoWatchPage.waitEmbedForDisplayed() await checkCorrectlyPlay(playerPage) }) + + it('Should play an internal HLS only video in embed', async () => { + await go(FIXTURE_URLS.INTERNAL_EMBED_HLS_ONLY_VIDEO) + + await videoWatchPage.waitEmbedForDisplayed() + await checkCorrectlyPlay(playerPage) + }) }) diff --git a/client/e2e/src/utils/urls.ts b/client/e2e/src/utils/urls.ts index f91d9a048..cc0bdfbff 100644 --- a/client/e2e/src/utils/urls.ts +++ b/client/e2e/src/utils/urls.ts @@ -1,9 +1,13 @@ const FIXTURE_URLS = { INTERNAL_WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/pwfz7NizSdPD4mJcbbmNwa?mode=webtorrent&start=0', INTERNAL_HLS_VIDEO: 'https://peertube2.cpy.re/w/pwfz7NizSdPD4mJcbbmNwa?start=0', + INTERNAL_EMBED_WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/videos/embed/pwfz7NizSdPD4mJcbbmNwa?mode=webtorrent&start=0', INTERNAL_EMBED_HLS_VIDEO: 'https://peertube2.cpy.re/videos/embed/pwfz7NizSdPD4mJcbbmNwa?start=0', + INTERNAL_HLS_ONLY_VIDEO: 'https://peertube2.cpy.re/w/tKQmHcqdYZRdCszLUiWM3V?start=0', + INTERNAL_EMBED_HLS_ONLY_VIDEO: 'https://peertube2.cpy.re/videos/embed/tKQmHcqdYZRdCszLUiWM3V?start=0', + WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/122d093a-1ede-43bd-bd34-59d2931ffc5e', HLS_EMBED: 'https://peertube2.cpy.re/videos/embed/969bf103-7818-43b5-94a0-de159e13de50', diff --git a/client/src/assets/player/shared/manager-options/hls-options-builder.ts b/client/src/assets/player/shared/manager-options/hls-options-builder.ts index 497a97436..63e9fa8c8 100644 --- a/client/src/assets/player/shared/manager-options/hls-options-builder.ts +++ b/client/src/assets/player/shared/manager-options/hls-options-builder.ts @@ -32,6 +32,7 @@ export class HLSOptionsBuilder { const p2pMediaLoader: P2PMediaLoaderPluginOptions = { requiresAuth: commonOptions.requiresAuth, + videoFileToken: commonOptions.videoFileToken, redundancyUrlManager, type: 'application/x-mpegURL', diff --git a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts index b608ee3e2..e6f525fea 100644 --- a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts +++ b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts @@ -3,7 +3,7 @@ import videojs from 'video.js' import { Events, Segment } from '@peertube/p2p-media-loader-core' import { Engine, initHlsJsPlayer, initVideoJsContribHlsJsPlayer } from '@peertube/p2p-media-loader-hlsjs' import { logger } from '@root-helpers/logger' -import { timeToInt } from '@shared/core-utils' +import { addQueryParams, timeToInt } from '@shared/core-utils' import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo } from '../../types' import { registerConfigPlugin, registerSourceHandler } from './hls-plugin' @@ -39,46 +39,37 @@ class P2pMediaLoaderPlugin extends Plugin { super(player) this.options = options + this.startTime = timeToInt(options.startTime) // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080 if (!(videojs as any).Html5Hlsjs) { - logger.warn('HLS.js does not seem to be supported. Try to fallback to built in HLS.') - - let message: string - if (!player.canPlayType('application/vnd.apple.mpegurl')) { - message = 'Cannot fallback to built-in HLS' - } else if (options.requiresAuth) { - message = 'Video requires auth which is not compatible to build-in HLS player' - } - - if (message) { - logger.warn(message) - - const error: MediaError = { - code: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED, - message, - MEDIA_ERR_ABORTED: MediaError.MEDIA_ERR_ABORTED, - MEDIA_ERR_DECODE: MediaError.MEDIA_ERR_DECODE, - MEDIA_ERR_NETWORK: MediaError.MEDIA_ERR_NETWORK, - MEDIA_ERR_SRC_NOT_SUPPORTED: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED - } - - player.ready(() => player.error(error)) + if (player.canPlayType('application/vnd.apple.mpegurl')) { + this.fallbackToBuiltInIOS() return } - // Workaround to force video.js to not re create a video element - (this.player as any).playerElIngest_ = this.player.el().parentNode - } else { - // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080 - (videojs as any).Html5Hlsjs.addHook('beforeinitialize', (videojsPlayer: any, hlsjs: any) => { - this.hlsjs = hlsjs - }) + const message = 'HLS.js does not seem to be supported. Cannot fallback to built-in HLS' + logger.warn(message) - initVideoJsContribHlsJsPlayer(player) + const error: MediaError = { + code: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED, + message, + MEDIA_ERR_ABORTED: MediaError.MEDIA_ERR_ABORTED, + MEDIA_ERR_DECODE: MediaError.MEDIA_ERR_DECODE, + MEDIA_ERR_NETWORK: MediaError.MEDIA_ERR_NETWORK, + MEDIA_ERR_SRC_NOT_SUPPORTED: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED + } + + player.ready(() => player.error(error)) + return } - this.startTime = timeToInt(options.startTime) + // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080 + (videojs as any).Html5Hlsjs.addHook('beforeinitialize', (_videojsPlayer: any, hlsjs: any) => { + this.hlsjs = hlsjs + }) + + initVideoJsContribHlsJsPlayer(player) player.src({ type: options.type, @@ -88,9 +79,7 @@ class P2pMediaLoaderPlugin extends Plugin { player.ready(() => { this.initializeCore() - if ((videojs as any).Html5Hlsjs) { - this.initializePlugin() - } + this.initializePlugin() }) } @@ -199,6 +188,25 @@ class P2pMediaLoaderPlugin extends Plugin { private arraySum (data: number[]) { return data.reduce((a: number, b: number) => a + b, 0) } + + private fallbackToBuiltInIOS () { + logger.info('HLS.js does not seem to be supported. Fallback to built-in HLS.'); + + // Workaround to force video.js to not re create a video element + (this.player as any).playerElIngest_ = this.player.el().parentNode + + this.player.src({ + type: this.options.type, + src: addQueryParams(this.options.src, { + videoFileToken: this.options.videoFileToken(), + reinjectVideoFileToken: 'true' + }) + }) + + this.player.ready(() => { + this.initializeCore() + }) + } } videojs.registerPlugin('p2pMediaLoader', P2pMediaLoaderPlugin) diff --git a/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts b/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts index a7ee91950..3c76d63f7 100644 --- a/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts +++ b/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts @@ -2,6 +2,7 @@ import { basename } from 'path' import { Segment } from '@peertube/p2p-media-loader-core' import { logger } from '@root-helpers/logger' import { wait } from '@root-helpers/utils' +import { removeQueryParams } from '@shared/core-utils' import { isSameOrigin } from '../common' type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } } @@ -24,7 +25,7 @@ function segmentValidatorFactory (options: { // Wait for hash generation from the server if (isLive) await wait(1000) - const filename = basename(segment.url) + const filename = basename(removeQueryParams(segment.url)) const segmentValue = (await segmentsJSON)[filename] diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts index 3d9d5270e..c60154f3b 100644 --- a/client/src/assets/player/types/peertube-videojs-typings.ts +++ b/client/src/assets/player/types/peertube-videojs-typings.ts @@ -168,6 +168,7 @@ type P2PMediaLoaderPluginOptions = { loader: P2PMediaLoader requiresAuth: boolean + videoFileToken: () => string } export type P2PMediaLoader = { diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts index 3ce279671..aa853a383 100644 --- a/server/controllers/object-storage-proxy.ts +++ b/server/controllers/object-storage-proxy.ts @@ -1,7 +1,10 @@ import cors from 'cors' import express from 'express' +import { PassThrough, pipeline } from 'stream' import { logger } from '@server/helpers/logger' +import { StreamReplacer } from '@server/helpers/stream-replacer' import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' +import { injectQueryToPlaylistUrls } from '@server/lib/hls' import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' import { asyncMiddleware, @@ -11,6 +14,7 @@ import { optionalAuthenticate } from '@server/middlewares' import { HttpStatusCode } from '@shared/models' +import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' const objectStorageProxyRouter = express.Router() @@ -67,7 +71,20 @@ async function proxifyHLS (req: express.Request, res: express.Response) { rangeHeader: req.header('range') }) - return stream.pipe(res) + const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) + ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req))) + : new PassThrough() + + return pipeline( + stream, + streamReplacer, + res, + err => { + if (!err) return + + handleObjectStorageFailure(res, err) + } + ) } catch (err) { return handleObjectStorageFailure(res, err) } @@ -75,6 +92,7 @@ async function proxifyHLS (req: express.Request, res: express.Response) { function handleObjectStorageFailure (res: express.Response, err: Error) { if (err.name === 'NoSuchKey') { + logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) } diff --git a/server/controllers/shared/m3u8-playlist.ts b/server/controllers/shared/m3u8-playlist.ts new file mode 100644 index 000000000..e2a66efc0 --- /dev/null +++ b/server/controllers/shared/m3u8-playlist.ts @@ -0,0 +1,14 @@ +import express from 'express' + +function doReinjectVideoFileToken (req: express.Request) { + return req.query.videoFileToken && req.query.reinjectVideoFileToken +} + +function buildReinjectVideoFileTokenQuery (req: express.Request) { + return 'videoFileToken=' + req.query.videoFileToken +} + +export { + doReinjectVideoFileToken, + buildReinjectVideoFileTokenQuery +} diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 6ef9154b9..52e48267f 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -1,5 +1,8 @@ import cors from 'cors' import express from 'express' +import { readFile } from 'fs-extra' +import { join } from 'path' +import { injectQueryToPlaylistUrls } from '@server/lib/hls' import { asyncMiddleware, ensureCanAccessPrivateVideoHLSFiles, @@ -7,8 +10,10 @@ import { handleStaticError, optionalAuthenticate } from '@server/middlewares' +import { HttpStatusCode } from '@shared/models' import { CONFIG } from '../initializers/config' import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' +import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' const staticRouter = express.Router() @@ -49,6 +54,12 @@ const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AU ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] : [] +staticRouter.use( + STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8', + ...privateHLSStaticMiddlewares, + asyncMiddleware(servePrivateM3U8) +) + staticRouter.use( STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, ...privateHLSStaticMiddlewares, @@ -74,3 +85,31 @@ staticRouter.use( export { staticRouter } + +// --------------------------------------------------------------------------- + +async function servePrivateM3U8 (req: express.Request, res: express.Response) { + const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8') + + let playlistContent: string + + try { + playlistContent = await readFile(path, 'utf-8') + } catch (err) { + if (err.message.includes('ENOENT')) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'File not found' + }) + } + + throw err + } + + // Inject token in playlist so players that cannot alter the HTTP request can still watch the video + const transformedContent = doReinjectVideoFileToken(req) + ? injectQueryToPlaylistUrls(playlistContent, buildReinjectVideoFileTokenQuery(req)) + : playlistContent + + return res.set('content-type', 'application/vnd.apple.mpegurl').send(transformedContent).end() +} diff --git a/server/helpers/stream-replacer.ts b/server/helpers/stream-replacer.ts new file mode 100644 index 000000000..4babab418 --- /dev/null +++ b/server/helpers/stream-replacer.ts @@ -0,0 +1,58 @@ +import { Transform, TransformCallback } from 'stream' + +// Thanks: https://stackoverflow.com/a/45126242 +class StreamReplacer extends Transform { + private pendingChunk: Buffer + + constructor (private readonly replacer: (line: string) => string) { + super() + } + + _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) { + try { + this.pendingChunk = this.pendingChunk?.length + ? Buffer.concat([ this.pendingChunk, chunk ]) + : chunk + + let index: number + + // As long as we keep finding newlines, keep making slices of the buffer and push them to the + // readable side of the transform stream + while ((index = this.pendingChunk.indexOf('\n')) !== -1) { + // The `end` parameter is non-inclusive, so increase it to include the newline we found + const line = this.pendingChunk.slice(0, ++index) + + // `start` is inclusive, but we are already one char ahead of the newline -> all good + this.pendingChunk = this.pendingChunk.slice(index) + + // We have a single line here! Prepend the string we want + this.push(this.doReplace(line)) + } + + return done() + } catch (err) { + return done(err) + } + } + + _flush (done: TransformCallback) { + // If we have any remaining data in the cache, send it out + if (!this.pendingChunk?.length) return done() + + try { + return done(null, this.doReplace(this.pendingChunk)) + } catch (err) { + return done(err) + } + } + + private doReplace (buffer: Buffer) { + const line = this.replacer(buffer.toString('utf8')) + + return Buffer.from(line, 'utf8') + } +} + +export { + StreamReplacer +} diff --git a/server/lib/hls.ts b/server/lib/hls.ts index a41f1ae48..053b5d326 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -234,13 +234,20 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, // --------------------------------------------------------------------------- +function injectQueryToPlaylistUrls (content: string, queryString: string) { + return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) +} + +// --------------------------------------------------------------------------- + export { updateMasterHLSPlaylist, updateSha256VODSegments, buildSha256Segment, downloadPlaylistSegments, updateStreamingPlaylistsInfohashesIfNeeded, - updatePlaylistAfterFileChange + updatePlaylistAfterFileChange, + injectQueryToPlaylistUrls } // --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts index 13fde6dd1..d3d307787 100644 --- a/server/middlewares/validators/static.ts +++ b/server/middlewares/validators/static.ts @@ -2,7 +2,7 @@ import express from 'express' import { query } from 'express-validator' import LRUCache from 'lru-cache' import { basename, dirname } from 'path' -import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc' +import { exists, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc' import { logger } from '@server/helpers/logger' import { LRU_CACHE } from '@server/initializers/constants' import { VideoModel } from '@server/models/video/video' @@ -60,7 +60,14 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [ ] const ensureCanAccessPrivateVideoHLSFiles = [ - query('videoFileToken').optional().custom(exists), + query('videoFileToken') + .optional() + .custom(exists), + + query('reinjectVideoFileToken') + .optional() + .customSanitizer(toBooleanOrNull) + .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 50327b6ae..64bd9ca70 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -44,7 +44,7 @@ const usersListValidator = [ query('blocked') .optional() .customSanitizer(toBooleanOrNull) - .isBoolean().withMessage('Should be a valid blocked boolena'), + .isBoolean().withMessage('Should be a valid blocked boolean'), (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts index 62edd10ba..71ad35a43 100644 --- a/server/tests/api/object-storage/video-static-file-privacy.ts +++ b/server/tests/api/object-storage/video-static-file-privacy.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { basename } from 'path' -import { expectStartWith } from '@server/tests/shared' +import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared' import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' import { @@ -191,6 +191,20 @@ describe('Object storage for video static file privacy', function () { } }) + it('Should reinject video file token', async function () { + this.timeout(120000) + + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) + + await checkVideoFileTokenReinjection({ + server, + videoUUID: privateVideoUUID, + videoFileToken, + resolutions: [ 240, 720 ], + isLive: false + }) + }) + it('Should update public video to private', async function () { this.timeout(60000) @@ -315,6 +329,26 @@ describe('Object storage for video static file privacy', function () { await checkLiveFiles(permanentLive, permanentLiveId) }) + it('Should reinject video file token in permanent live', async function () { + this.timeout(240000) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) + await server.live.waitUntilPublished({ videoId: permanentLiveId }) + + const video = await server.videos.getWithToken({ id: permanentLiveId }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + + await checkVideoFileTokenReinjection({ + server, + videoUUID: permanentLiveId, + videoFileToken, + resolutions: [ 720 ], + isLive: true + }) + + await stopFfmpeg(ffmpegCommand) + }) + it('Should have created a replay of the normal live with a private static path', async function () { this.timeout(240000) diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts index eaaed5aad..ef0774b41 100644 --- a/server/tests/api/videos/video-static-file-privacy.ts +++ b/server/tests/api/videos/video-static-file-privacy.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { decode } from 'magnet-uri' -import { expectStartWith } from '@server/tests/shared' +import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared' import { getAllFiles, wait } from '@shared/core-utils' import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' import { @@ -248,6 +248,35 @@ describe('Test video static file privacy', function () { await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) }) + it('Should reinject video file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + await waitJobs([ server ]) + + const video = await server.videos.getWithToken({ id: uuid }) + const hls = video.streamingPlaylists[0] + + { + const query = { videoFileToken } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text).to.not.include(videoFileToken) + } + + { + await checkVideoFileTokenReinjection({ + server, + videoUUID: uuid, + videoFileToken, + resolutions: [ 240, 720 ], + isLive: false + }) + } + }) + it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () { this.timeout(120000) @@ -360,6 +389,36 @@ describe('Test video static file privacy', function () { await checkLiveFiles(permanentLive, permanentLiveId) }) + it('Should reinject video file token on permanent live', async function () { + this.timeout(240000) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) + await server.live.waitUntilPublished({ videoId: permanentLiveId }) + + const video = await server.videos.getWithToken({ id: permanentLiveId }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + const hls = video.streamingPlaylists[0] + + { + const query = { videoFileToken } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text).to.not.include(videoFileToken) + } + + { + await checkVideoFileTokenReinjection({ + server, + videoUUID: permanentLiveId, + videoFileToken, + resolutions: [ 720 ], + isLive: true + }) + } + + await stopFfmpeg(ffmpegCommand) + }) + it('Should have created a replay of the normal live with a private static path', async function () { this.timeout(240000) diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index 55ebc6c3e..523d37420 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -23,6 +23,12 @@ function expectNotStartWith (str: string, start: string) { expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false } +function expectEndWith (str: string, end: string) { + expect(str.endsWith(end), `${str} does not end with ${end}`).to.be.true +} + +// --------------------------------------------------------------------------- + async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { const content = await server.servers.getLogContent() @@ -103,6 +109,7 @@ export { testFileExistsOrNot, expectStartWith, expectNotStartWith, + expectEndWith, checkBadStartPagination, checkBadCountPagination, checkBadSortPagination, diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts index 9f7ade53d..963ef8fe6 100644 --- a/server/tests/shared/index.ts +++ b/server/tests/shared/index.ts @@ -6,7 +6,7 @@ export * from './directories' export * from './generate' export * from './live' export * from './notifications' -export * from './playlists' +export * from './video-playlists' export * from './plugins' export * from './requests' export * from './streaming-playlists' diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts index 824c3dcef..5c62af812 100644 --- a/server/tests/shared/streaming-playlists.ts +++ b/server/tests/shared/streaming-playlists.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { basename } from 'path' +import { basename, dirname, join } from 'path' import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' import { sha256 } from '@shared/extra-utils' import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models' @@ -188,9 +188,55 @@ async function completeCheckHlsPlaylist (options: { } } +async function checkVideoFileTokenReinjection (options: { + server: PeerTubeServer + videoUUID: string + videoFileToken: string + resolutions: number[] + isLive: boolean +}) { + const { server, resolutions, videoFileToken, videoUUID, isLive } = options + + const video = await server.videos.getWithToken({ id: videoUUID }) + const hls = video.streamingPlaylists[0] + + const query = { videoFileToken, reinjectVideoFileToken: 'true' } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + for (let i = 0; i < resolutions.length; i++) { + const resolution = resolutions[i] + + const suffix = isLive + ? i + : `-${resolution}` + + expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}`) + } + + const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text) + expect(resolutionPlaylists).to.have.lengthOf(resolutions.length) + + for (const url of resolutionPlaylists) { + const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 }) + + const extension = isLive + ? '.ts' + : '.mp4' + + expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`) + } +} + +function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) { + return masterContent.match(/^([^.]+\.m3u8.*)/mg) + .map(filename => join(dirname(masterPath), filename)) +} + export { checkSegmentHash, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, - completeCheckHlsPlaylist + completeCheckHlsPlaylist, + extractResolutionPlaylistUrls, + checkVideoFileTokenReinjection } diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/video-playlists.ts similarity index 100% rename from server/tests/shared/playlists.ts rename to server/tests/shared/video-playlists.ts diff --git a/shared/core-utils/common/url.ts b/shared/core-utils/common/url.ts index d1c399f7b..33fc5ee3a 100644 --- a/shared/core-utils/common/url.ts +++ b/shared/core-utils/common/url.ts @@ -11,6 +11,14 @@ function addQueryParams (url: string, params: { [ id: string ]: string }) { return objUrl.toString() } +function removeQueryParams (url: string) { + const objUrl = new URL(url) + + objUrl.searchParams.forEach((_v, k) => objUrl.searchParams.delete(k)) + + return objUrl.toString() +} + function buildPlaylistLink (playlist: Pick, base?: string) { return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist) } @@ -114,6 +122,7 @@ function decoratePlaylistLink (options: { export { addQueryParams, + removeQueryParams, buildPlaylistLink, buildVideoLink, diff --git a/shared/server-commands/videos/streaming-playlists-command.ts b/shared/server-commands/videos/streaming-playlists-command.ts index 25e446e72..26ab2735f 100644 --- a/shared/server-commands/videos/streaming-playlists-command.ts +++ b/shared/server-commands/videos/streaming-playlists-command.ts @@ -7,16 +7,24 @@ export class StreamingPlaylistsCommand extends AbstractCommand { async get (options: OverrideCommandOptions & { url: string + + videoFileToken?: string + reinjectVideoFileToken?: boolean + withRetry?: boolean // default false currentRetry?: number }) { - const { withRetry, currentRetry = 1 } = options + const { videoFileToken, reinjectVideoFileToken, withRetry, currentRetry = 1 } = options try { const result = await unwrapTextOrDecode(this.getRawRequest({ ...options, url: options.url, + query: { + videoFileToken, + reinjectVideoFileToken + }, implicitToken: false, defaultExpectedStatus: HttpStatusCode.OK_200 })) diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 2062f2e3a..c2f9d424e 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -408,6 +408,7 @@ paths: parameters: - $ref: '#/components/parameters/staticFilename' - $ref: '#/components/parameters/videoFileToken' + - $ref: '#/components/parameters/reinjectVideoFileToken' security: - OAuth2: [] responses: @@ -5711,7 +5712,13 @@ components: description: Video file token [generated](#operation/requestVideoToken) by PeerTube so you don't need to provide an OAuth token in the request header. schema: type: string - + reinjectVideoFileToken: + name: reinjectVideoFileToken + in: query + required: false + description: Ask the server to reinject videoFileToken in URLs in m3u8 playlist + schema: + type: boolean securitySchemes: OAuth2: