From 1022e273092f539c0b77a5cf6411b06c74fd15e5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 1 Sep 2023 16:47:25 +0200 Subject: [PATCH] Fix live replay privacy change --- packages/ffmpeg/src/ffmpeg-vod.ts | 4 +- packages/tests/src/api/live/index.ts | 1 + .../tests/src/api/live/live-privacy-update.ts | 83 +++++++++++++++++++ .../job-queue/handlers/video-live-ending.ts | 59 +++++++++---- .../server/lib/transcoding/hls-transcoding.ts | 17 +++- 5 files changed, 144 insertions(+), 20 deletions(-) create mode 100644 packages/tests/src/api/live/live-privacy-update.ts diff --git a/packages/ffmpeg/src/ffmpeg-vod.ts b/packages/ffmpeg/src/ffmpeg-vod.ts index 6dd272b8d..373dc6e81 100644 --- a/packages/ffmpeg/src/ffmpeg-vod.ts +++ b/packages/ffmpeg/src/ffmpeg-vod.ts @@ -102,7 +102,9 @@ export class FFmpegVOD { command.on('start', () => { setTimeout(() => { - options.inputFileMutexReleaser() + if (options.inputFileMutexReleaser) { + options.inputFileMutexReleaser() + } }, 1000) }) diff --git a/packages/tests/src/api/live/index.ts b/packages/tests/src/api/live/index.ts index e61e6c611..bb5177d95 100644 --- a/packages/tests/src/api/live/index.ts +++ b/packages/tests/src/api/live/index.ts @@ -1,6 +1,7 @@ import './live-constraints.js' import './live-fast-restream.js' import './live-socket-messages.js' +import './live-privacy-update.js' import './live-permanent.js' import './live-rtmps.js' import './live-save-replay.js' diff --git a/packages/tests/src/api/live/live-privacy-update.ts b/packages/tests/src/api/live/live-privacy-update.ts new file mode 100644 index 000000000..ff12ff3e3 --- /dev/null +++ b/packages/tests/src/api/live/live-privacy-update.ts @@ -0,0 +1,83 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, createSingleServer, makeRawRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs, + waitUntilLivePublishedOnAllServers, + waitUntilLiveReplacedByReplayOnAllServers +} from '@peertube/peertube-server-commands' + +async function testVideoFiles (server: PeerTubeServer, uuid: string) { + const video = await server.videos.getWithToken({ id: uuid }) + + const expectedStatus = HttpStatusCode.OK_200 + + await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, token: server.accessToken, expectedStatus }) + await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, token: server.accessToken, expectedStatus }) +} + +describe('Live privacy update', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableMinimumTranscoding() + await server.config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) + }) + + describe('Normal live', function () { + let uuid: string + + it('Should create a public live with private replay', async function () { + this.timeout(120000) + + const fields: LiveVideoCreate = { + name: 'live', + privacy: VideoPrivacy.PUBLIC, + permanentLive: false, + replaySettings: { privacy: VideoPrivacy.PRIVATE }, + saveReplay: true, + channelId: server.store.channel.id + } + + const video = await server.live.create({ fields }) + uuid = video.uuid + + const ffmpegCommand = await server.live.sendRTMPStreamInVideo({ videoId: uuid }) + await waitUntilLivePublishedOnAllServers([ server ], uuid) + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveReplacedByReplayOnAllServers([ server ], uuid) + await waitJobs([ server ]) + + await testVideoFiles(server, uuid) + }) + + it('Should update the replay to public and re-update it to private', async function () { + this.timeout(120000) + + await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + await waitJobs([ server ]) + await testVideoFiles(server, uuid) + + await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PRIVATE } }) + await waitJobs([ server ]) + await testVideoFiles(server, uuid) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/server/lib/job-queue/handlers/video-live-ending.ts b/server/server/lib/job-queue/handlers/video-live-ending.ts index 0b4a4fd8b..f10cc763c 100644 --- a/server/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/server/lib/job-queue/handlers/video-live-ending.ts @@ -8,7 +8,12 @@ import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live/index.js' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths.js' +import { + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + getHLSDirectory, + getLiveReplayBaseDirectory +} from '@server/lib/paths.js' import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js' import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js' import { VideoPathManager } from '@server/lib/video-path-manager.js' @@ -24,6 +29,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' import { logger, loggerTagsFactory } from '../../../helpers/logger.js' import { JobQueue } from '../job-queue.js' +import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js' const lTags = loggerTagsFactory('live', 'job') @@ -139,9 +145,15 @@ async function saveReplayToExternalVideo (options: { }) } - await assignReplayFilesToVideo({ video: replayVideo, replayDirectory }) + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(liveVideo.uuid) - await remove(replayDirectory) + try { + await assignReplayFilesToVideo({ video: replayVideo, replayDirectory }) + + await remove(replayDirectory) + } finally { + inputFileMutexReleaser() + } for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) @@ -160,11 +172,14 @@ async function replaceLiveByReplay (options: { permanentLive: boolean replayDirectory: string }) { - const { video, liveSession, live, permanentLive, replayDirectory } = options + const { video: liveVideo, liveSession, live, permanentLive, replayDirectory } = options const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) - const videoWithFiles = await VideoModel.loadFull(video.id) + const videoWithFiles = await VideoModel.loadFull(liveVideo.id) const hlsPlaylist = videoWithFiles.getHLSPlaylist() + const replayInAnotherDirectory = isVideoInPublicDirectory(liveVideo.privacy) !== isVideoInPublicDirectory(replaySettings.privacy) + + logger.info(`Replacing live ${liveVideo.uuid} by replay ${replayDirectory}.`, { replayInAnotherDirectory, ...lTags(liveVideo.uuid) }) await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) @@ -188,13 +203,25 @@ async function replaceLiveByReplay (options: { hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() await hlsPlaylist.save() - await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoWithFiles.uuid) - // Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay - if (permanentLive) { // Remove session replay - await remove(replayDirectory) - } else { // We won't stream again in this live, we can delete the base replay directory - await remove(getLiveReplayBaseDirectory(videoWithFiles)) + try { + await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) + + // Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay + if (permanentLive) { // Remove session replay + await remove(replayDirectory) + } else { + // We won't stream again in this live, we can delete the base replay directory + await remove(getLiveReplayBaseDirectory(liveVideo)) + + // If the live was in another base directory, also delete it + if (replayInAnotherDirectory) { + await remove(getHLSDirectory(liveVideo)) + } + } + } finally { + inputFileMutexReleaser() } // Regenerate the thumbnail & preview? @@ -214,8 +241,10 @@ async function assignReplayFilesToVideo (options: { const concatenatedTsFiles = await readdir(replayDirectory) + logger.info(`Assigning replays ${replayDirectory} to video ${video.uuid}.`, { concatenatedTsFiles, ...lTags(video.uuid) }) + for (const concatenatedTsFile of concatenatedTsFiles) { - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + // Generating hls playlist can be long, reload the video in this case await video.reload() const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) @@ -228,17 +257,17 @@ async function assignReplayFilesToVideo (options: { try { await generateHlsPlaylistResolutionFromTS({ video, - inputFileMutexReleaser, + inputFileMutexReleaser: null, // Already locked in parent concatenatedTsFilePath, resolution, fps, isAAC: audioStream?.codec_name === 'aac' }) + + logger.error('coucou') } catch (err) { logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) } - - inputFileMutexReleaser() } return video diff --git a/server/server/lib/transcoding/hls-transcoding.ts b/server/server/lib/transcoding/hls-transcoding.ts index 5f07f112a..15182f5e6 100644 --- a/server/server/lib/transcoding/hls-transcoding.ts +++ b/server/server/lib/transcoding/hls-transcoding.ts @@ -58,8 +58,9 @@ export async function onHLSVideoFileTranscoding (options: { videoFile: MVideoFile videoOutputPath: string m3u8OutputPath: string + filesLockedInParent?: boolean // default false }) { - const { video, videoFile, videoOutputPath, m3u8OutputPath } = options + const { video, videoFile, videoOutputPath, m3u8OutputPath, filesLockedInParent = false } = options // Create or update the playlist const playlist = await retryTransactionWrapper(() => { @@ -69,7 +70,9 @@ export async function onHLSVideoFileTranscoding (options: { }) videoFile.videoStreamingPlaylistId = playlist.id - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + const mutexReleaser = !filesLockedInParent + ? await VideoPathManager.Instance.lockFiles(video.uuid) + : null try { await video.reload() @@ -114,7 +117,7 @@ export async function onHLSVideoFileTranscoding (options: { return { resolutionPlaylistPath, videoFile: savedVideoFile } } finally { - mutexReleaser() + if (mutexReleaser) mutexReleaser() } } @@ -176,5 +179,11 @@ async function generateHlsPlaylistCommon (options: { fps: -1 }) - await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) + await onHLSVideoFileTranscoding({ + video, + videoFile: newVideoFile, + videoOutputPath, + m3u8OutputPath, + filesLockedInParent: !inputFileMutexReleaser + }) }