diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index 066b48ab0..66e738200 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts @@ -26,6 +26,10 @@ function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } +function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) { + return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +} + function removeWebTorrentObjectStorage (videoFile: MVideoFile) { return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) } @@ -63,6 +67,7 @@ export { storeHLSFile, removeHLSObjectStorage, + removeHLSFileObjectStorage, removeWebTorrentObjectStorage, makeWebTorrentFileAvailable, diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts index b64ce6e1f..69a973fbd 100644 --- a/server/lib/transcoding/transcoding.ts +++ b/server/lib/transcoding/transcoding.ts @@ -2,7 +2,9 @@ import { Job } from 'bull' import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' import { basename, extname as extnameUtil, join } from 'path' import { toEven } from '@server/helpers/core-utils' +import { retryTransactionWrapper } from '@server/helpers/database-utils' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' +import { sequelizeTypescript } from '@server/initializers/database' import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoResolution, VideoStorage } from '../../../shared/models/videos' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' @@ -29,8 +31,6 @@ import { } from '../paths' import { VideoPathManager } from '../video-path-manager' import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { sequelizeTypescript } from '@server/initializers/database' /** * @@ -259,6 +259,9 @@ async function onWebTorrentVideoFileTranscoding ( await createTorrentAndSetInfoHash(video, videoFile) + const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) + if (oldFile) await video.removeWebTorrentFileAndTorrent(oldFile) + await VideoFileModel.customUpsert(videoFile, 'video', undefined) video.VideoFiles = await video.$get('VideoFiles') @@ -311,17 +314,15 @@ async function generateHlsPlaylistCommon (options: { await transcodeVOD(transcodeOptions) // Create or update the playlist - const playlist = await retryTransactionWrapper(() => { + const { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } = await retryTransactionWrapper(() => { return sequelizeTypescript.transaction(async transaction => { const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) - if (!playlist.playlistFilename) { - playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) - } + const oldPlaylistFilename = playlist.playlistFilename + const oldSegmentsSha256Filename = playlist.segmentsSha256Filename - if (!playlist.segmentsSha256Filename) { - playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - } + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) playlist.p2pMediaLoaderInfohashes = [] playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION @@ -330,10 +331,13 @@ async function generateHlsPlaylistCommon (options: { await playlist.save({ transaction }) - return playlist + return { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } }) }) + if (oldPlaylistFilename) await video.removeStreamingPlaylistFile(playlist, oldPlaylistFilename) + if (oldSegmentsSha256Filename) await video.removeStreamingPlaylistFile(playlist, oldSegmentsSha256Filename) + // Build the new playlist file const extname = extnameUtil(videoFilename) const newVideoFile = new VideoFileModel({ @@ -364,11 +368,15 @@ async function generateHlsPlaylistCommon (options: { await createTorrentAndSetInfoHash(playlist, newVideoFile) + const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) + if (oldFile) await video.removeStreamingPlaylistVideoFile(playlist, oldFile) + const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + playlist.storage = VideoStorage.FILE_SYSTEM await playlist.save() diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 4aaee1ffa..d4f07f85f 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -405,15 +405,16 @@ export class VideoFileModel extends Model mode: 'streaming-playlist' | 'video', transaction: Transaction ) { - const baseWhere = { + const baseFind = { fps: videoFile.fps, - resolution: videoFile.resolution + resolution: videoFile.resolution, + transaction } - if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId }) - else Object.assign(baseWhere, { videoId: videoFile.videoId }) + const element = mode === 'streaming-playlist' + ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId }) + : await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId }) - const element = await VideoFileModel.findOne({ where: baseWhere, transaction }) if (!element) return videoFile.save({ transaction }) for (const k of Object.keys(videoFile.toJSON())) { @@ -423,6 +424,36 @@ export class VideoFileModel extends Model return element.save({ transaction }) } + static async loadWebTorrentFile (options: { + videoId: number + fps: number + resolution: number + transaction?: Transaction + }) { + const where = { + fps: options.fps, + resolution: options.resolution, + videoId: options.videoId + } + + return VideoFileModel.findOne({ where, transaction: options.transaction }) + } + + static async loadHLSFile (options: { + playlistId: number + fps: number + resolution: number + transaction?: Transaction + }) { + const where = { + fps: options.fps, + resolution: options.resolution, + videoStreamingPlaylistId: options.playlistId + } + + return VideoFileModel.findOne({ where, transaction: options.transaction }) + } + static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) { const options = { where: { videoStreamingPlaylistId } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 55da53058..27e605be6 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -26,7 +26,7 @@ import { } from 'sequelize-typescript' import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' import { LiveManager } from '@server/lib/live/live-manager' -import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' +import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths' import { VideoPathManager } from '@server/lib/video-path-manager' import { getServerActor } from '@server/models/application/application' @@ -1816,6 +1816,25 @@ export class VideoModel extends Model>> { } } + async removeStreamingPlaylistVideoFile (streamingPlaylist: MStreamingPlaylist, videoFile: MVideoFile) { + const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, videoFile.filename) + await videoFile.removeTorrent() + await remove(filePath) + + if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename) + } + } + + async removeStreamingPlaylistFile (streamingPlaylist: MStreamingPlaylist, filename: string) { + const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, filename) + await remove(filePath) + + if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), filename) + } + } + isOutdated () { if (this.isOwned()) return false diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts index a4defdf51..e3867fdad 100644 --- a/server/tests/api/transcoding/create-transcoding.ts +++ b/server/tests/api/transcoding/create-transcoding.ts @@ -46,6 +46,8 @@ function runTests (objectStorage: boolean) { let videoUUID: string let publishedAt: string + let shouldBeDeleted: string[] + before(async function () { this.timeout(120000) @@ -187,6 +189,12 @@ function runTests (objectStorage: boolean) { expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1) if (objectStorage) await checkFilesInObjectStorage(videoDetails) + + shouldBeDeleted = [ + videoDetails.streamingPlaylists[0].files[0].fileUrl, + videoDetails.streamingPlaylists[0].playlistUrl, + videoDetails.streamingPlaylists[0].segmentsSha256Url + ] } await servers[0].config.updateExistingSubConfig({ @@ -227,6 +235,12 @@ function runTests (objectStorage: boolean) { } }) + it('Should have correctly deleted previous files', async function () { + for (const fileUrl of shouldBeDeleted) { + await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404) + } + }) + it('Should not have updated published at attributes', async function () { const video = await servers[0].videos.get({ id: videoUUID })