forks/peertube
1
0
Fork 0
Code Releases Activity

Regenerate video filenames on transcoding

Browse source 
In particular when using manual transcoding, to invalidate potential
HTTP caches in front of peertube
This commit is contained in:
Chocobozzz 2022-07-25 10:57:16 +02:00
parent 4f50475c67
commit 7b6b445d91
No known key found for this signature in database
GPG key ID: 583A612D890159BE
5 changed files with 93 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
server/lib/object-storage/videos.ts
View file

@ -26,6 +26,10 @@ function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) {
return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
}
@ -63,6 +67,7 @@ export {
storeHLSFile,
removeHLSObjectStorage,
removeHLSFileObjectStorage,
removeWebTorrentObjectStorage,
makeWebTorrentFileAvailable,

28
server/lib/transcoding/transcoding.ts
View file

@ -2,7 +2,9 @@ import { Job } from 'bull'
import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
import { basename, extname as extnameUtil, join } from 'path'
import { toEven } from '@server/helpers/core-utils'
import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { sequelizeTypescript } from '@server/initializers/database'
import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@ -29,8 +31,6 @@ import {
} from '../paths'
import { VideoPathManager } from '../video-path-manager'
import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { sequelizeTypescript } from '@server/initializers/database'
/**
*
@ -259,6 +259,9 @@ async function onWebTorrentVideoFileTranscoding (
await createTorrentAndSetInfoHash(video, videoFile)
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebTorrentFileAndTorrent(oldFile)
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
@ -311,17 +314,15 @@ async function generateHlsPlaylistCommon (options: {
await transcodeVOD(transcodeOptions)
// Create or update the playlist
const playlist = await retryTransactionWrapper(() => {
const { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } = await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async transaction => {
const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
if (!playlist.playlistFilename) {
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
}
const oldPlaylistFilename = playlist.playlistFilename
const oldSegmentsSha256Filename = playlist.segmentsSha256Filename
if (!playlist.segmentsSha256Filename) {
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
}
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
playlist.p2pMediaLoaderInfohashes = []
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
@ -330,10 +331,13 @@ async function generateHlsPlaylistCommon (options: {
await playlist.save({ transaction })
return playlist
return { playlist, oldPlaylistFilename, oldSegmentsSha256Filename }
})
})
if (oldPlaylistFilename) await video.removeStreamingPlaylistFile(playlist, oldPlaylistFilename)
if (oldSegmentsSha256Filename) await video.removeStreamingPlaylistFile(playlist, oldSegmentsSha256Filename)
// Build the new playlist file
const extname = extnameUtil(videoFilename)
const newVideoFile = new VideoFileModel({
@ -364,11 +368,15 @@ async function generateHlsPlaylistCommon (options: {
await createTorrentAndSetInfoHash(playlist, newVideoFile)
const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution })
if (oldFile) await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
playlist.storage = VideoStorage.FILE_SYSTEM
await playlist.save()

41
server/models/video/video-file.ts
View file

@ -405,15 +405,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
mode: 'streaming-playlist' | 'video',
transaction: Transaction
) {
const baseWhere = {
const baseFind = {
fps: videoFile.fps,
resolution: videoFile.resolution
resolution: videoFile.resolution,
transaction
}
if (mode === 'streaming-playlist') Object.assign(baseWhere, { videoStreamingPlaylistId: videoFile.videoStreamingPlaylistId })
else Object.assign(baseWhere, { videoId: videoFile.videoId })
const element = mode === 'streaming-playlist'
? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId })
: await VideoFileModel.loadWebTorrentFile({ ...baseFind, videoId: videoFile.videoId })
const element = await VideoFileModel.findOne({ where: baseWhere, transaction })
if (!element) return videoFile.save({ transaction })
for (const k of Object.keys(videoFile.toJSON())) {
@ -423,6 +424,36 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return element.save({ transaction })
}
static async loadWebTorrentFile (options: {
videoId: number
fps: number
resolution: number
transaction?: Transaction
}) {
const where = {
fps: options.fps,
resolution: options.resolution,
videoId: options.videoId
}
return VideoFileModel.findOne({ where, transaction: options.transaction })
}
static async loadHLSFile (options: {
playlistId: number
fps: number
resolution: number
transaction?: Transaction
}) {
const where = {
fps: options.fps,
resolution: options.resolution,
videoStreamingPlaylistId: options.playlistId
}
return VideoFileModel.findOne({ where, transaction: options.transaction })
}
static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) {
const options = {
where: { videoStreamingPlaylistId }

21
server/models/video/video.ts
View file

@ -26,7 +26,7 @@ import {
} from 'sequelize-typescript'
import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video'
import { LiveManager } from '@server/lib/live/live-manager'
import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { getServerActor } from '@server/models/application/application'
@ -1816,6 +1816,25 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
}
}
async removeStreamingPlaylistVideoFile (streamingPlaylist: MStreamingPlaylist, videoFile: MVideoFile) {
const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, videoFile.filename)
await videoFile.removeTorrent()
await remove(filePath)
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename)
}
}
async removeStreamingPlaylistFile (streamingPlaylist: MStreamingPlaylist, filename: string) {
const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, filename)
await remove(filePath)
if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), filename)
}
}
isOutdated () {
if (this.isOwned()) return false

14
server/tests/api/transcoding/create-transcoding.ts
View file

@ -46,6 +46,8 @@ function runTests (objectStorage: boolean) {
let videoUUID: string
let publishedAt: string
let shouldBeDeleted: string[]
before(async function () {
this.timeout(120000)
@ -187,6 +189,12 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
shouldBeDeleted = [
videoDetails.streamingPlaylists[0].files[0].fileUrl,
videoDetails.streamingPlaylists[0].playlistUrl,
videoDetails.streamingPlaylists[0].segmentsSha256Url
]
}
await servers[0].config.updateExistingSubConfig({
@ -227,6 +235,12 @@ function runTests (objectStorage: boolean) {
}
})
it('Should have correctly deleted previous files', async function () {
for (const fileUrl of shouldBeDeleted) {
await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404)
}
})
it('Should not have updated published at attributes', async function () {
const video = await servers[0].videos.get({ id: videoUUID })