1
0
Fork 0

Fix HLS re transcoding with object storage enabled

This commit is contained in:
Chocobozzz 2022-02-01 14:19:44 +01:00
parent 0f11ec8dd3
commit a2caee9f51
No known key found for this signature in database
GPG key ID: 583A612D890159BE
6 changed files with 119 additions and 19 deletions

View file

@ -82,7 +82,7 @@
<td> <td>
<span *ngIf="isHLS(video)" class="badge badge-blue">HLS</span> <span *ngIf="isHLS(video)" class="badge badge-blue">HLS</span>
<span *ngIf="isWebTorrent(video)" class="badge badge-blue">WebTorrent</span> <span *ngIf="isWebTorrent(video)" class="badge badge-blue">WebTorrent ({{ video.files.length }})</span>
<span *ngIf="video.isLive" class="badge badge-blue">Live</span> <span *ngIf="video.isLive" class="badge badge-blue">Live</span>
<span *ngIf="!isImport(video) && !video.isLive && video.isLocal">{{ getFilesSize(video) | bytes: 1 }}</span> <span *ngIf="!isImport(video) && !video.isLive && video.isLocal">{{ getFilesSize(video) | bytes: 1 }}</span>

View file

@ -3,6 +3,7 @@ import { flatten, uniq } from 'lodash'
import { basename, dirname, join } from 'path' import { basename, dirname, join } from 'path'
import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
import { sha256 } from '@shared/extra-utils' import { sha256 } from '@shared/extra-utils'
import { VideoStorage } from '@shared/models'
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
import { logger } from '../helpers/logger' import { logger } from '../helpers/logger'
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
@ -12,6 +13,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
import { sequelizeTypescript } from '../initializers/database' import { sequelizeTypescript } from '../initializers/database'
import { VideoFileModel } from '../models/video/video-file' import { VideoFileModel } from '../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
import { storeHLSFile } from './object-storage'
import { getHlsResolutionPlaylistFilename } from './paths' import { getHlsResolutionPlaylistFilename } from './paths'
import { VideoPathManager } from './video-path-manager' import { VideoPathManager } from './video-path-manager'
@ -58,8 +60,12 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
}) })
} }
await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => {
return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath)
}
}) })
} }
@ -94,6 +100,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json) await outputJSON(outputPath, json)
if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
await storeHLSFile(playlist, playlist.segmentsSha256Filename)
await remove(outputPath)
}
} }
async function buildSha256Segment (segmentPath: string) { async function buildSha256Segment (segmentPath: string) {

View file

@ -1,7 +1,7 @@
import { Job } from 'bull' import { Job } from 'bull'
import { remove } from 'fs-extra' import { remove } from 'fs-extra'
import { join } from 'path' import { join } from 'path'
import { logger } from '@server/helpers/logger' import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { updateTorrentMetadata } from '@server/helpers/webtorrent' import { updateTorrentMetadata } from '@server/helpers/webtorrent'
import { CONFIG } from '@server/initializers/config' import { CONFIG } from '@server/initializers/config'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
@ -13,6 +13,8 @@ import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
import { MoveObjectStoragePayload, VideoStorage } from '@shared/models' import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
const lTagsBase = loggerTagsFactory('move-object-storage')
export async function processMoveToObjectStorage (job: Job) { export async function processMoveToObjectStorage (job: Job) {
const payload = job.data as MoveObjectStoragePayload const payload = job.data as MoveObjectStoragePayload
logger.info('Moving video %s in job %d.', payload.videoUUID, job.id) logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
@ -20,26 +22,33 @@ export async function processMoveToObjectStorage (job: Job) {
const video = await VideoModel.loadWithFiles(payload.videoUUID) const video = await VideoModel.loadWithFiles(payload.videoUUID)
// No video, maybe deleted? // No video, maybe deleted?
if (!video) { if (!video) {
logger.info('Can\'t process job %d, video does not exist.', job.id) logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
return undefined return undefined
} }
const lTags = lTagsBase(video.uuid, video.url)
try { try {
if (video.VideoFiles) { if (video.VideoFiles) {
logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
await moveWebTorrentFiles(video) await moveWebTorrentFiles(video)
} }
if (video.VideoStreamingPlaylists) { if (video.VideoStreamingPlaylists) {
logger.debug('Moving HLS playlist of %s.', video.uuid)
await moveHLSFiles(video) await moveHLSFiles(video)
} }
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
if (pendingMove === 0) { if (pendingMove === 0) {
logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id) logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id, lTags)
await doAfterLastJob(video, payload.isNewVideo) await doAfterLastJob(video, payload.isNewVideo)
} }
} catch (err) { } catch (err) {
logger.error('Cannot move video %s to object storage.', video.url, { err }) logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags })
await moveToFailedMoveToObjectStorageState(video) await moveToFailedMoveToObjectStorageState(video)
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')

View file

@ -6,11 +6,9 @@ import { getHLSDirectory } from '../paths'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) { function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
const baseHlsDirectory = getHLSDirectory(playlist.Video)
return storeObject({ return storeObject({
inputPath: join(baseHlsDirectory, filename), inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, filename), objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
}) })

View file

@ -2,11 +2,12 @@
import 'mocha' import 'mocha'
import * as chai from 'chai' import * as chai from 'chai'
import { expectStartWith } from '@server/tests/shared' import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils' import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoDetails } from '@shared/models' import { HttpStatusCode, VideoDetails } from '@shared/models'
import { import {
cleanupTests, cleanupTests,
ConfigCommand,
createMultipleServers, createMultipleServers,
doubleFollow, doubleFollow,
expectNoFailedTranscodingJob, expectNoFailedTranscodingJob,
@ -25,14 +26,19 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
} }
const streamingPlaylistFiles = video.streamingPlaylists.length === 0 if (video.streamingPlaylists.length === 0) return
? []
: video.streamingPlaylists[0].files
for (const file of streamingPlaylistFiles) { const hlsPlaylist = video.streamingPlaylists[0]
for (const file of hlsPlaylist.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
} }
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
} }
function runTests (objectStorage: boolean) { function runTests (objectStorage: boolean) {
@ -150,6 +156,75 @@ function runTests (objectStorage: boolean) {
} }
}) })
it('Should correctly update HLS playlist on resolution change', async function () {
await servers[0].config.updateExistingSubConfig({
newConfig: {
transcoding: {
enabled: true,
resolutions: ConfigCommand.getCustomConfigResolutions(false),
webtorrent: {
enabled: true
},
hls: {
enabled: true
}
}
}
})
const { uuid } = await servers[0].videos.quickUpload({ name: 'quick' })
await waitJobs(servers)
for (const server of servers) {
const videoDetails = await server.videos.get({ id: uuid })
expect(videoDetails.files).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
}
await servers[0].config.updateExistingSubConfig({
newConfig: {
transcoding: {
enabled: true,
resolutions: ConfigCommand.getCustomConfigResolutions(true),
webtorrent: {
enabled: true
},
hls: {
enabled: true
}
}
}
})
await servers[0].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' })
await waitJobs(servers)
for (const server of servers) {
const videoDetails = await server.videos.get({ id: uuid })
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) {
await checkFilesInObjectStorage(videoDetails)
const hlsPlaylist = videoDetails.streamingPlaylists[0]
const resolutions = hlsPlaylist.files.map(f => f.resolution.id)
await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions })
const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
expect(Object.keys(shaBody)).to.have.lengthOf(5)
}
}
})
it('Should not have updated published at attributes', async function () { it('Should not have updated published at attributes', async function () {
const video = await servers[0].videos.get({ id: videoUUID }) const video = await servers[0].videos.get({ id: videoUUID })

View file

@ -14,7 +14,7 @@ import {
setAccessTokensToServers, setAccessTokensToServers,
waitJobs waitJobs
} from '@shared/server-commands' } from '@shared/server-commands'
import { expectStartWith } from '../shared' import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared'
const expect = chai.expect const expect = chai.expect
@ -163,11 +163,18 @@ function runTests (objectStorage: boolean) {
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
const files = videoDetails.streamingPlaylists[0].files const hlsPlaylist = videoDetails.streamingPlaylists[0]
const files = hlsPlaylist.files
expect(files).to.have.lengthOf(1) expect(files).to.have.lengthOf(1)
expect(files[0].resolution.id).to.equal(480) expect(files[0].resolution.id).to.equal(480)
if (objectStorage) await checkFilesInObjectStorage(files, 'playlist') if (objectStorage) {
await checkFilesInObjectStorage(files, 'playlist')
const resolutions = files.map(f => f.resolution.id)
await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
}
} }
}) })