Fix HLS re transcoding with object storage enabled
This commit is contained in:
parent
0f11ec8dd3
commit
a2caee9f51
6 changed files with 119 additions and 19 deletions
|
@ -82,7 +82,7 @@
|
||||||
|
|
||||||
<td>
|
<td>
|
||||||
<span *ngIf="isHLS(video)" class="badge badge-blue">HLS</span>
|
<span *ngIf="isHLS(video)" class="badge badge-blue">HLS</span>
|
||||||
<span *ngIf="isWebTorrent(video)" class="badge badge-blue">WebTorrent</span>
|
<span *ngIf="isWebTorrent(video)" class="badge badge-blue">WebTorrent ({{ video.files.length }})</span>
|
||||||
<span *ngIf="video.isLive" class="badge badge-blue">Live</span>
|
<span *ngIf="video.isLive" class="badge badge-blue">Live</span>
|
||||||
|
|
||||||
<span *ngIf="!isImport(video) && !video.isLive && video.isLocal">{{ getFilesSize(video) | bytes: 1 }}</span>
|
<span *ngIf="!isImport(video) && !video.isLive && video.isLocal">{{ getFilesSize(video) | bytes: 1 }}</span>
|
||||||
|
|
|
@ -3,6 +3,7 @@ import { flatten, uniq } from 'lodash'
|
||||||
import { basename, dirname, join } from 'path'
|
import { basename, dirname, join } from 'path'
|
||||||
import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
|
import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
|
||||||
import { sha256 } from '@shared/extra-utils'
|
import { sha256 } from '@shared/extra-utils'
|
||||||
|
import { VideoStorage } from '@shared/models'
|
||||||
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
|
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
|
||||||
import { logger } from '../helpers/logger'
|
import { logger } from '../helpers/logger'
|
||||||
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
|
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
|
||||||
|
@ -12,6 +13,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
|
||||||
import { sequelizeTypescript } from '../initializers/database'
|
import { sequelizeTypescript } from '../initializers/database'
|
||||||
import { VideoFileModel } from '../models/video/video-file'
|
import { VideoFileModel } from '../models/video/video-file'
|
||||||
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
|
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
|
||||||
|
import { storeHLSFile } from './object-storage'
|
||||||
import { getHlsResolutionPlaylistFilename } from './paths'
|
import { getHlsResolutionPlaylistFilename } from './paths'
|
||||||
import { VideoPathManager } from './video-path-manager'
|
import { VideoPathManager } from './video-path-manager'
|
||||||
|
|
||||||
|
@ -58,8 +60,12 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
|
await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => {
|
||||||
return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
|
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
|
||||||
|
|
||||||
|
if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
|
||||||
|
await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,6 +100,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
|
||||||
|
|
||||||
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
|
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
|
||||||
await outputJSON(outputPath, json)
|
await outputJSON(outputPath, json)
|
||||||
|
|
||||||
|
if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
|
||||||
|
await storeHLSFile(playlist, playlist.segmentsSha256Filename)
|
||||||
|
await remove(outputPath)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function buildSha256Segment (segmentPath: string) {
|
async function buildSha256Segment (segmentPath: string) {
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import { Job } from 'bull'
|
import { Job } from 'bull'
|
||||||
import { remove } from 'fs-extra'
|
import { remove } from 'fs-extra'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { logger } from '@server/helpers/logger'
|
import { logger, loggerTagsFactory } from '@server/helpers/logger'
|
||||||
import { updateTorrentMetadata } from '@server/helpers/webtorrent'
|
import { updateTorrentMetadata } from '@server/helpers/webtorrent'
|
||||||
import { CONFIG } from '@server/initializers/config'
|
import { CONFIG } from '@server/initializers/config'
|
||||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
|
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
|
||||||
|
@ -13,6 +13,8 @@ import { VideoJobInfoModel } from '@server/models/video/video-job-info'
|
||||||
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
|
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models'
|
||||||
import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
|
import { MoveObjectStoragePayload, VideoStorage } from '@shared/models'
|
||||||
|
|
||||||
|
const lTagsBase = loggerTagsFactory('move-object-storage')
|
||||||
|
|
||||||
export async function processMoveToObjectStorage (job: Job) {
|
export async function processMoveToObjectStorage (job: Job) {
|
||||||
const payload = job.data as MoveObjectStoragePayload
|
const payload = job.data as MoveObjectStoragePayload
|
||||||
logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
|
logger.info('Moving video %s in job %d.', payload.videoUUID, job.id)
|
||||||
|
@ -20,26 +22,33 @@ export async function processMoveToObjectStorage (job: Job) {
|
||||||
const video = await VideoModel.loadWithFiles(payload.videoUUID)
|
const video = await VideoModel.loadWithFiles(payload.videoUUID)
|
||||||
// No video, maybe deleted?
|
// No video, maybe deleted?
|
||||||
if (!video) {
|
if (!video) {
|
||||||
logger.info('Can\'t process job %d, video does not exist.', job.id)
|
logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const lTags = lTagsBase(video.uuid, video.url)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (video.VideoFiles) {
|
if (video.VideoFiles) {
|
||||||
|
logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags)
|
||||||
|
|
||||||
await moveWebTorrentFiles(video)
|
await moveWebTorrentFiles(video)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (video.VideoStreamingPlaylists) {
|
if (video.VideoStreamingPlaylists) {
|
||||||
|
logger.debug('Moving HLS playlist of %s.', video.uuid)
|
||||||
|
|
||||||
await moveHLSFiles(video)
|
await moveHLSFiles(video)
|
||||||
}
|
}
|
||||||
|
|
||||||
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
||||||
if (pendingMove === 0) {
|
if (pendingMove === 0) {
|
||||||
logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id)
|
logger.info('Running cleanup after moving files to object storage (video %s in job %d)', video.uuid, job.id, lTags)
|
||||||
|
|
||||||
await doAfterLastJob(video, payload.isNewVideo)
|
await doAfterLastJob(video, payload.isNewVideo)
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error('Cannot move video %s to object storage.', video.url, { err })
|
logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTags })
|
||||||
|
|
||||||
await moveToFailedMoveToObjectStorageState(video)
|
await moveToFailedMoveToObjectStorageState(video)
|
||||||
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
||||||
|
|
|
@ -6,11 +6,9 @@ import { getHLSDirectory } from '../paths'
|
||||||
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
|
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
|
||||||
import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
|
import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
|
||||||
|
|
||||||
function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) {
|
function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
|
||||||
const baseHlsDirectory = getHLSDirectory(playlist.Video)
|
|
||||||
|
|
||||||
return storeObject({
|
return storeObject({
|
||||||
inputPath: join(baseHlsDirectory, filename),
|
inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
|
||||||
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
|
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
|
||||||
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
|
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
|
||||||
})
|
})
|
||||||
|
|
|
@ -2,11 +2,12 @@
|
||||||
|
|
||||||
import 'mocha'
|
import 'mocha'
|
||||||
import * as chai from 'chai'
|
import * as chai from 'chai'
|
||||||
import { expectStartWith } from '@server/tests/shared'
|
import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared'
|
||||||
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
|
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
|
||||||
import { HttpStatusCode, VideoDetails } from '@shared/models'
|
import { HttpStatusCode, VideoDetails } from '@shared/models'
|
||||||
import {
|
import {
|
||||||
cleanupTests,
|
cleanupTests,
|
||||||
|
ConfigCommand,
|
||||||
createMultipleServers,
|
createMultipleServers,
|
||||||
doubleFollow,
|
doubleFollow,
|
||||||
expectNoFailedTranscodingJob,
|
expectNoFailedTranscodingJob,
|
||||||
|
@ -25,14 +26,19 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
|
||||||
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
|
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
|
||||||
}
|
}
|
||||||
|
|
||||||
const streamingPlaylistFiles = video.streamingPlaylists.length === 0
|
if (video.streamingPlaylists.length === 0) return
|
||||||
? []
|
|
||||||
: video.streamingPlaylists[0].files
|
|
||||||
|
|
||||||
for (const file of streamingPlaylistFiles) {
|
const hlsPlaylist = video.streamingPlaylists[0]
|
||||||
|
for (const file of hlsPlaylist.files) {
|
||||||
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
|
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
|
||||||
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
|
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
|
||||||
|
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
|
||||||
|
|
||||||
|
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
|
||||||
|
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
|
||||||
}
|
}
|
||||||
|
|
||||||
function runTests (objectStorage: boolean) {
|
function runTests (objectStorage: boolean) {
|
||||||
|
@ -150,6 +156,75 @@ function runTests (objectStorage: boolean) {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('Should correctly update HLS playlist on resolution change', async function () {
|
||||||
|
await servers[0].config.updateExistingSubConfig({
|
||||||
|
newConfig: {
|
||||||
|
transcoding: {
|
||||||
|
enabled: true,
|
||||||
|
resolutions: ConfigCommand.getCustomConfigResolutions(false),
|
||||||
|
|
||||||
|
webtorrent: {
|
||||||
|
enabled: true
|
||||||
|
},
|
||||||
|
hls: {
|
||||||
|
enabled: true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const { uuid } = await servers[0].videos.quickUpload({ name: 'quick' })
|
||||||
|
|
||||||
|
await waitJobs(servers)
|
||||||
|
|
||||||
|
for (const server of servers) {
|
||||||
|
const videoDetails = await server.videos.get({ id: uuid })
|
||||||
|
|
||||||
|
expect(videoDetails.files).to.have.lengthOf(1)
|
||||||
|
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
|
||||||
|
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
|
||||||
|
|
||||||
|
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
|
||||||
|
}
|
||||||
|
|
||||||
|
await servers[0].config.updateExistingSubConfig({
|
||||||
|
newConfig: {
|
||||||
|
transcoding: {
|
||||||
|
enabled: true,
|
||||||
|
resolutions: ConfigCommand.getCustomConfigResolutions(true),
|
||||||
|
|
||||||
|
webtorrent: {
|
||||||
|
enabled: true
|
||||||
|
},
|
||||||
|
hls: {
|
||||||
|
enabled: true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
await servers[0].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' })
|
||||||
|
await waitJobs(servers)
|
||||||
|
|
||||||
|
for (const server of servers) {
|
||||||
|
const videoDetails = await server.videos.get({ id: uuid })
|
||||||
|
|
||||||
|
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
|
||||||
|
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
|
||||||
|
|
||||||
|
if (objectStorage) {
|
||||||
|
await checkFilesInObjectStorage(videoDetails)
|
||||||
|
|
||||||
|
const hlsPlaylist = videoDetails.streamingPlaylists[0]
|
||||||
|
const resolutions = hlsPlaylist.files.map(f => f.resolution.id)
|
||||||
|
await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions })
|
||||||
|
|
||||||
|
const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url })
|
||||||
|
expect(Object.keys(shaBody)).to.have.lengthOf(5)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
it('Should not have updated published at attributes', async function () {
|
it('Should not have updated published at attributes', async function () {
|
||||||
const video = await servers[0].videos.get({ id: videoUUID })
|
const video = await servers[0].videos.get({ id: videoUUID })
|
||||||
|
|
||||||
|
|
|
@ -14,7 +14,7 @@ import {
|
||||||
setAccessTokensToServers,
|
setAccessTokensToServers,
|
||||||
waitJobs
|
waitJobs
|
||||||
} from '@shared/server-commands'
|
} from '@shared/server-commands'
|
||||||
import { expectStartWith } from '../shared'
|
import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared'
|
||||||
|
|
||||||
const expect = chai.expect
|
const expect = chai.expect
|
||||||
|
|
||||||
|
@ -163,11 +163,18 @@ function runTests (objectStorage: boolean) {
|
||||||
|
|
||||||
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
|
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
|
||||||
|
|
||||||
const files = videoDetails.streamingPlaylists[0].files
|
const hlsPlaylist = videoDetails.streamingPlaylists[0]
|
||||||
|
|
||||||
|
const files = hlsPlaylist.files
|
||||||
expect(files).to.have.lengthOf(1)
|
expect(files).to.have.lengthOf(1)
|
||||||
expect(files[0].resolution.id).to.equal(480)
|
expect(files[0].resolution.id).to.equal(480)
|
||||||
|
|
||||||
if (objectStorage) await checkFilesInObjectStorage(files, 'playlist')
|
if (objectStorage) {
|
||||||
|
await checkFilesInObjectStorage(files, 'playlist')
|
||||||
|
|
||||||
|
const resolutions = files.map(f => f.resolution.id)
|
||||||
|
await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue