forks/peertube
1
0
Fork 0
Code Releases Activity

Support CLI move of original video file

Browse source
This commit is contained in:
Chocobozzz 2024-06-04 09:08:24 +02:00
parent 54c140c800
commit 96b9748585
No known key found for this signature in database
GPG key ID: 583A612D890159BE
6 changed files with 129 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/server-commands/src/requests/requests.ts
View file

@ -166,9 +166,10 @@ export function makePutBodyRequest (options: {
// ---------------------------------------------------------------------------
export async function getRedirectionUrl (url: string) {
export async function getRedirectionUrl (url: string, token?: string) {
const res = await makeRawRequest({
url,
token,
redirects: 0,
expectedStatus: HttpStatusCode.FOUND_302
})

55
packages/tests/src/cli/create-move-video-storage-job.ts
View file

@ -1,23 +1,26 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { join } from 'path'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import { getAllFiles } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
ObjectStorageCommand,
PeerTubeServer,
cleanupTests,
createMultipleServers,
doubleFollow,
getRedirectionUrl,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
} from '@peertube/peertube-server-commands'
import { expectStartWith } from '../shared/checks.js'
import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
import { getAllFiles } from '@peertube/peertube-core-utils'
import { join } from 'path'
import { expectStartWith } from '../shared/checks.js'
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
// Web videos
for (const file of video.files) {
const start = objectStorage
? objectStorage.getMockWebVideosBaseUrl()
@ -28,18 +31,36 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const start = objectStorage
? objectStorage.getMockPlaylistBaseUrl()
: origin.url
// Playlists
{
const start = objectStorage
? objectStorage.getMockPlaylistBaseUrl()
: origin.url
const hls = video.streamingPlaylists[0]
expectStartWith(hls.playlistUrl, start)
expectStartWith(hls.segmentsSha256Url, start)
const hls = video.streamingPlaylists[0]
expectStartWith(hls.playlistUrl, start)
expectStartWith(hls.segmentsSha256Url, start)
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
// Original file
{
const source = await origin.videos.getSource({ id: video.uuid })
if (objectStorage) {
await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.FOUND_302 })
const redirected = await getRedirectionUrl(source.fileDownloadUrl, origin.accessToken)
expectStartWith(redirected, objectStorage.getMockOriginalFileBaseUrl())
} else {
await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.OK_200 })
expectStartWith(source.fileDownloadUrl, origin.url)
}
}
}
@ -61,10 +82,10 @@ describe('Test create move video storage job', function () {
await objectStorage.prepareDefaultMockBuckets()
await servers[0].config.enableTranscoding()
await servers[0].config.enableMinimumTranscoding({ keepOriginal: true })
for (let i = 0; i < 3; i++) {
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' + i })
uuids.push(uuid)
}

36
server/core/lib/job-queue/handlers/move-to-file-system.ts
View file

@ -1,20 +1,23 @@
import { Job } from 'bullmq'
import { join } from 'path'
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
import {
makeHLSFileAvailable,
makeOriginalFileAvailable,
makeWebVideoFileAvailable,
removeHLSFileObjectStorageByFilename,
removeHLSObjectStorage,
removeOriginalFileObjectStorage,
removeWebVideoObjectStorage
} from '@server/lib/object-storage/index.js'
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { Job } from 'bullmq'
import { join } from 'path'
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('move-file-system')
@ -30,6 +33,8 @@ export async function processMoveToFileSystem (job: Job) {
moveWebVideoFiles,
moveHLSFiles,
moveVideoSourceFile,
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
moveToFailedState: moveToFailedMoveToFileSystemState
})
@ -50,12 +55,31 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
// Private
// ---------------------------------------------------------------------------
async function moveVideoSourceFile (source: MVideoSource) {
if (source.storage === FileStorage.FILE_SYSTEM) return
await makeOriginalFileAvailable(
source.keptOriginalFilename,
VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
)
const oldFileUrl = source.fileUrl
source.fileUrl = null
source.storage = FileStorage.FILE_SYSTEM
await source.save()
logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
await removeOriginalFileObjectStorage(source)
}
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage === FileStorage.FILE_SYSTEM) continue
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
await onFileMoved({
await onVideoFileMoved({
videoOrPlaylist: video,
file,
objetStorageRemover: () => removeWebVideoObjectStorage(file)
@ -75,7 +99,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
await onFileMoved({
await onVideoFileMoved({
videoOrPlaylist: playlistWithVideo,
file,
objetStorageRemover: async () => {
@ -87,7 +111,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
}
}
async function onFileMoved (options: {
async function onVideoFileMoved (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
file: MVideoFile
objetStorageRemover: () => Promise<any>

33
server/core/lib/job-queue/handlers/move-to-object-storage.ts
View file

@ -1,15 +1,16 @@
import { Job } from 'bullmq'
import { remove } from 'fs-extra/esm'
import { join } from 'path'
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage/index.js'
import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js'
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { Job } from 'bullmq'
import { remove } from 'fs-extra/esm'
import { join } from 'path'
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('move-object-storage')
@ -25,6 +26,7 @@ export async function processMoveToObjectStorage (job: Job) {
moveWebVideoFiles,
moveHLSFiles,
moveVideoSourceFile,
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
moveToFailedState: moveToFailedMoveToObjectStorageState
})
@ -43,6 +45,21 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
// ---------------------------------------------------------------------------
async function moveVideoSourceFile (source: MVideoSource) {
if (source.storage !== FileStorage.FILE_SYSTEM) return
const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
source.storage = FileStorage.OBJECT_STORAGE
source.fileUrl = fileUrl
await source.save()
logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
await remove(sourcePath)
}
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage !== FileStorage.FILE_SYSTEM) continue
@ -50,7 +67,7 @@ async function moveWebVideoFiles (video: MVideoWithAllFiles) {
const fileUrl = await storeWebVideoFile(video, file)
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
}
}
@ -70,12 +87,12 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
const oldPath = join(getHLSDirectory(video), file.filename)
await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
}
}
}
async function onFileMoved (options: {
async function onVideoFileMoved (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
file: MVideoFile
fileUrl: string

21
server/core/lib/job-queue/handlers/shared/move-video.ts
View file

@ -1,8 +1,10 @@
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
export async function moveToJob (options: {
jobId: string
@ -11,10 +13,20 @@ export async function moveToJob (options: {
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
}) {
const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options
const {
jobId,
loggerTags,
videoUUID,
moveVideoSourceFile,
moveHLSFiles,
moveWebVideoFiles,
moveToFailedState,
doAfterLastMove
} = options
const lTagsBase = loggerTagsFactory(...loggerTags)
@ -31,6 +43,13 @@ export async function moveToJob (options: {
const lTags = lTagsBase(video.uuid, video.url)
try {
const source = await VideoSourceModel.loadLatest(video.id)
if (source) {
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
await moveVideoSourceFile(source)
}
if (video.VideoFiles) {
logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)

14
server/core/lib/object-storage/videos.ts
View file

@ -159,6 +159,20 @@ export async function makeWebVideoFileAvailable (filename: string, destination:
return destination
}
export async function makeOriginalFileAvailable (keptOriginalFilename: string, destination: string) {
const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
logger.info('Fetching Original Video file %s from object storage to %s.', key, destination, lTags())
await makeAvailable({
key,
destination,
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES
})
return destination
}
// ---------------------------------------------------------------------------
export function getWebVideoFileReadStream (options: {