1
0
Fork 0

Support CLI move of original video file

This commit is contained in:
Chocobozzz 2024-06-04 09:08:24 +02:00
parent 54c140c800
commit 96b9748585
No known key found for this signature in database
GPG key ID: 583A612D890159BE
6 changed files with 129 additions and 33 deletions

View file

@ -166,9 +166,10 @@ export function makePutBodyRequest (options: {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export async function getRedirectionUrl (url: string) { export async function getRedirectionUrl (url: string, token?: string) {
const res = await makeRawRequest({ const res = await makeRawRequest({
url, url,
token,
redirects: 0, redirects: 0,
expectedStatus: HttpStatusCode.FOUND_302 expectedStatus: HttpStatusCode.FOUND_302
}) })

View file

@ -1,23 +1,26 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { join } from 'path' import { getAllFiles } from '@peertube/peertube-core-utils'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models' import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import { import {
ObjectStorageCommand,
PeerTubeServer,
cleanupTests, cleanupTests,
createMultipleServers, createMultipleServers,
doubleFollow, doubleFollow,
getRedirectionUrl,
makeRawRequest, makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers, setAccessTokensToServers,
waitJobs waitJobs
} from '@peertube/peertube-server-commands' } from '@peertube/peertube-server-commands'
import { expectStartWith } from '../shared/checks.js'
import { checkDirectoryIsEmpty } from '@tests/shared/directories.js' import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
import { getAllFiles } from '@peertube/peertube-core-utils' import { join } from 'path'
import { expectStartWith } from '../shared/checks.js'
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) { async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
// Web videos
for (const file of video.files) { for (const file of video.files) {
const start = objectStorage const start = objectStorage
? objectStorage.getMockWebVideosBaseUrl() ? objectStorage.getMockWebVideosBaseUrl()
@ -28,6 +31,8 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
} }
// Playlists
{
const start = objectStorage const start = objectStorage
? objectStorage.getMockPlaylistBaseUrl() ? objectStorage.getMockPlaylistBaseUrl()
: origin.url : origin.url
@ -41,6 +46,22 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
} }
}
// Original file
{
const source = await origin.videos.getSource({ id: video.uuid })
if (objectStorage) {
await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.FOUND_302 })
const redirected = await getRedirectionUrl(source.fileDownloadUrl, origin.accessToken)
expectStartWith(redirected, objectStorage.getMockOriginalFileBaseUrl())
} else {
await makeRawRequest({ url: source.fileDownloadUrl, token: origin.accessToken, expectedStatus: HttpStatusCode.OK_200 })
expectStartWith(source.fileDownloadUrl, origin.url)
}
}
} }
describe('Test create move video storage job', function () { describe('Test create move video storage job', function () {
@ -61,10 +82,10 @@ describe('Test create move video storage job', function () {
await objectStorage.prepareDefaultMockBuckets() await objectStorage.prepareDefaultMockBuckets()
await servers[0].config.enableTranscoding() await servers[0].config.enableMinimumTranscoding({ keepOriginal: true })
for (let i = 0; i < 3; i++) { for (let i = 0; i < 3; i++) {
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) const { uuid } = await servers[0].videos.quickUpload({ name: 'video' + i })
uuids.push(uuid) uuids.push(uuid)
} }

View file

@ -1,20 +1,23 @@
import { Job } from 'bullmq' import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
import { join } from 'path'
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
import { import {
makeHLSFileAvailable, makeHLSFileAvailable,
makeOriginalFileAvailable,
makeWebVideoFileAvailable, makeWebVideoFileAvailable,
removeHLSFileObjectStorageByFilename, removeHLSFileObjectStorageByFilename,
removeHLSObjectStorage, removeHLSObjectStorage,
removeOriginalFileObjectStorage,
removeWebVideoObjectStorage removeWebVideoObjectStorage
} from '@server/lib/object-storage/index.js' } from '@server/lib/object-storage/index.js'
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js' import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js' import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { Job } from 'bullmq'
import { join } from 'path'
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js' import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('move-file-system') const lTagsBase = loggerTagsFactory('move-file-system')
@ -30,6 +33,8 @@ export async function processMoveToFileSystem (job: Job) {
moveWebVideoFiles, moveWebVideoFiles,
moveHLSFiles, moveHLSFiles,
moveVideoSourceFile,
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
moveToFailedState: moveToFailedMoveToFileSystemState moveToFailedState: moveToFailedMoveToFileSystemState
}) })
@ -50,12 +55,31 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
// Private // Private
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
async function moveVideoSourceFile (source: MVideoSource) {
if (source.storage === FileStorage.FILE_SYSTEM) return
await makeOriginalFileAvailable(
source.keptOriginalFilename,
VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
)
const oldFileUrl = source.fileUrl
source.fileUrl = null
source.storage = FileStorage.FILE_SYSTEM
await source.save()
logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
await removeOriginalFileObjectStorage(source)
}
async function moveWebVideoFiles (video: MVideoWithAllFiles) { async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) { for (const file of video.VideoFiles) {
if (file.storage === FileStorage.FILE_SYSTEM) continue if (file.storage === FileStorage.FILE_SYSTEM) continue
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)) await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
await onFileMoved({ await onVideoFileMoved({
videoOrPlaylist: video, videoOrPlaylist: video,
file, file,
objetStorageRemover: () => removeWebVideoObjectStorage(file) objetStorageRemover: () => removeWebVideoObjectStorage(file)
@ -75,7 +99,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename)) await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename)) await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
await onFileMoved({ await onVideoFileMoved({
videoOrPlaylist: playlistWithVideo, videoOrPlaylist: playlistWithVideo,
file, file,
objetStorageRemover: async () => { objetStorageRemover: async () => {
@ -87,7 +111,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
} }
} }
async function onFileMoved (options: { async function onVideoFileMoved (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo videoOrPlaylist: MVideo | MStreamingPlaylistVideo
file: MVideoFile file: MVideoFile
objetStorageRemover: () => Promise<any> objetStorageRemover: () => Promise<any>

View file

@ -1,15 +1,16 @@
import { Job } from 'bullmq' import { FileStorage, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
import { remove } from 'fs-extra/esm'
import { join } from 'path'
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage/index.js' import { storeHLSFileFromFilename, storeOriginalVideoFile, storeWebVideoFile } from '@server/lib/object-storage/index.js'
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js' import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js' import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { Job } from 'bullmq'
import { remove } from 'fs-extra/esm'
import { join } from 'path'
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js' import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('move-object-storage') const lTagsBase = loggerTagsFactory('move-object-storage')
@ -25,6 +26,7 @@ export async function processMoveToObjectStorage (job: Job) {
moveWebVideoFiles, moveWebVideoFiles,
moveHLSFiles, moveHLSFiles,
moveVideoSourceFile,
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }), doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
moveToFailedState: moveToFailedMoveToObjectStorageState moveToFailedState: moveToFailedMoveToObjectStorageState
}) })
@ -43,6 +45,21 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
async function moveVideoSourceFile (source: MVideoSource) {
if (source.storage !== FileStorage.FILE_SYSTEM) return
const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
source.storage = FileStorage.OBJECT_STORAGE
source.fileUrl = fileUrl
await source.save()
logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
await remove(sourcePath)
}
async function moveWebVideoFiles (video: MVideoWithAllFiles) { async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) { for (const file of video.VideoFiles) {
if (file.storage !== FileStorage.FILE_SYSTEM) continue if (file.storage !== FileStorage.FILE_SYSTEM) continue
@ -50,7 +67,7 @@ async function moveWebVideoFiles (video: MVideoWithAllFiles) {
const fileUrl = await storeWebVideoFile(video, file) const fileUrl = await storeWebVideoFile(video, file)
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
} }
} }
@ -70,12 +87,12 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
const oldPath = join(getHLSDirectory(video), file.filename) const oldPath = join(getHLSDirectory(video), file.filename)
await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
} }
} }
} }
async function onFileMoved (options: { async function onVideoFileMoved (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo videoOrPlaylist: MVideo | MStreamingPlaylistVideo
file: MVideoFile file: MVideoFile
fileUrl: string fileUrl: string

View file

@ -1,8 +1,10 @@
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js' import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js' import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js' import { VideoModel } from '@server/models/video/video.js'
import { MVideoWithAllFiles } from '@server/types/models/index.js' import { MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
export async function moveToJob (options: { export async function moveToJob (options: {
jobId: string jobId: string
@ -11,10 +13,20 @@ export async function moveToJob (options: {
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void> moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void> moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void> moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void> doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
}) { }) {
const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options const {
jobId,
loggerTags,
videoUUID,
moveVideoSourceFile,
moveHLSFiles,
moveWebVideoFiles,
moveToFailedState,
doAfterLastMove
} = options
const lTagsBase = loggerTagsFactory(...loggerTags) const lTagsBase = loggerTagsFactory(...loggerTags)
@ -31,6 +43,13 @@ export async function moveToJob (options: {
const lTags = lTagsBase(video.uuid, video.url) const lTags = lTagsBase(video.uuid, video.url)
try { try {
const source = await VideoSourceModel.loadLatest(video.id)
if (source) {
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
await moveVideoSourceFile(source)
}
if (video.VideoFiles) { if (video.VideoFiles) {
logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)

View file

@ -159,6 +159,20 @@ export async function makeWebVideoFileAvailable (filename: string, destination:
return destination return destination
} }
export async function makeOriginalFileAvailable (keptOriginalFilename: string, destination: string) {
const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
logger.info('Fetching Original Video file %s from object storage to %s.', key, destination, lTags())
await makeAvailable({
key,
destination,
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES
})
return destination
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export function getWebVideoFileReadStream (options: { export function getWebVideoFileReadStream (options: {