Add script to move videos to file system
This commit is contained in:
parent
443358ccce
commit
d3c9a2e5b9
24 changed files with 545 additions and 237 deletions
|
@ -36,7 +36,7 @@ export class LoginPage {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.isMobileDevice) {
|
if (this.isMobileDevice) {
|
||||||
const menuToggle = $('.top-left-block span[role=button]')
|
const menuToggle = $('.top-left-block button')
|
||||||
|
|
||||||
await $('h2=Our content selection').waitForDisplayed()
|
await $('h2=Our content selection').waitForDisplayed()
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,7 @@ export class PlayerPage {
|
||||||
|
|
||||||
await browser.waitUntil(async () => {
|
await browser.waitUntil(async () => {
|
||||||
return (await this.getWatchVideoPlayerCurrentTime()) >= waitUntilSec
|
return (await this.getWatchVideoPlayerCurrentTime()) >= waitUntilSec
|
||||||
}, { timeout: waitUntilSec * 2 * 1000 })
|
}, { timeout: Math.max(waitUntilSec * 2 * 1000, 30000) })
|
||||||
|
|
||||||
// Pause video
|
// Pause video
|
||||||
await $('div.video-js').click()
|
await $('div.video-js').click()
|
||||||
|
|
|
@ -37,6 +37,7 @@ export class JobsComponent extends RestTable implements OnInit {
|
||||||
'federate-video',
|
'federate-video',
|
||||||
'manage-video-torrent',
|
'manage-video-torrent',
|
||||||
'move-to-object-storage',
|
'move-to-object-storage',
|
||||||
|
'move-to-file-system',
|
||||||
'notify',
|
'notify',
|
||||||
'video-channel-import',
|
'video-channel-import',
|
||||||
'video-file-import',
|
'video-file-import',
|
||||||
|
|
|
@ -1,27 +1,3 @@
|
||||||
<div i18n class="alert alert-warning" *ngIf="isVideoTranscodingFailed()">
|
|
||||||
Transcoding failed, this video may not work properly.
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div i18n class="alert alert-warning" *ngIf="isVideoMoveToObjectStorageFailed()">
|
|
||||||
Move to external storage failed, this video may not work properly.
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div i18n class="alert alert-warning" *ngIf="isVideoToImport()">
|
|
||||||
The video is being imported, it will be available when the import is finished.
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div i18n class="alert alert-warning" *ngIf="isVideoToTranscode()">
|
|
||||||
The video is being transcoded, it may not work properly.
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div i18n class="alert alert-warning" *ngIf="isVideoToEdit()">
|
|
||||||
The video is being edited, it may not work properly.
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div i18n class="alert alert-warning" *ngIf="isVideoToMoveToExternalStorage()">
|
|
||||||
The video is being moved to an external server, it may not work properly.
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div i18n class="alert pt-alert-primary" *ngIf="hasVideoScheduledPublication()">
|
<div i18n class="alert pt-alert-primary" *ngIf="hasVideoScheduledPublication()">
|
||||||
This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}.
|
This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}.
|
||||||
</div>
|
</div>
|
||||||
|
@ -34,6 +10,10 @@
|
||||||
This live has ended.
|
This live has ended.
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="alert alert-warning" *ngIf="getAlertWarning()">
|
||||||
|
{{ getAlertWarning() }}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div i18n class="alert alert-warning" *ngIf="noPlaylistVideoFound">
|
<div i18n class="alert alert-warning" *ngIf="noPlaylistVideoFound">
|
||||||
There are no videos available in this playlist.
|
There are no videos available in this playlist.
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -13,28 +13,34 @@ export class VideoAlertComponent {
|
||||||
@Input() video: VideoDetails
|
@Input() video: VideoDetails
|
||||||
@Input() noPlaylistVideoFound: boolean
|
@Input() noPlaylistVideoFound: boolean
|
||||||
|
|
||||||
isVideoToTranscode () {
|
getAlertWarning () {
|
||||||
return this.video && this.video.state.id === VideoState.TO_TRANSCODE
|
if (!this.video) return
|
||||||
}
|
|
||||||
|
|
||||||
isVideoToEdit () {
|
switch (this.video.state.id) {
|
||||||
return this.video && this.video.state.id === VideoState.TO_EDIT
|
case VideoState.TO_TRANSCODE:
|
||||||
}
|
return $localize`The video is being transcoded, it may not work properly.`
|
||||||
|
|
||||||
isVideoTranscodingFailed () {
|
case VideoState.TO_IMPORT:
|
||||||
return this.video && this.video.state.id === VideoState.TRANSCODING_FAILED
|
return $localize`The video is being imported, it will be available when the import is finished.`
|
||||||
}
|
|
||||||
|
|
||||||
isVideoMoveToObjectStorageFailed () {
|
case VideoState.TO_MOVE_TO_FILE_SYSTEM:
|
||||||
return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED
|
return $localize`The video is being moved to server file system, it may not work properly`
|
||||||
}
|
|
||||||
|
|
||||||
isVideoToImport () {
|
case VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED:
|
||||||
return this.video && this.video.state.id === VideoState.TO_IMPORT
|
return $localize`Move to file system failed, this video may not work properly.`
|
||||||
}
|
|
||||||
|
|
||||||
isVideoToMoveToExternalStorage () {
|
case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE:
|
||||||
return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
|
return $localize`The video is being moved to an external server, it may not work properly.`
|
||||||
|
|
||||||
|
case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED:
|
||||||
|
return $localize`Move to external storage failed, this video may not work properly.`
|
||||||
|
|
||||||
|
case VideoState.TO_EDIT:
|
||||||
|
return $localize`The video is being edited, it may not work properly.`
|
||||||
|
|
||||||
|
case VideoState.TRANSCODING_FAILED:
|
||||||
|
return $localize`Transcoding failed, this video may not work properly.`
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hasVideoScheduledPublication () {
|
hasVideoScheduledPublication () {
|
||||||
|
|
|
@ -187,27 +187,31 @@ export class VideoMiniatureComponent implements OnInit {
|
||||||
return $localize`Publication scheduled on ${updateAt}`
|
return $localize`Publication scheduled on ${updateAt}`
|
||||||
}
|
}
|
||||||
|
|
||||||
if (video.state.id === VideoState.TRANSCODING_FAILED) {
|
switch (video.state.id) {
|
||||||
|
case VideoState.TRANSCODING_FAILED:
|
||||||
return $localize`Transcoding failed`
|
return $localize`Transcoding failed`
|
||||||
}
|
|
||||||
|
|
||||||
if (video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) {
|
case VideoState.TO_MOVE_TO_FILE_SYSTEM:
|
||||||
|
return $localize`Moving to file system`
|
||||||
|
|
||||||
|
case VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED:
|
||||||
|
return $localize`Moving to file system failed`
|
||||||
|
|
||||||
|
case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE:
|
||||||
|
return $localize`Moving to external storage`
|
||||||
|
|
||||||
|
case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED:
|
||||||
return $localize`Move to external storage failed`
|
return $localize`Move to external storage failed`
|
||||||
}
|
|
||||||
|
|
||||||
if (video.state.id === VideoState.TO_TRANSCODE && video.waitTranscoding === true) {
|
case VideoState.TO_TRANSCODE:
|
||||||
return $localize`Waiting transcoding`
|
return video.waitTranscoding === true
|
||||||
}
|
? $localize`Waiting transcoding`
|
||||||
|
: $localize`To transcode`
|
||||||
|
|
||||||
if (video.state.id === VideoState.TO_TRANSCODE) {
|
case VideoState.TO_IMPORT:
|
||||||
return $localize`To transcode`
|
|
||||||
}
|
|
||||||
|
|
||||||
if (video.state.id === VideoState.TO_IMPORT) {
|
|
||||||
return $localize`To import`
|
return $localize`To import`
|
||||||
}
|
|
||||||
|
|
||||||
if (video.state.id === VideoState.TO_EDIT) {
|
case VideoState.TO_EDIT:
|
||||||
return $localize`To edit`
|
return $localize`To edit`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@ export type JobType =
|
||||||
| 'transcoding-job-builder'
|
| 'transcoding-job-builder'
|
||||||
| 'manage-video-torrent'
|
| 'manage-video-torrent'
|
||||||
| 'move-to-object-storage'
|
| 'move-to-object-storage'
|
||||||
|
| 'move-to-file-system'
|
||||||
| 'notify'
|
| 'notify'
|
||||||
| 'video-channel-import'
|
| 'video-channel-import'
|
||||||
| 'video-file-import'
|
| 'video-file-import'
|
||||||
|
@ -196,7 +197,7 @@ export interface DeleteResumableUploadMetaFilePayload {
|
||||||
filepath: string
|
filepath: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MoveObjectStoragePayload {
|
export interface MoveStoragePayload {
|
||||||
videoUUID: string
|
videoUUID: string
|
||||||
isNewVideo: boolean
|
isNewVideo: boolean
|
||||||
previousVideoState: VideoStateType
|
previousVideoState: VideoStateType
|
||||||
|
|
|
@ -7,7 +7,9 @@ export const VideoState = {
|
||||||
TO_MOVE_TO_EXTERNAL_STORAGE: 6,
|
TO_MOVE_TO_EXTERNAL_STORAGE: 6,
|
||||||
TRANSCODING_FAILED: 7,
|
TRANSCODING_FAILED: 7,
|
||||||
TO_MOVE_TO_EXTERNAL_STORAGE_FAILED: 8,
|
TO_MOVE_TO_EXTERNAL_STORAGE_FAILED: 8,
|
||||||
TO_EDIT: 9
|
TO_EDIT: 9,
|
||||||
|
TO_MOVE_TO_FILE_SYSTEM: 10,
|
||||||
|
TO_MOVE_TO_FILE_SYSTEM_FAILED: 11
|
||||||
} as const
|
} as const
|
||||||
|
|
||||||
export type VideoStateType = typeof VideoState[keyof typeof VideoState]
|
export type VideoStateType = typeof VideoState[keyof typeof VideoState]
|
||||||
|
|
|
@ -15,6 +15,7 @@ import {
|
||||||
} from '@peertube/peertube-server-commands'
|
} from '@peertube/peertube-server-commands'
|
||||||
import { expectStartWith } from '../shared/checks.js'
|
import { expectStartWith } from '../shared/checks.js'
|
||||||
import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
|
import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
|
||||||
|
import { getAllFiles } from '@peertube/peertube-core-utils'
|
||||||
|
|
||||||
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
|
async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
|
||||||
for (const file of video.files) {
|
for (const file of video.files) {
|
||||||
|
@ -73,6 +74,8 @@ describe('Test create move video storage job', function () {
|
||||||
await servers[0].run(objectStorage.getDefaultMockConfig())
|
await servers[0].run(objectStorage.getDefaultMockConfig())
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('To object storage', function () {
|
||||||
|
|
||||||
it('Should move only one file', async function () {
|
it('Should move only one file', async function () {
|
||||||
this.timeout(120000)
|
this.timeout(120000)
|
||||||
|
|
||||||
|
@ -116,6 +119,62 @@ describe('Test create move video storage job', function () {
|
||||||
await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls'), [ 'private' ])
|
await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls'), [ 'private' ])
|
||||||
await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls', 'private'))
|
await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls', 'private'))
|
||||||
})
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('To file system', function () {
|
||||||
|
let oldFileUrls: string[]
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
const video = await servers[0].videos.get({ id: uuids[1] })
|
||||||
|
|
||||||
|
oldFileUrls = [
|
||||||
|
...getAllFiles(video).map(f => f.fileUrl),
|
||||||
|
video.streamingPlaylists[0].playlistUrl
|
||||||
|
]
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should move only one file', async function () {
|
||||||
|
this.timeout(120000)
|
||||||
|
|
||||||
|
const command = `npm run create-move-video-storage-job -- --to-file-system -v ${uuids[1]}`
|
||||||
|
await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
|
||||||
|
await waitJobs(servers)
|
||||||
|
|
||||||
|
for (const server of servers) {
|
||||||
|
const video = await server.videos.get({ id: uuids[1] })
|
||||||
|
|
||||||
|
await checkFiles(servers[0], video)
|
||||||
|
|
||||||
|
for (const id of [ uuids[0], uuids[2] ]) {
|
||||||
|
const video = await server.videos.get({ id })
|
||||||
|
|
||||||
|
await checkFiles(servers[0], video, objectStorage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should move all files', async function () {
|
||||||
|
this.timeout(120000)
|
||||||
|
|
||||||
|
const command = `npm run create-move-video-storage-job -- --to-file-system --all-videos`
|
||||||
|
await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
|
||||||
|
await waitJobs(servers)
|
||||||
|
|
||||||
|
for (const server of servers) {
|
||||||
|
for (const id of [ uuids[0], uuids[2] ]) {
|
||||||
|
const video = await server.videos.get({ id })
|
||||||
|
|
||||||
|
await checkFiles(servers[0], video)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should not have files on disk anymore', async function () {
|
||||||
|
for (const fileUrl of oldFileUrls) {
|
||||||
|
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
after(async function () {
|
after(async function () {
|
||||||
await objectStorage.cleanupMock()
|
await objectStorage.cleanupMock()
|
||||||
|
|
|
@ -5,7 +5,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-q
|
||||||
import { Hooks } from '@server/lib/plugins/hooks.js'
|
import { Hooks } from '@server/lib/plugins/hooks.js'
|
||||||
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
|
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
|
||||||
import { uploadx } from '@server/lib/uploadx.js'
|
import { uploadx } from '@server/lib/uploadx.js'
|
||||||
import { buildMoveToObjectStorageJob } from '@server/lib/video.js'
|
import { buildMoveJob } from '@server/lib/video.js'
|
||||||
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
|
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
|
||||||
import { buildNewFile } from '@server/lib/video-file.js'
|
import { buildNewFile } from '@server/lib/video-file.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
|
@ -171,7 +171,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
|
||||||
]
|
]
|
||||||
|
|
||||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
||||||
jobs.push(await buildMoveToObjectStorageJob({ video, isNewVideo: false, previousVideoState: undefined }))
|
jobs.push(await buildMoveJob({ video, isNewVideo: false, previousVideoState: undefined, type: 'move-to-object-storage' }))
|
||||||
}
|
}
|
||||||
|
|
||||||
if (video.state === VideoState.TO_TRANSCODE) {
|
if (video.state === VideoState.TO_TRANSCODE) {
|
||||||
|
|
|
@ -6,7 +6,7 @@ import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
|
||||||
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
|
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
|
||||||
import { Redis } from '@server/lib/redis.js'
|
import { Redis } from '@server/lib/redis.js'
|
||||||
import { uploadx } from '@server/lib/uploadx.js'
|
import { uploadx } from '@server/lib/uploadx.js'
|
||||||
import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
|
import { buildLocalVideoFromReq, buildMoveJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
|
||||||
import { buildNewFile } from '@server/lib/video-file.js'
|
import { buildNewFile } from '@server/lib/video-file.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
import { buildNextVideoState } from '@server/lib/video-state.js'
|
import { buildNextVideoState } from '@server/lib/video-state.js'
|
||||||
|
@ -275,7 +275,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
|
||||||
]
|
]
|
||||||
|
|
||||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
||||||
jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }))
|
jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' }))
|
||||||
}
|
}
|
||||||
|
|
||||||
if (video.state === VideoState.TO_TRANSCODE) {
|
if (video.state === VideoState.TO_TRANSCODE) {
|
||||||
|
|
|
@ -121,7 +121,8 @@ const bunyanLogger = {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
|
type LoggerTags = { tags: string[] }
|
||||||
|
type LoggerTagsFn = (...tags: string[]) => LoggerTags
|
||||||
function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
|
function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
|
||||||
return (...tags: string[]) => {
|
return (...tags: string[]) => {
|
||||||
return { tags: defaultTags.concat(tags) }
|
return { tags: defaultTags.concat(tags) }
|
||||||
|
@ -154,6 +155,7 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
|
||||||
|
|
||||||
export {
|
export {
|
||||||
type LoggerTagsFn,
|
type LoggerTagsFn,
|
||||||
|
type LoggerTags,
|
||||||
|
|
||||||
buildLogger,
|
buildLogger,
|
||||||
timestampFormatter,
|
timestampFormatter,
|
||||||
|
|
|
@ -186,6 +186,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
|
||||||
'video-channel-import': 1,
|
'video-channel-import': 1,
|
||||||
'after-video-channel-import': 1,
|
'after-video-channel-import': 1,
|
||||||
'move-to-object-storage': 3,
|
'move-to-object-storage': 3,
|
||||||
|
'move-to-file-system': 3,
|
||||||
'transcoding-job-builder': 1,
|
'transcoding-job-builder': 1,
|
||||||
'generate-video-storyboard': 1,
|
'generate-video-storyboard': 1,
|
||||||
'notify': 1,
|
'notify': 1,
|
||||||
|
@ -209,6 +210,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
|
||||||
'video-studio-edition': 1,
|
'video-studio-edition': 1,
|
||||||
'manage-video-torrent': 1,
|
'manage-video-torrent': 1,
|
||||||
'move-to-object-storage': 1,
|
'move-to-object-storage': 1,
|
||||||
|
'move-to-file-system': 1,
|
||||||
'video-channel-import': 1,
|
'video-channel-import': 1,
|
||||||
'after-video-channel-import': 1,
|
'after-video-channel-import': 1,
|
||||||
'transcoding-job-builder': 1,
|
'transcoding-job-builder': 1,
|
||||||
|
@ -236,6 +238,7 @@ const JOB_TTL: { [id in JobType]: number } = {
|
||||||
'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
|
'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
|
||||||
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
|
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
|
||||||
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
|
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
|
||||||
|
'move-to-file-system': 1000 * 60 * 60 * 3, // 3 hours
|
||||||
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
|
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
|
||||||
'after-video-channel-import': 60000 * 5, // 5 minutes
|
'after-video-channel-import': 60000 * 5, // 5 minutes
|
||||||
'transcoding-job-builder': 60000, // 1 minute
|
'transcoding-job-builder': 60000, // 1 minute
|
||||||
|
@ -557,7 +560,9 @@ const VIDEO_STATES: { [ id in VideoStateType ]: string } = {
|
||||||
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
|
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
|
||||||
[VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
|
[VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
|
||||||
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed',
|
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed',
|
||||||
[VideoState.TO_EDIT]: 'To edit*'
|
[VideoState.TO_EDIT]: 'To edit',
|
||||||
|
[VideoState.TO_MOVE_TO_FILE_SYSTEM]: 'To move to file system',
|
||||||
|
[VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED]: 'Move to file system failed'
|
||||||
}
|
}
|
||||||
|
|
||||||
const VIDEO_IMPORT_STATES: { [ id in VideoImportStateType ]: string } = {
|
const VIDEO_IMPORT_STATES: { [ id in VideoImportStateType ]: string } = {
|
||||||
|
|
138
server/core/lib/job-queue/handlers/move-to-file-system.ts
Normal file
138
server/core/lib/job-queue/handlers/move-to-file-system.ts
Normal file
|
@ -0,0 +1,138 @@
|
||||||
|
import { Job } from 'bullmq'
|
||||||
|
import { join } from 'path'
|
||||||
|
import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
|
||||||
|
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||||
|
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||||
|
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||||
|
import {
|
||||||
|
makeHLSFileAvailable,
|
||||||
|
makeWebVideoFileAvailable,
|
||||||
|
removeHLSFileObjectStorageByFilename,
|
||||||
|
removeHLSObjectStorage,
|
||||||
|
removeWebVideoObjectStorage
|
||||||
|
} from '@server/lib/object-storage/index.js'
|
||||||
|
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||||
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
|
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
|
||||||
|
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||||
|
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
||||||
|
|
||||||
|
const lTagsBase = loggerTagsFactory('move-file-system')
|
||||||
|
|
||||||
|
export async function processMoveToFileSystem (job: Job) {
|
||||||
|
const payload = job.data as MoveStoragePayload
|
||||||
|
logger.info('Moving video %s to file system in job %s.', payload.videoUUID, job.id)
|
||||||
|
|
||||||
|
await moveToJob({
|
||||||
|
jobId: job.id,
|
||||||
|
videoUUID: payload.videoUUID,
|
||||||
|
loggerTags: lTagsBase().tags,
|
||||||
|
|
||||||
|
moveWebVideoFiles,
|
||||||
|
moveHLSFiles,
|
||||||
|
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
||||||
|
moveToFailedState: moveToFailedMoveToFileSystemState
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function onMoveToFileSystemFailure (job: Job, err: any) {
|
||||||
|
const payload = job.data as MoveStoragePayload
|
||||||
|
|
||||||
|
await onMoveToStorageFailure({
|
||||||
|
videoUUID: payload.videoUUID,
|
||||||
|
err,
|
||||||
|
lTags: lTagsBase(),
|
||||||
|
moveToFailedState: moveToFailedMoveToFileSystemState
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Private
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||||
|
for (const file of video.VideoFiles) {
|
||||||
|
if (file.storage === VideoStorage.FILE_SYSTEM) continue
|
||||||
|
|
||||||
|
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
|
||||||
|
await onFileMoved({
|
||||||
|
videoOrPlaylist: video,
|
||||||
|
file,
|
||||||
|
objetStorageRemover: () => removeWebVideoObjectStorage(file)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||||
|
for (const playlist of video.VideoStreamingPlaylists) {
|
||||||
|
const playlistWithVideo = playlist.withVideo(video)
|
||||||
|
|
||||||
|
for (const file of playlist.VideoFiles) {
|
||||||
|
if (file.storage === VideoStorage.FILE_SYSTEM) continue
|
||||||
|
|
||||||
|
// Resolution playlist
|
||||||
|
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
|
||||||
|
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
|
||||||
|
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
|
||||||
|
|
||||||
|
await onFileMoved({
|
||||||
|
videoOrPlaylist: playlistWithVideo,
|
||||||
|
file,
|
||||||
|
objetStorageRemover: async () => {
|
||||||
|
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
|
||||||
|
await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onFileMoved (options: {
|
||||||
|
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||||
|
file: MVideoFile
|
||||||
|
objetStorageRemover: () => Promise<any>
|
||||||
|
}) {
|
||||||
|
const { videoOrPlaylist, file, objetStorageRemover } = options
|
||||||
|
|
||||||
|
const oldFileUrl = file.fileUrl
|
||||||
|
|
||||||
|
file.fileUrl = null
|
||||||
|
file.storage = VideoStorage.FILE_SYSTEM
|
||||||
|
|
||||||
|
await updateTorrentMetadata(videoOrPlaylist, file)
|
||||||
|
await file.save()
|
||||||
|
|
||||||
|
logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||||
|
await objetStorageRemover()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function doAfterLastMove (options: {
|
||||||
|
video: MVideoWithAllFiles
|
||||||
|
previousVideoState: VideoStateType
|
||||||
|
isNewVideo: boolean
|
||||||
|
}) {
|
||||||
|
const { video, previousVideoState, isNewVideo } = options
|
||||||
|
|
||||||
|
for (const playlist of video.VideoStreamingPlaylists) {
|
||||||
|
if (playlist.storage === VideoStorage.FILE_SYSTEM) continue
|
||||||
|
|
||||||
|
const playlistWithVideo = playlist.withVideo(video)
|
||||||
|
|
||||||
|
for (const filename of [ playlist.playlistFilename, playlist.segmentsSha256Filename ]) {
|
||||||
|
await makeHLSFileAvailable(playlistWithVideo, filename, join(getHLSDirectory(video), filename))
|
||||||
|
}
|
||||||
|
|
||||||
|
playlist.playlistUrl = null
|
||||||
|
playlist.segmentsSha256Url = null
|
||||||
|
playlist.storage = VideoStorage.FILE_SYSTEM
|
||||||
|
|
||||||
|
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
||||||
|
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
||||||
|
|
||||||
|
await playlist.save()
|
||||||
|
|
||||||
|
await removeHLSObjectStorage(playlistWithVideo)
|
||||||
|
}
|
||||||
|
|
||||||
|
await moveToNextState({ video, previousVideoState, isNewVideo })
|
||||||
|
}
|
|
@ -1,7 +1,7 @@
|
||||||
import { Job } from 'bullmq'
|
import { Job } from 'bullmq'
|
||||||
import { remove } from 'fs-extra/esm'
|
import { remove } from 'fs-extra/esm'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { MoveObjectStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
|
import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
|
||||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||||
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
|
||||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||||
|
@ -9,68 +9,36 @@ import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-
|
||||||
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
|
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
|
||||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
|
||||||
import { VideoModel } from '@server/models/video/video.js'
|
|
||||||
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||||
|
import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
|
||||||
|
|
||||||
const lTagsBase = loggerTagsFactory('move-object-storage')
|
const lTagsBase = loggerTagsFactory('move-object-storage')
|
||||||
|
|
||||||
export async function processMoveToObjectStorage (job: Job) {
|
export async function processMoveToObjectStorage (job: Job) {
|
||||||
const payload = job.data as MoveObjectStoragePayload
|
const payload = job.data as MoveStoragePayload
|
||||||
logger.info('Moving video %s in job %s.', payload.videoUUID, job.id)
|
logger.info('Moving video %s to object storage in job %s.', payload.videoUUID, job.id)
|
||||||
|
|
||||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
|
await moveToJob({
|
||||||
|
jobId: job.id,
|
||||||
|
videoUUID: payload.videoUUID,
|
||||||
|
loggerTags: lTagsBase().tags,
|
||||||
|
|
||||||
const video = await VideoModel.loadWithFiles(payload.videoUUID)
|
moveWebVideoFiles,
|
||||||
// No video, maybe deleted?
|
moveHLSFiles,
|
||||||
if (!video) {
|
doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
|
||||||
logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
|
moveToFailedState: moveToFailedMoveToObjectStorageState
|
||||||
fileMutexReleaser()
|
})
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
const lTags = lTagsBase(video.uuid, video.url)
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (video.VideoFiles) {
|
|
||||||
logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
|
|
||||||
|
|
||||||
await moveWebVideoFiles(video)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (video.VideoStreamingPlaylists) {
|
|
||||||
logger.debug('Moving HLS playlist of %s.', video.uuid)
|
|
||||||
|
|
||||||
await moveHLSFiles(video)
|
|
||||||
}
|
|
||||||
|
|
||||||
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
|
||||||
if (pendingMove === 0) {
|
|
||||||
logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags)
|
|
||||||
|
|
||||||
await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
await onMoveToObjectStorageFailure(job, err)
|
|
||||||
|
|
||||||
throw err
|
|
||||||
} finally {
|
|
||||||
fileMutexReleaser()
|
|
||||||
}
|
|
||||||
|
|
||||||
return payload.videoUUID
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function onMoveToObjectStorageFailure (job: Job, err: any) {
|
export async function onMoveToObjectStorageFailure (job: Job, err: any) {
|
||||||
const payload = job.data as MoveObjectStoragePayload
|
const payload = job.data as MoveStoragePayload
|
||||||
|
|
||||||
const video = await VideoModel.loadWithFiles(payload.videoUUID)
|
await onMoveToStorageFailure({
|
||||||
if (!video) return
|
videoUUID: payload.videoUUID,
|
||||||
|
err,
|
||||||
logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
|
lTags: lTagsBase(),
|
||||||
|
moveToFailedState: moveToFailedMoveToObjectStorageState
|
||||||
await moveToFailedMoveToObjectStorageState(video)
|
})
|
||||||
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
@ -107,39 +75,6 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function doAfterLastJob (options: {
|
|
||||||
video: MVideoWithAllFiles
|
|
||||||
previousVideoState: VideoStateType
|
|
||||||
isNewVideo: boolean
|
|
||||||
}) {
|
|
||||||
const { video, previousVideoState, isNewVideo } = options
|
|
||||||
|
|
||||||
for (const playlist of video.VideoStreamingPlaylists) {
|
|
||||||
if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
|
|
||||||
|
|
||||||
const playlistWithVideo = playlist.withVideo(video)
|
|
||||||
|
|
||||||
// Master playlist
|
|
||||||
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
|
|
||||||
// Sha256 segments file
|
|
||||||
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
|
|
||||||
|
|
||||||
playlist.storage = VideoStorage.OBJECT_STORAGE
|
|
||||||
|
|
||||||
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
|
||||||
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
|
||||||
|
|
||||||
await playlist.save()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove empty hls video directory
|
|
||||||
if (video.VideoStreamingPlaylists) {
|
|
||||||
await remove(getHLSDirectory(video))
|
|
||||||
}
|
|
||||||
|
|
||||||
await moveToNextState({ video, previousVideoState, isNewVideo })
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onFileMoved (options: {
|
async function onFileMoved (options: {
|
||||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||||
file: MVideoFile
|
file: MVideoFile
|
||||||
|
@ -154,6 +89,32 @@ async function onFileMoved (options: {
|
||||||
await updateTorrentMetadata(videoOrPlaylist, file)
|
await updateTorrentMetadata(videoOrPlaylist, file)
|
||||||
await file.save()
|
await file.save()
|
||||||
|
|
||||||
logger.debug('Removing %s because it\'s now on object storage', oldPath)
|
logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
|
||||||
await remove(oldPath)
|
await remove(oldPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function doAfterLastMove (options: {
|
||||||
|
video: MVideoWithAllFiles
|
||||||
|
previousVideoState: VideoStateType
|
||||||
|
isNewVideo: boolean
|
||||||
|
}) {
|
||||||
|
const { video, previousVideoState, isNewVideo } = options
|
||||||
|
|
||||||
|
for (const playlist of video.VideoStreamingPlaylists) {
|
||||||
|
if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
|
||||||
|
|
||||||
|
const playlistWithVideo = playlist.withVideo(video)
|
||||||
|
|
||||||
|
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
|
||||||
|
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
|
||||||
|
playlist.storage = VideoStorage.OBJECT_STORAGE
|
||||||
|
|
||||||
|
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
||||||
|
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
||||||
|
|
||||||
|
await playlist.save()
|
||||||
|
}
|
||||||
|
|
||||||
|
await remove(getHLSDirectory(video))
|
||||||
|
await moveToNextState({ video, previousVideoState, isNewVideo })
|
||||||
|
}
|
||||||
|
|
76
server/core/lib/job-queue/handlers/shared/move-video.ts
Normal file
76
server/core/lib/job-queue/handlers/shared/move-video.ts
Normal file
|
@ -0,0 +1,76 @@
|
||||||
|
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||||
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
|
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||||
|
import { VideoModel } from '@server/models/video/video.js'
|
||||||
|
import { MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||||
|
|
||||||
|
export async function moveToJob (options: {
|
||||||
|
jobId: string
|
||||||
|
videoUUID: string
|
||||||
|
loggerTags: string[]
|
||||||
|
|
||||||
|
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||||
|
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||||
|
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
|
||||||
|
doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
|
||||||
|
}) {
|
||||||
|
const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options
|
||||||
|
|
||||||
|
const lTagsBase = loggerTagsFactory(...loggerTags)
|
||||||
|
|
||||||
|
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoUUID)
|
||||||
|
|
||||||
|
const video = await VideoModel.loadWithFiles(videoUUID)
|
||||||
|
// No video, maybe deleted?
|
||||||
|
if (!video) {
|
||||||
|
logger.info('Can\'t process job %d, video does not exist.', jobId, lTagsBase(videoUUID))
|
||||||
|
fileMutexReleaser()
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
const lTags = lTagsBase(video.uuid, video.url)
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (video.VideoFiles) {
|
||||||
|
logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
|
||||||
|
|
||||||
|
await moveWebVideoFiles(video)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (video.VideoStreamingPlaylists) {
|
||||||
|
logger.debug('Moving HLS playlist of %s.', video.uuid)
|
||||||
|
|
||||||
|
await moveHLSFiles(video)
|
||||||
|
}
|
||||||
|
|
||||||
|
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
||||||
|
if (pendingMove === 0) {
|
||||||
|
logger.info('Running cleanup after moving files (video %s in job %s)', video.uuid, jobId, lTags)
|
||||||
|
|
||||||
|
await doAfterLastMove(video)
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
await onMoveToStorageFailure({ videoUUID, err, lTags, moveToFailedState })
|
||||||
|
|
||||||
|
throw err
|
||||||
|
} finally {
|
||||||
|
fileMutexReleaser()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function onMoveToStorageFailure (options: {
|
||||||
|
videoUUID: string
|
||||||
|
err: any
|
||||||
|
lTags: LoggerTags
|
||||||
|
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
|
||||||
|
}) {
|
||||||
|
const { videoUUID, err, lTags, moveToFailedState } = options
|
||||||
|
|
||||||
|
const video = await VideoModel.loadWithFiles(videoUUID)
|
||||||
|
if (!video) return
|
||||||
|
|
||||||
|
logger.error('Cannot move video %s storage.', video.url, { err, ...lTags })
|
||||||
|
|
||||||
|
await moveToFailedState(video)
|
||||||
|
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
||||||
|
}
|
|
@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config.js'
|
||||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||||
import { generateWebVideoFilename } from '@server/lib/paths.js'
|
import { generateWebVideoFilename } from '@server/lib/paths.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
import { buildMoveToObjectStorageJob } from '@server/lib/video.js'
|
import { buildMoveJob } from '@server/lib/video.js'
|
||||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||||
import { VideoModel } from '@server/models/video/video.js'
|
import { VideoModel } from '@server/models/video/video.js'
|
||||||
import { MVideoFullLight } from '@server/types/models/index.js'
|
import { MVideoFullLight } from '@server/types/models/index.js'
|
||||||
|
@ -30,7 +30,7 @@ async function processVideoFileImport (job: Job) {
|
||||||
await updateVideoFile(video, payload.filePath)
|
await updateVideoFile(video, payload.filePath)
|
||||||
|
|
||||||
if (CONFIG.OBJECT_STORAGE.ENABLED) {
|
if (CONFIG.OBJECT_STORAGE.ENABLED) {
|
||||||
await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState: video.state }))
|
await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState: video.state, type: 'move-to-object-storage' }))
|
||||||
} else {
|
} else {
|
||||||
await federateVideoIfNeeded(video, false)
|
await federateVideoIfNeeded(video, false)
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,7 @@ import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-t
|
||||||
import { isAbleToUploadVideo } from '@server/lib/user.js'
|
import { isAbleToUploadVideo } from '@server/lib/user.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
import { buildNextVideoState } from '@server/lib/video-state.js'
|
import { buildNextVideoState } from '@server/lib/video-state.js'
|
||||||
import { buildMoveToObjectStorageJob } from '@server/lib/video.js'
|
import { buildMoveJob } from '@server/lib/video.js'
|
||||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
|
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
|
||||||
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
|
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
|
||||||
import { getLowercaseExtension } from '@peertube/peertube-node-utils'
|
import { getLowercaseExtension } from '@peertube/peertube-node-utils'
|
||||||
|
@ -317,7 +317,7 @@ async function afterImportSuccess (options: {
|
||||||
|
|
||||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
||||||
await JobQueue.Instance.createJob(
|
await JobQueue.Instance.createJob(
|
||||||
await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
|
await buildMoveJob({ video, previousVideoState: VideoState.TO_IMPORT, type: 'move-to-object-storage' })
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,7 @@ import {
|
||||||
JobState,
|
JobState,
|
||||||
JobType,
|
JobType,
|
||||||
ManageVideoTorrentPayload,
|
ManageVideoTorrentPayload,
|
||||||
MoveObjectStoragePayload,
|
MoveStoragePayload,
|
||||||
NotifyPayload,
|
NotifyPayload,
|
||||||
RefreshPayload,
|
RefreshPayload,
|
||||||
TranscodingJobBuilderPayload,
|
TranscodingJobBuilderPayload,
|
||||||
|
@ -70,6 +70,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending.js'
|
||||||
import { processVideoStudioEdition } from './handlers/video-studio-edition.js'
|
import { processVideoStudioEdition } from './handlers/video-studio-edition.js'
|
||||||
import { processVideoTranscoding } from './handlers/video-transcoding.js'
|
import { processVideoTranscoding } from './handlers/video-transcoding.js'
|
||||||
import { processVideosViewsStats } from './handlers/video-views-stats.js'
|
import { processVideosViewsStats } from './handlers/video-views-stats.js'
|
||||||
|
import { onMoveToFileSystemFailure, processMoveToFileSystem } from './handlers/move-to-file-system.js'
|
||||||
|
|
||||||
export type CreateJobArgument =
|
export type CreateJobArgument =
|
||||||
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
|
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
|
||||||
|
@ -91,11 +92,11 @@ export type CreateJobArgument =
|
||||||
{ type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
|
{ type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
|
||||||
{ type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
|
{ type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
|
||||||
{ type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
|
{ type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
|
||||||
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
|
{ type: 'move-to-object-storage', payload: MoveStoragePayload } |
|
||||||
|
{ type: 'move-to-file-system', payload: MoveStoragePayload } |
|
||||||
{ type: 'video-channel-import', payload: VideoChannelImportPayload } |
|
{ type: 'video-channel-import', payload: VideoChannelImportPayload } |
|
||||||
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
|
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
|
||||||
{ type: 'notify', payload: NotifyPayload } |
|
{ type: 'notify', payload: NotifyPayload } |
|
||||||
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
|
|
||||||
{ type: 'federate-video', payload: FederateVideoPayload } |
|
{ type: 'federate-video', payload: FederateVideoPayload } |
|
||||||
{ type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
|
{ type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
|
||||||
|
|
||||||
|
@ -120,6 +121,7 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
|
||||||
'transcoding-job-builder': processTranscodingJobBuilder,
|
'transcoding-job-builder': processTranscodingJobBuilder,
|
||||||
'manage-video-torrent': processManageVideoTorrent,
|
'manage-video-torrent': processManageVideoTorrent,
|
||||||
'move-to-object-storage': processMoveToObjectStorage,
|
'move-to-object-storage': processMoveToObjectStorage,
|
||||||
|
'move-to-file-system': processMoveToFileSystem,
|
||||||
'notify': processNotify,
|
'notify': processNotify,
|
||||||
'video-channel-import': processVideoChannelImport,
|
'video-channel-import': processVideoChannelImport,
|
||||||
'video-file-import': processVideoFileImport,
|
'video-file-import': processVideoFileImport,
|
||||||
|
@ -133,7 +135,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
|
||||||
}
|
}
|
||||||
|
|
||||||
const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
|
const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
|
||||||
'move-to-object-storage': onMoveToObjectStorageFailure
|
'move-to-object-storage': onMoveToObjectStorageFailure,
|
||||||
|
'move-to-file-system': onMoveToFileSystemFailure
|
||||||
}
|
}
|
||||||
|
|
||||||
const jobTypes: JobType[] = [
|
const jobTypes: JobType[] = [
|
||||||
|
@ -151,6 +154,7 @@ const jobTypes: JobType[] = [
|
||||||
'generate-video-storyboard',
|
'generate-video-storyboard',
|
||||||
'manage-video-torrent',
|
'manage-video-torrent',
|
||||||
'move-to-object-storage',
|
'move-to-object-storage',
|
||||||
|
'move-to-file-system',
|
||||||
'notify',
|
'notify',
|
||||||
'transcoding-job-builder',
|
'transcoding-job-builder',
|
||||||
'video-channel-import',
|
'video-channel-import',
|
||||||
|
|
|
@ -10,7 +10,7 @@ import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models/index.
|
||||||
import { federateVideoIfNeeded } from './activitypub/videos/index.js'
|
import { federateVideoIfNeeded } from './activitypub/videos/index.js'
|
||||||
import { JobQueue } from './job-queue/index.js'
|
import { JobQueue } from './job-queue/index.js'
|
||||||
import { Notifier } from './notifier/index.js'
|
import { Notifier } from './notifier/index.js'
|
||||||
import { buildMoveToObjectStorageJob } from './video.js'
|
import { buildMoveJob } from './video.js'
|
||||||
|
|
||||||
function buildNextVideoState (currentState?: VideoStateType) {
|
function buildNextVideoState (currentState?: VideoStateType) {
|
||||||
if (currentState === VideoState.PUBLISHED) {
|
if (currentState === VideoState.PUBLISHED) {
|
||||||
|
@ -21,6 +21,7 @@ function buildNextVideoState (currentState?: VideoStateType) {
|
||||||
currentState !== VideoState.TO_EDIT &&
|
currentState !== VideoState.TO_EDIT &&
|
||||||
currentState !== VideoState.TO_TRANSCODE &&
|
currentState !== VideoState.TO_TRANSCODE &&
|
||||||
currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
|
currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
|
||||||
|
currentState !== VideoState.TO_MOVE_TO_FILE_SYSTEM &&
|
||||||
CONFIG.TRANSCODING.ENABLED
|
CONFIG.TRANSCODING.ENABLED
|
||||||
) {
|
) {
|
||||||
return VideoState.TO_TRANSCODE
|
return VideoState.TO_TRANSCODE
|
||||||
|
@ -28,6 +29,7 @@ function buildNextVideoState (currentState?: VideoStateType) {
|
||||||
|
|
||||||
if (
|
if (
|
||||||
currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
|
currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
|
||||||
|
currentState !== VideoState.TO_MOVE_TO_FILE_SYSTEM &&
|
||||||
CONFIG.OBJECT_STORAGE.ENABLED
|
CONFIG.OBJECT_STORAGE.ENABLED
|
||||||
) {
|
) {
|
||||||
return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
|
return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
|
||||||
|
@ -68,6 +70,8 @@ function moveToNextState (options: {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function moveToExternalStorageState (options: {
|
async function moveToExternalStorageState (options: {
|
||||||
video: MVideoFullLight
|
video: MVideoFullLight
|
||||||
isNewVideo: boolean
|
isNewVideo: boolean
|
||||||
|
@ -90,7 +94,7 @@ async function moveToExternalStorageState (options: {
|
||||||
logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
|
logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState, isNewVideo }))
|
await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState, isNewVideo, type: 'move-to-object-storage' }))
|
||||||
|
|
||||||
return true
|
return true
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
@ -100,6 +104,34 @@ async function moveToExternalStorageState (options: {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function moveToFileSystemState (options: {
|
||||||
|
video: MVideoFullLight
|
||||||
|
isNewVideo: boolean
|
||||||
|
transaction: Transaction
|
||||||
|
}) {
|
||||||
|
const { video, isNewVideo, transaction } = options
|
||||||
|
|
||||||
|
const previousVideoState = video.state
|
||||||
|
|
||||||
|
if (video.state !== VideoState.TO_MOVE_TO_FILE_SYSTEM) {
|
||||||
|
await video.setNewState(VideoState.TO_MOVE_TO_FILE_SYSTEM, false, transaction)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info('Creating move to file system job for video %s.', video.uuid, { tags: [ video.uuid ] })
|
||||||
|
|
||||||
|
try {
|
||||||
|
await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState, isNewVideo, type: 'move-to-file-system' }))
|
||||||
|
|
||||||
|
return true
|
||||||
|
} catch (err) {
|
||||||
|
logger.error('Cannot add move to file system job', { err })
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
function moveToFailedTranscodingState (video: MVideo) {
|
function moveToFailedTranscodingState (video: MVideo) {
|
||||||
if (video.state === VideoState.TRANSCODING_FAILED) return
|
if (video.state === VideoState.TRANSCODING_FAILED) return
|
||||||
|
|
||||||
|
@ -112,11 +144,19 @@ function moveToFailedMoveToObjectStorageState (video: MVideo) {
|
||||||
return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
|
return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function moveToFailedMoveToFileSystemState (video: MVideo) {
|
||||||
|
if (video.state === VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED) return
|
||||||
|
|
||||||
|
return video.setNewState(VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED, false, undefined)
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export {
|
export {
|
||||||
buildNextVideoState,
|
buildNextVideoState,
|
||||||
|
moveToFailedMoveToFileSystemState,
|
||||||
moveToExternalStorageState,
|
moveToExternalStorageState,
|
||||||
|
moveToFileSystemState,
|
||||||
moveToFailedTranscodingState,
|
moveToFailedTranscodingState,
|
||||||
moveToFailedMoveToObjectStorageState,
|
moveToFailedMoveToObjectStorageState,
|
||||||
moveToNextState
|
moveToNextState
|
||||||
|
|
|
@ -21,7 +21,7 @@ import { CreateJobArgument, JobQueue } from './job-queue/job-queue.js'
|
||||||
import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
|
import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
|
||||||
import { moveFilesIfPrivacyChanged } from './video-privacy.js'
|
import { moveFilesIfPrivacyChanged } from './video-privacy.js'
|
||||||
|
|
||||||
function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
|
export function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
|
||||||
return {
|
return {
|
||||||
name: videoInfo.name,
|
name: videoInfo.name,
|
||||||
remote: false,
|
remote: false,
|
||||||
|
@ -42,7 +42,7 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function buildVideoThumbnailsFromReq (options: {
|
export async function buildVideoThumbnailsFromReq (options: {
|
||||||
video: MVideoThumbnail
|
video: MVideoThumbnail
|
||||||
files: UploadFiles
|
files: UploadFiles
|
||||||
fallback: (type: ThumbnailType_Type) => Promise<MThumbnail>
|
fallback: (type: ThumbnailType_Type) => Promise<MThumbnail>
|
||||||
|
@ -79,7 +79,7 @@ async function buildVideoThumbnailsFromReq (options: {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function setVideoTags (options: {
|
export async function setVideoTags (options: {
|
||||||
video: MVideoTag
|
video: MVideoTag
|
||||||
tags: string[]
|
tags: string[]
|
||||||
transaction?: Transaction
|
transaction?: Transaction
|
||||||
|
@ -95,17 +95,18 @@ async function setVideoTags (options: {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function buildMoveToObjectStorageJob (options: {
|
export async function buildMoveJob (options: {
|
||||||
video: MVideoUUID
|
video: MVideoUUID
|
||||||
previousVideoState: VideoStateType
|
previousVideoState: VideoStateType
|
||||||
|
type: 'move-to-object-storage' | 'move-to-file-system'
|
||||||
isNewVideo?: boolean // Default true
|
isNewVideo?: boolean // Default true
|
||||||
}) {
|
}) {
|
||||||
const { video, previousVideoState, isNewVideo = true } = options
|
const { video, previousVideoState, isNewVideo = true, type } = options
|
||||||
|
|
||||||
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
|
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
|
||||||
|
|
||||||
return {
|
return {
|
||||||
type: 'move-to-object-storage' as 'move-to-object-storage',
|
type,
|
||||||
payload: {
|
payload: {
|
||||||
videoUUID: video.uuid,
|
videoUUID: video.uuid,
|
||||||
isNewVideo,
|
isNewVideo,
|
||||||
|
@ -116,7 +117,7 @@ async function buildMoveToObjectStorageJob (options: {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function getVideoDuration (videoId: number | string) {
|
export async function getVideoDuration (videoId: number | string) {
|
||||||
const video = await VideoModel.load(videoId)
|
const video = await VideoModel.load(videoId)
|
||||||
|
|
||||||
const duration = video.isLive
|
const duration = video.isLive
|
||||||
|
@ -126,7 +127,7 @@ async function getVideoDuration (videoId: number | string) {
|
||||||
return { duration, isLive: video.isLive }
|
return { duration, isLive: video.isLive }
|
||||||
}
|
}
|
||||||
|
|
||||||
const getCachedVideoDuration = memoizee(getVideoDuration, {
|
export const getCachedVideoDuration = memoizee(getVideoDuration, {
|
||||||
promise: true,
|
promise: true,
|
||||||
max: MEMOIZE_LENGTH.VIDEO_DURATION,
|
max: MEMOIZE_LENGTH.VIDEO_DURATION,
|
||||||
maxAge: MEMOIZE_TTL.VIDEO_DURATION
|
maxAge: MEMOIZE_TTL.VIDEO_DURATION
|
||||||
|
@ -134,7 +135,7 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function addVideoJobsAfterUpdate (options: {
|
export async function addVideoJobsAfterUpdate (options: {
|
||||||
video: MVideoFullLight
|
video: MVideoFullLight
|
||||||
isNewVideo: boolean
|
isNewVideo: boolean
|
||||||
|
|
||||||
|
@ -188,14 +189,3 @@ async function addVideoJobsAfterUpdate (options: {
|
||||||
|
|
||||||
return JobQueue.Instance.createSequentialJobFlow(...jobs)
|
return JobQueue.Instance.createSequentialJobFlow(...jobs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
export {
|
|
||||||
buildLocalVideoFromReq,
|
|
||||||
buildVideoThumbnailsFromReq,
|
|
||||||
setVideoTags,
|
|
||||||
buildMoveToObjectStorageJob,
|
|
||||||
addVideoJobsAfterUpdate,
|
|
||||||
getCachedVideoDuration
|
|
||||||
}
|
|
||||||
|
|
|
@ -89,6 +89,7 @@ export function checkVideoFileCanBeEdited (video: MVideo, res: express.Response)
|
||||||
const validStates = new Set<VideoStateType>([
|
const validStates = new Set<VideoStateType>([
|
||||||
VideoState.PUBLISHED,
|
VideoState.PUBLISHED,
|
||||||
VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED,
|
VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED,
|
||||||
|
VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED,
|
||||||
VideoState.TRANSCODING_FAILED
|
VideoState.TRANSCODING_FAILED
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
|
@ -3,21 +3,23 @@ import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
|
||||||
import { CONFIG } from '@server/initializers/config.js'
|
import { CONFIG } from '@server/initializers/config.js'
|
||||||
import { initDatabaseModels } from '@server/initializers/database.js'
|
import { initDatabaseModels } from '@server/initializers/database.js'
|
||||||
import { JobQueue } from '@server/lib/job-queue/index.js'
|
import { JobQueue } from '@server/lib/job-queue/index.js'
|
||||||
import { moveToExternalStorageState } from '@server/lib/video-state.js'
|
import { moveToExternalStorageState, moveToFileSystemState } from '@server/lib/video-state.js'
|
||||||
import { VideoModel } from '@server/models/video/video.js'
|
import { VideoModel } from '@server/models/video/video.js'
|
||||||
import { VideoState, VideoStorage } from '@peertube/peertube-models'
|
import { VideoState, VideoStorage } from '@peertube/peertube-models'
|
||||||
|
import { MStreamingPlaylist, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
|
||||||
|
|
||||||
program
|
program
|
||||||
.description('Move videos to another storage.')
|
.description('Move videos to another storage.')
|
||||||
.option('-o, --to-object-storage', 'Move videos in object storage')
|
.option('-o, --to-object-storage', 'Move videos in object storage')
|
||||||
|
.option('-f, --to-file-system', 'Move videos to file system')
|
||||||
.option('-v, --video [videoUUID]', 'Move a specific video')
|
.option('-v, --video [videoUUID]', 'Move a specific video')
|
||||||
.option('-a, --all-videos', 'Migrate all videos')
|
.option('-a, --all-videos', 'Migrate all videos')
|
||||||
.parse(process.argv)
|
.parse(process.argv)
|
||||||
|
|
||||||
const options = program.opts()
|
const options = program.opts()
|
||||||
|
|
||||||
if (!options['toObjectStorage']) {
|
if (!options['toObjectStorage'] && !options['toFileSystem']) {
|
||||||
console.error('You need to choose where to send video files.')
|
console.error('You need to choose where to send video files using --to-object-storage or --to-file-system.')
|
||||||
process.exit(-1)
|
process.exit(-1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -63,8 +65,8 @@ async function run () {
|
||||||
process.exit(-1)
|
process.exit(-1)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE || video.state === VideoState.TO_MOVE_TO_FILE_SYSTEM) {
|
||||||
console.error('This video is already being moved to external storage')
|
console.error('This video is already being moved to external storage/file system')
|
||||||
process.exit(-1)
|
process.exit(-1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,25 +77,58 @@ async function run () {
|
||||||
|
|
||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
const videoFull = await VideoModel.loadFull(id)
|
const videoFull = await VideoModel.loadFull(id)
|
||||||
|
|
||||||
if (videoFull.isLive) continue
|
if (videoFull.isLive) continue
|
||||||
|
|
||||||
const files = videoFull.VideoFiles || []
|
if (options['toObjectStorage']) {
|
||||||
const hls = videoFull.getHLSPlaylist()
|
await createMoveJobIfNeeded({
|
||||||
|
video: videoFull,
|
||||||
|
type: 'to object storage',
|
||||||
|
canProcessVideo: (files, hls) => {
|
||||||
|
return files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM
|
||||||
|
},
|
||||||
|
handler: () => moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined })
|
||||||
|
})
|
||||||
|
|
||||||
if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) {
|
continue
|
||||||
console.log('Processing video %s.', videoFull.name)
|
}
|
||||||
|
|
||||||
const success = await moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined })
|
if (options['toFileSystem']) {
|
||||||
|
await createMoveJobIfNeeded({
|
||||||
|
video: videoFull,
|
||||||
|
type: 'to file system',
|
||||||
|
|
||||||
|
canProcessVideo: (files, hls) => {
|
||||||
|
return files.some(f => f.storage === VideoStorage.OBJECT_STORAGE) || hls?.storage === VideoStorage.OBJECT_STORAGE
|
||||||
|
},
|
||||||
|
handler: () => moveToFileSystemState({ video: videoFull, isNewVideo: false, transaction: undefined })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createMoveJobIfNeeded (options: {
|
||||||
|
video: MVideoFullLight
|
||||||
|
type: 'to object storage' | 'to file system'
|
||||||
|
|
||||||
|
canProcessVideo: (files: MVideoFile[], hls: MStreamingPlaylist) => boolean
|
||||||
|
handler: () => Promise<any>
|
||||||
|
}) {
|
||||||
|
const { video, type, canProcessVideo, handler } = options
|
||||||
|
|
||||||
|
const files = video.VideoFiles || []
|
||||||
|
const hls = video.getHLSPlaylist()
|
||||||
|
|
||||||
|
if (canProcessVideo(files, hls)) {
|
||||||
|
console.log(`Moving ${type} video ${video.name}`)
|
||||||
|
|
||||||
|
const success = await handler()
|
||||||
|
|
||||||
if (!success) {
|
if (!success) {
|
||||||
console.error(
|
console.error(
|
||||||
'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video',
|
`Cannot create move ${type} for ${video.name}: job creation may have failed or there may be pending transcoding jobs for this video`
|
||||||
videoFull.name
|
|
||||||
)
|
)
|
||||||
|
} else {
|
||||||
|
console.log(`Created job ${type} for ${video.name}.`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Created move-to-object-storage job for ${videoFull.name}.`)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,5 +18,8 @@
|
||||||
],
|
],
|
||||||
"include": [
|
"include": [
|
||||||
"./**/*.ts"
|
"./**/*.ts"
|
||||||
|
],
|
||||||
|
"exclude": [
|
||||||
|
"./dist/**/*.ts"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Reference in a new issue