Add script to move videos to file system

client/e2e/src/po/login.po.ts

@@ -36,7 +36,7 @@ export class LoginPage {
     }
 
     if (this.isMobileDevice) {
-      const menuToggle = $('.top-left-block span[role=button]')
+      const menuToggle = $('.top-left-block button')
 
       await $('h2=Our content selection').waitForDisplayed()
 

client/e2e/src/po/player.po.ts

@@ -40,7 +40,7 @@ export class PlayerPage {
 
     await browser.waitUntil(async () => {
       return (await this.getWatchVideoPlayerCurrentTime()) >= waitUntilSec
-    }, { timeout: waitUntilSec * 2 * 1000 })
+    }, { timeout: Math.max(waitUntilSec * 2 * 1000, 30000) })
 
     // Pause video
     await $('div.video-js').click()

client/src/app/+admin/system/jobs/jobs.component.ts

@@ -37,6 +37,7 @@ export class JobsComponent extends RestTable implements OnInit {
     'federate-video',
     'manage-video-torrent',
     'move-to-object-storage',
+    'move-to-file-system',
     'notify',
     'video-channel-import',
     'video-file-import',

client/src/app/+videos/+video-watch/shared/information/video-alert.component.html

@@ -1,27 +1,3 @@
-<div i18n class="alert alert-warning" *ngIf="isVideoTranscodingFailed()">
-  Transcoding failed, this video may not work properly.
-</div>
-
-<div i18n class="alert alert-warning" *ngIf="isVideoMoveToObjectStorageFailed()">
-  Move to external storage failed, this video may not work properly.
-</div>
-
-<div i18n class="alert alert-warning" *ngIf="isVideoToImport()">
-  The video is being imported, it will be available when the import is finished.
-</div>
-
-<div i18n class="alert alert-warning" *ngIf="isVideoToTranscode()">
-  The video is being transcoded, it may not work properly.
-</div>
-
-<div i18n class="alert alert-warning" *ngIf="isVideoToEdit()">
-  The video is being edited, it may not work properly.
-</div>
-
-<div i18n class="alert alert-warning" *ngIf="isVideoToMoveToExternalStorage()">
-  The video is being moved to an external server, it may not work properly.
-</div>
-
 <div i18n class="alert pt-alert-primary" *ngIf="hasVideoScheduledPublication()">
   This video will be published on {{ video.scheduledUpdate.updateAt | date: 'full' }}.
 </div>
@@ -34,6 +10,10 @@
   This live has ended.
 </div>
 
+<div class="alert alert-warning" *ngIf="getAlertWarning()">
+  {{ getAlertWarning() }}
+</div>
+
 <div i18n class="alert alert-warning" *ngIf="noPlaylistVideoFound">
   There are no videos available in this playlist.
 </div>

client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts

@@ -13,28 +13,34 @@ export class VideoAlertComponent {
   @Input() video: VideoDetails
   @Input() noPlaylistVideoFound: boolean
 
-  isVideoToTranscode () {
+  getAlertWarning () {
-    return this.video && this.video.state.id === VideoState.TO_TRANSCODE
+    if (!this.video) return
-  }
 
-  isVideoToEdit () {
+    switch (this.video.state.id) {
-    return this.video && this.video.state.id === VideoState.TO_EDIT
+      case VideoState.TO_TRANSCODE:
-  }
+        return $localize`The video is being transcoded, it may not work properly.`
 
-  isVideoTranscodingFailed () {
+      case VideoState.TO_IMPORT:
-    return this.video && this.video.state.id === VideoState.TRANSCODING_FAILED
+        return $localize`The video is being imported, it will be available when the import is finished.`
-  }
 
-  isVideoMoveToObjectStorageFailed () {
+      case VideoState.TO_MOVE_TO_FILE_SYSTEM:
-    return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED
+        return $localize`The video is being moved to server file system, it may not work properly`
-  }
 
-  isVideoToImport () {
+      case VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED:
-    return this.video && this.video.state.id === VideoState.TO_IMPORT
+        return $localize`Move to file system failed, this video may not work properly.`
-  }
 
-  isVideoToMoveToExternalStorage () {
+      case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE:
-    return this.video && this.video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
+        return $localize`The video is being moved to an external server, it may not work properly.`
+
+      case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED:
+        return $localize`Move to external storage failed, this video may not work properly.`
+
+      case VideoState.TO_EDIT:
+        return $localize`The video is being edited, it may not work properly.`
+
+      case VideoState.TRANSCODING_FAILED:
+        return $localize`Transcoding failed, this video may not work properly.`
+    }
   }
 
   hasVideoScheduledPublication () {

client/src/app/shared/shared-video-miniature/video-miniature.component.ts

@@ -187,28 +187,32 @@ export class VideoMiniatureComponent implements OnInit {
       return $localize`Publication scheduled on ${updateAt}`
     }
 
-    if (video.state.id === VideoState.TRANSCODING_FAILED) {
+    switch (video.state.id) {
-      return $localize`Transcoding failed`
+      case VideoState.TRANSCODING_FAILED:
-    }
+        return $localize`Transcoding failed`
 
-    if (video.state.id === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) {
+      case VideoState.TO_MOVE_TO_FILE_SYSTEM:
-      return $localize`Move to external storage failed`
+        return $localize`Moving to file system`
-    }
 
-    if (video.state.id === VideoState.TO_TRANSCODE && video.waitTranscoding === true) {
+      case VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED:
-      return $localize`Waiting transcoding`
+        return $localize`Moving to file system failed`
-    }
 
-    if (video.state.id === VideoState.TO_TRANSCODE) {
+      case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE:
-      return $localize`To transcode`
+        return $localize`Moving to external storage`
-    }
 
-    if (video.state.id === VideoState.TO_IMPORT) {
+      case VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED:
-      return $localize`To import`
+        return $localize`Move to external storage failed`
-    }
 
-    if (video.state.id === VideoState.TO_EDIT) {
+      case VideoState.TO_TRANSCODE:
-      return $localize`To edit`
+        return video.waitTranscoding === true
+          ? $localize`Waiting transcoding`
+          : $localize`To transcode`
+
+      case VideoState.TO_IMPORT:
+        return $localize`To import`
+
+      case VideoState.TO_EDIT:
+        return $localize`To edit`
     }
 
     return ''

packages/models/src/server/job.model.ts

@@ -20,6 +20,7 @@ export type JobType =
   | 'transcoding-job-builder'
   | 'manage-video-torrent'
   | 'move-to-object-storage'
+  | 'move-to-file-system'
   | 'notify'
   | 'video-channel-import'
   | 'video-file-import'
@@ -196,7 +197,7 @@ export interface DeleteResumableUploadMetaFilePayload {
   filepath: string
 }
 
-export interface MoveObjectStoragePayload {
+export interface MoveStoragePayload {
   videoUUID: string
   isNewVideo: boolean
   previousVideoState: VideoStateType

packages/models/src/videos/video-state.enum.ts

@@ -7,7 +7,9 @@ export const VideoState = {
   TO_MOVE_TO_EXTERNAL_STORAGE: 6,
   TRANSCODING_FAILED: 7,
   TO_MOVE_TO_EXTERNAL_STORAGE_FAILED: 8,
-  TO_EDIT: 9
+  TO_EDIT: 9,
+  TO_MOVE_TO_FILE_SYSTEM: 10,
+  TO_MOVE_TO_FILE_SYSTEM_FAILED: 11
 } as const
 
 export type VideoStateType = typeof VideoState[keyof typeof VideoState]

packages/tests/src/cli/create-move-video-storage-job.ts

@@ -15,6 +15,7 @@ import {
 } from '@peertube/peertube-server-commands'
 import { expectStartWith } from '../shared/checks.js'
 import { checkDirectoryIsEmpty } from '@tests/shared/directories.js'
+import { getAllFiles } from '@peertube/peertube-core-utils'
 
 async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
   for (const file of video.files) {
@@ -73,48 +74,106 @@ describe('Test create move video storage job', function () {
     await servers[0].run(objectStorage.getDefaultMockConfig())
   })
 
-  it('Should move only one file', async function () {
+  describe('To object storage', function () {
-    this.timeout(120000)
 
-    const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
+    it('Should move only one file', async function () {
-    await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
+      this.timeout(120000)
-    await waitJobs(servers)
 
-    for (const server of servers) {
+      const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
-      const video = await server.videos.get({ id: uuids[1] })
+      await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
+      await waitJobs(servers)
 
-      await checkFiles(servers[0], video, objectStorage)
+      for (const server of servers) {
-
+        const video = await server.videos.get({ id: uuids[1] })
-      for (const id of [ uuids[0], uuids[2] ]) {
-        const video = await server.videos.get({ id })
-
-        await checkFiles(servers[0], video)
-      }
-    }
-  })
-
-  it('Should move all files', async function () {
-    this.timeout(120000)
-
-    const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
-    await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
-    await waitJobs(servers)
-
-    for (const server of servers) {
-      for (const id of [ uuids[0], uuids[2] ]) {
-        const video = await server.videos.get({ id })
 
         await checkFiles(servers[0], video, objectStorage)
+
+        for (const id of [ uuids[0], uuids[2] ]) {
+          const video = await server.videos.get({ id })
+
+          await checkFiles(servers[0], video)
+        }
       }
-    }
+    })
+
+    it('Should move all files', async function () {
+      this.timeout(120000)
+
+      const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
+      await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
+      await waitJobs(servers)
+
+      for (const server of servers) {
+        for (const id of [ uuids[0], uuids[2] ]) {
+          const video = await server.videos.get({ id })
+
+          await checkFiles(servers[0], video, objectStorage)
+        }
+      }
+    })
+
+    it('Should not have files on disk anymore', async function () {
+      await checkDirectoryIsEmpty(servers[0], 'web-videos', [ 'private' ])
+      await checkDirectoryIsEmpty(servers[0], join('web-videos', 'private'))
+
+      await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls'), [ 'private' ])
+      await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls', 'private'))
+    })
   })
 
-  it('Should not have files on disk anymore', async function () {
+  describe('To file system', function () {
-    await checkDirectoryIsEmpty(servers[0], 'web-videos', [ 'private' ])
+    let oldFileUrls: string[]
-    await checkDirectoryIsEmpty(servers[0], join('web-videos', 'private'))
 
-    await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls'), [ 'private' ])
+    before(async function () {
-    await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls', 'private'))
+      const video = await servers[0].videos.get({ id: uuids[1] })
+
+      oldFileUrls = [
+        ...getAllFiles(video).map(f => f.fileUrl),
+        video.streamingPlaylists[0].playlistUrl
+      ]
+    })
+
+    it('Should move only one file', async function () {
+      this.timeout(120000)
+
+      const command = `npm run create-move-video-storage-job -- --to-file-system -v ${uuids[1]}`
+      await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
+      await waitJobs(servers)
+
+      for (const server of servers) {
+        const video = await server.videos.get({ id: uuids[1] })
+
+        await checkFiles(servers[0], video)
+
+        for (const id of [ uuids[0], uuids[2] ]) {
+          const video = await server.videos.get({ id })
+
+          await checkFiles(servers[0], video, objectStorage)
+        }
+      }
+    })
+
+    it('Should move all files', async function () {
+      this.timeout(120000)
+
+      const command = `npm run create-move-video-storage-job -- --to-file-system --all-videos`
+      await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
+      await waitJobs(servers)
+
+      for (const server of servers) {
+        for (const id of [ uuids[0], uuids[2] ]) {
+          const video = await server.videos.get({ id })
+
+          await checkFiles(servers[0], video)
+        }
+      }
+    })
+
+    it('Should not have files on disk anymore', async function () {
+      for (const fileUrl of oldFileUrls) {
+        await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+      }
+    })
   })
 
   after(async function () {

server/core/controllers/api/videos/source.ts

@@ -5,7 +5,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-q
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
 import { uploadx } from '@server/lib/uploadx.js'
-import { buildMoveToObjectStorageJob } from '@server/lib/video.js'
+import { buildMoveJob } from '@server/lib/video.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { buildNewFile } from '@server/lib/video-file.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
@@ -171,7 +171,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
   ]
 
   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
-    jobs.push(await buildMoveToObjectStorageJob({ video, isNewVideo: false, previousVideoState: undefined }))
+    jobs.push(await buildMoveJob({ video, isNewVideo: false, previousVideoState: undefined, type: 'move-to-object-storage' }))
   }
 
   if (video.state === VideoState.TO_TRANSCODE) {

server/core/controllers/api/videos/upload.ts

@@ -6,7 +6,7 @@ import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
 import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
 import { Redis } from '@server/lib/redis.js'
 import { uploadx } from '@server/lib/uploadx.js'
-import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
+import { buildLocalVideoFromReq, buildMoveJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
 import { buildNewFile } from '@server/lib/video-file.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
@@ -275,7 +275,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
   ]
 
   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
-    jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }))
+    jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' }))
   }
 
   if (video.state === VideoState.TO_TRANSCODE) {

server/core/helpers/logger.ts

@@ -121,7 +121,8 @@ const bunyanLogger = {
 
 // ---------------------------------------------------------------------------
 
-type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
+type LoggerTags = { tags: string[] }
+type LoggerTagsFn = (...tags: string[]) => LoggerTags
 function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
   return (...tags: string[]) => {
     return { tags: defaultTags.concat(tags) }
@@ -154,6 +155,7 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
 
 export {
   type LoggerTagsFn,
+  type LoggerTags,
 
   buildLogger,
   timestampFormatter,

server/core/initializers/constants.ts

@@ -186,6 +186,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'video-channel-import': 1,
   'after-video-channel-import': 1,
   'move-to-object-storage': 3,
+  'move-to-file-system': 3,
   'transcoding-job-builder': 1,
   'generate-video-storyboard': 1,
   'notify': 1,
@@ -209,6 +210,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
   'video-studio-edition': 1,
   'manage-video-torrent': 1,
   'move-to-object-storage': 1,
+  'move-to-file-system': 1,
   'video-channel-import': 1,
   'after-video-channel-import': 1,
   'transcoding-job-builder': 1,
@@ -236,6 +238,7 @@ const JOB_TTL: { [id in JobType]: number } = {
   'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes
   'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
   'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
+  'move-to-file-system': 1000 * 60 * 60 * 3, // 3 hours
   'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
   'after-video-channel-import': 60000 * 5, // 5 minutes
   'transcoding-job-builder': 60000, // 1 minute
@@ -557,7 +560,9 @@ const VIDEO_STATES: { [ id in VideoStateType ]: string } = {
   [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
   [VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
   [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed',
-  [VideoState.TO_EDIT]: 'To edit*'
+  [VideoState.TO_EDIT]: 'To edit',
+  [VideoState.TO_MOVE_TO_FILE_SYSTEM]: 'To move to file system',
+  [VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED]: 'Move to file system failed'
 }
 
 const VIDEO_IMPORT_STATES: { [ id in VideoImportStateType ]: string } = {

server/core/lib/job-queue/handlers/move-to-file-system.ts

@@ -0,0 +1,138 @@
+import { Job } from 'bullmq'
+import { join } from 'path'
+import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
+import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
+import {
+  makeHLSFileAvailable,
+  makeWebVideoFileAvailable,
+  removeHLSFileObjectStorageByFilename,
+  removeHLSObjectStorage,
+  removeWebVideoObjectStorage
+} from '@server/lib/object-storage/index.js'
+import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
+import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
+
+const lTagsBase = loggerTagsFactory('move-file-system')
+
+export async function processMoveToFileSystem (job: Job) {
+  const payload = job.data as MoveStoragePayload
+  logger.info('Moving video %s to file system in job %s.', payload.videoUUID, job.id)
+
+  await moveToJob({
+    jobId: job.id,
+    videoUUID: payload.videoUUID,
+    loggerTags: lTagsBase().tags,
+
+    moveWebVideoFiles,
+    moveHLSFiles,
+    doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
+    moveToFailedState: moveToFailedMoveToFileSystemState
+  })
+}
+
+export async function onMoveToFileSystemFailure (job: Job, err: any) {
+  const payload = job.data as MoveStoragePayload
+
+  await onMoveToStorageFailure({
+    videoUUID: payload.videoUUID,
+    err,
+    lTags: lTagsBase(),
+    moveToFailedState: moveToFailedMoveToFileSystemState
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function moveWebVideoFiles (video: MVideoWithAllFiles) {
+  for (const file of video.VideoFiles) {
+    if (file.storage === VideoStorage.FILE_SYSTEM) continue
+
+    await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
+    await onFileMoved({
+      videoOrPlaylist: video,
+      file,
+      objetStorageRemover: () => removeWebVideoObjectStorage(file)
+    })
+  }
+}
+
+async function moveHLSFiles (video: MVideoWithAllFiles) {
+  for (const playlist of video.VideoStreamingPlaylists) {
+    const playlistWithVideo = playlist.withVideo(video)
+
+    for (const file of playlist.VideoFiles) {
+      if (file.storage === VideoStorage.FILE_SYSTEM) continue
+
+      // Resolution playlist
+      const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
+      await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
+      await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
+
+      await onFileMoved({
+        videoOrPlaylist: playlistWithVideo,
+        file,
+        objetStorageRemover: async () => {
+          await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
+          await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
+        }
+      })
+    }
+  }
+}
+
+async function onFileMoved (options: {
+  videoOrPlaylist: MVideo | MStreamingPlaylistVideo
+  file: MVideoFile
+  objetStorageRemover: () => Promise<any>
+}) {
+  const { videoOrPlaylist, file, objetStorageRemover } = options
+
+  const oldFileUrl = file.fileUrl
+
+  file.fileUrl = null
+  file.storage = VideoStorage.FILE_SYSTEM
+
+  await updateTorrentMetadata(videoOrPlaylist, file)
+  await file.save()
+
+  logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
+  await objetStorageRemover()
+}
+
+async function doAfterLastMove (options: {
+  video: MVideoWithAllFiles
+  previousVideoState: VideoStateType
+  isNewVideo: boolean
+}) {
+  const { video, previousVideoState, isNewVideo } = options
+
+  for (const playlist of video.VideoStreamingPlaylists) {
+    if (playlist.storage === VideoStorage.FILE_SYSTEM) continue
+
+    const playlistWithVideo = playlist.withVideo(video)
+
+    for (const filename of [ playlist.playlistFilename, playlist.segmentsSha256Filename ]) {
+      await makeHLSFileAvailable(playlistWithVideo, filename, join(getHLSDirectory(video), filename))
+    }
+
+    playlist.playlistUrl = null
+    playlist.segmentsSha256Url = null
+    playlist.storage = VideoStorage.FILE_SYSTEM
+
+    playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
+    playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
+    await playlist.save()
+
+    await removeHLSObjectStorage(playlistWithVideo)
+  }
+
+  await moveToNextState({ video, previousVideoState, isNewVideo })
+}

server/core/lib/job-queue/handlers/move-to-object-storage.ts

@@ -1,7 +1,7 @@
 import { Job } from 'bullmq'
 import { remove } from 'fs-extra/esm'
 import { join } from 'path'
-import { MoveObjectStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
+import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
 import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
@@ -9,68 +9,36 @@ import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
-import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
-import { VideoModel } from '@server/models/video/video.js'
 import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { moveToJob, onMoveToStorageFailure } from './shared/move-video.js'
 
 const lTagsBase = loggerTagsFactory('move-object-storage')
 
 export async function processMoveToObjectStorage (job: Job) {
-  const payload = job.data as MoveObjectStoragePayload
+  const payload = job.data as MoveStoragePayload
-  logger.info('Moving video %s in job %s.', payload.videoUUID, job.id)
+  logger.info('Moving video %s to object storage in job %s.', payload.videoUUID, job.id)
 
-  const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
+  await moveToJob({
+    jobId: job.id,
+    videoUUID: payload.videoUUID,
+    loggerTags: lTagsBase().tags,
 
-  const video = await VideoModel.loadWithFiles(payload.videoUUID)
+    moveWebVideoFiles,
-  // No video, maybe deleted?
+    moveHLSFiles,
-  if (!video) {
+    doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }),
-    logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID))
+    moveToFailedState: moveToFailedMoveToObjectStorageState
-    fileMutexReleaser()
+  })
-    return undefined
-  }
-
-  const lTags = lTagsBase(video.uuid, video.url)
-
-  try {
-    if (video.VideoFiles) {
-      logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
-
-      await moveWebVideoFiles(video)
-    }
-
-    if (video.VideoStreamingPlaylists) {
-      logger.debug('Moving HLS playlist of %s.', video.uuid)
-
-      await moveHLSFiles(video)
-    }
-
-    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
-    if (pendingMove === 0) {
-      logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags)
-
-      await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
-    }
-  } catch (err) {
-    await onMoveToObjectStorageFailure(job, err)
-
-    throw err
-  } finally {
-    fileMutexReleaser()
-  }
-
-  return payload.videoUUID
 }
 
 export async function onMoveToObjectStorageFailure (job: Job, err: any) {
-  const payload = job.data as MoveObjectStoragePayload
+  const payload = job.data as MoveStoragePayload
 
-  const video = await VideoModel.loadWithFiles(payload.videoUUID)
+  await onMoveToStorageFailure({
-  if (!video) return
+    videoUUID: payload.videoUUID,
-
+    err,
-  logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) })
+    lTags: lTagsBase(),
-
+    moveToFailedState: moveToFailedMoveToObjectStorageState
-  await moveToFailedMoveToObjectStorageState(video)
+  })
-  await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
 }
 
 // ---------------------------------------------------------------------------
@@ -107,39 +75,6 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
   }
 }
 
-async function doAfterLastJob (options: {
-  video: MVideoWithAllFiles
-  previousVideoState: VideoStateType
-  isNewVideo: boolean
-}) {
-  const { video, previousVideoState, isNewVideo } = options
-
-  for (const playlist of video.VideoStreamingPlaylists) {
-    if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
-
-    const playlistWithVideo = playlist.withVideo(video)
-
-    // Master playlist
-    playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
-    // Sha256 segments file
-    playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
-
-    playlist.storage = VideoStorage.OBJECT_STORAGE
-
-    playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
-    playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
-
-    await playlist.save()
-  }
-
-  // Remove empty hls video directory
-  if (video.VideoStreamingPlaylists) {
-    await remove(getHLSDirectory(video))
-  }
-
-  await moveToNextState({ video, previousVideoState, isNewVideo })
-}
-
 async function onFileMoved (options: {
   videoOrPlaylist: MVideo | MStreamingPlaylistVideo
   file: MVideoFile
@@ -154,6 +89,32 @@ async function onFileMoved (options: {
   await updateTorrentMetadata(videoOrPlaylist, file)
   await file.save()
 
-  logger.debug('Removing %s because it\'s now on object storage', oldPath)
+  logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
   await remove(oldPath)
 }
+
+async function doAfterLastMove (options: {
+  video: MVideoWithAllFiles
+  previousVideoState: VideoStateType
+  isNewVideo: boolean
+}) {
+  const { video, previousVideoState, isNewVideo } = options
+
+  for (const playlist of video.VideoStreamingPlaylists) {
+    if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
+
+    const playlistWithVideo = playlist.withVideo(video)
+
+    playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
+    playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
+    playlist.storage = VideoStorage.OBJECT_STORAGE
+
+    playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
+    playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
+    await playlist.save()
+  }
+
+  await remove(getHLSDirectory(video))
+  await moveToNextState({ video, previousVideoState, isNewVideo })
+}

server/core/lib/job-queue/handlers/shared/move-video.ts

@@ -0,0 +1,76 @@
+import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideoWithAllFiles } from '@server/types/models/index.js'
+
+export async function moveToJob (options: {
+  jobId: string
+  videoUUID: string
+  loggerTags: string[]
+
+  moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
+  moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
+  moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
+  doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
+}) {
+  const { jobId, loggerTags, videoUUID, moveHLSFiles, moveWebVideoFiles, moveToFailedState, doAfterLastMove } = options
+
+  const lTagsBase = loggerTagsFactory(...loggerTags)
+
+  const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoUUID)
+
+  const video = await VideoModel.loadWithFiles(videoUUID)
+  // No video, maybe deleted?
+  if (!video) {
+    logger.info('Can\'t process job %d, video does not exist.', jobId, lTagsBase(videoUUID))
+    fileMutexReleaser()
+    return undefined
+  }
+
+  const lTags = lTagsBase(video.uuid, video.url)
+
+  try {
+    if (video.VideoFiles) {
+      logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags)
+
+      await moveWebVideoFiles(video)
+    }
+
+    if (video.VideoStreamingPlaylists) {
+      logger.debug('Moving HLS playlist of %s.', video.uuid)
+
+      await moveHLSFiles(video)
+    }
+
+    const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
+    if (pendingMove === 0) {
+      logger.info('Running cleanup after moving files (video %s in job %s)', video.uuid, jobId, lTags)
+
+      await doAfterLastMove(video)
+    }
+  } catch (err) {
+    await onMoveToStorageFailure({ videoUUID, err, lTags, moveToFailedState })
+
+    throw err
+  } finally {
+    fileMutexReleaser()
+  }
+}
+
+export async function onMoveToStorageFailure (options: {
+  videoUUID: string
+  err: any
+  lTags: LoggerTags
+  moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
+}) {
+  const { videoUUID, err, lTags, moveToFailedState } = options
+
+  const video = await VideoModel.loadWithFiles(videoUUID)
+  if (!video) return
+
+  logger.error('Cannot move video %s storage.', video.url, { err, ...lTags })
+
+  await moveToFailedState(video)
+  await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
+}

server/core/lib/job-queue/handlers/video-file-import.ts

@@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config.js'
 import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
 import { generateWebVideoFilename } from '@server/lib/paths.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { buildMoveToObjectStorageJob } from '@server/lib/video.js'
+import { buildMoveJob } from '@server/lib/video.js'
 import { VideoFileModel } from '@server/models/video/video-file.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { MVideoFullLight } from '@server/types/models/index.js'
@@ -30,7 +30,7 @@ async function processVideoFileImport (job: Job) {
   await updateVideoFile(video, payload.filePath)
 
   if (CONFIG.OBJECT_STORAGE.ENABLED) {
-    await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState: video.state }))
+    await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState: video.state, type: 'move-to-object-storage' }))
   } else {
     await federateVideoIfNeeded(video, false)
   }

server/core/lib/job-queue/handlers/video-import.ts

@@ -25,7 +25,7 @@ import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-t
 import { isAbleToUploadVideo } from '@server/lib/user.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
-import { buildMoveToObjectStorageJob } from '@server/lib/video.js'
+import { buildMoveJob } from '@server/lib/video.js'
 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
 import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
 import { getLowercaseExtension } from '@peertube/peertube-node-utils'
@@ -317,7 +317,7 @@ async function afterImportSuccess (options: {
 
   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
     await JobQueue.Instance.createJob(
-      await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT })
+      await buildMoveJob({ video, previousVideoState: VideoState.TO_IMPORT, type: 'move-to-object-storage' })
     )
     return
   }

server/core/lib/job-queue/job-queue.ts

@@ -25,7 +25,7 @@ import {
   JobState,
   JobType,
   ManageVideoTorrentPayload,
-  MoveObjectStoragePayload,
+  MoveStoragePayload,
   NotifyPayload,
   RefreshPayload,
   TranscodingJobBuilderPayload,
@@ -70,6 +70,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending.js'
 import { processVideoStudioEdition } from './handlers/video-studio-edition.js'
 import { processVideoTranscoding } from './handlers/video-transcoding.js'
 import { processVideosViewsStats } from './handlers/video-views-stats.js'
+import { onMoveToFileSystemFailure, processMoveToFileSystem } from './handlers/move-to-file-system.js'
 
 export type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -91,11 +92,11 @@ export type CreateJobArgument =
   { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
   { type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
   { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
-  { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
+  { type: 'move-to-object-storage', payload: MoveStoragePayload } |
+  { type: 'move-to-file-system', payload: MoveStoragePayload } |
   { type: 'video-channel-import', payload: VideoChannelImportPayload } |
   { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
   { type: 'notify', payload: NotifyPayload } |
-  { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
   { type: 'federate-video', payload: FederateVideoPayload } |
   { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
 
@@ -120,6 +121,7 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
   'transcoding-job-builder': processTranscodingJobBuilder,
   'manage-video-torrent': processManageVideoTorrent,
   'move-to-object-storage': processMoveToObjectStorage,
+  'move-to-file-system': processMoveToFileSystem,
   'notify': processNotify,
   'video-channel-import': processVideoChannelImport,
   'video-file-import': processVideoFileImport,
@@ -133,7 +135,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
 }
 
 const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
-  'move-to-object-storage': onMoveToObjectStorageFailure
+  'move-to-object-storage': onMoveToObjectStorageFailure,
+  'move-to-file-system': onMoveToFileSystemFailure
 }
 
 const jobTypes: JobType[] = [
@@ -151,6 +154,7 @@ const jobTypes: JobType[] = [
   'generate-video-storyboard',
   'manage-video-torrent',
   'move-to-object-storage',
+  'move-to-file-system',
   'notify',
   'transcoding-job-builder',
   'video-channel-import',

server/core/lib/video-state.ts

@@ -10,7 +10,7 @@ import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models/index.
 import { federateVideoIfNeeded } from './activitypub/videos/index.js'
 import { JobQueue } from './job-queue/index.js'
 import { Notifier } from './notifier/index.js'
-import { buildMoveToObjectStorageJob } from './video.js'
+import { buildMoveJob } from './video.js'
 
 function buildNextVideoState (currentState?: VideoStateType) {
   if (currentState === VideoState.PUBLISHED) {
@@ -21,6 +21,7 @@ function buildNextVideoState (currentState?: VideoStateType) {
     currentState !== VideoState.TO_EDIT &&
     currentState !== VideoState.TO_TRANSCODE &&
     currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
+    currentState !== VideoState.TO_MOVE_TO_FILE_SYSTEM &&
     CONFIG.TRANSCODING.ENABLED
   ) {
     return VideoState.TO_TRANSCODE
@@ -28,6 +29,7 @@ function buildNextVideoState (currentState?: VideoStateType) {
 
   if (
     currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE &&
+    currentState !== VideoState.TO_MOVE_TO_FILE_SYSTEM &&
     CONFIG.OBJECT_STORAGE.ENABLED
   ) {
     return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE
@@ -68,6 +70,8 @@ function moveToNextState (options: {
   })
 }
 
+// ---------------------------------------------------------------------------
+
 async function moveToExternalStorageState (options: {
   video: MVideoFullLight
   isNewVideo: boolean
@@ -90,7 +94,7 @@ async function moveToExternalStorageState (options: {
   logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
 
   try {
-    await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState, isNewVideo }))
+    await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState, isNewVideo, type: 'move-to-object-storage' }))
 
     return true
   } catch (err) {
@@ -100,6 +104,34 @@ async function moveToExternalStorageState (options: {
   }
 }
 
+async function moveToFileSystemState (options: {
+  video: MVideoFullLight
+  isNewVideo: boolean
+  transaction: Transaction
+}) {
+  const { video, isNewVideo, transaction } = options
+
+  const previousVideoState = video.state
+
+  if (video.state !== VideoState.TO_MOVE_TO_FILE_SYSTEM) {
+    await video.setNewState(VideoState.TO_MOVE_TO_FILE_SYSTEM, false, transaction)
+  }
+
+  logger.info('Creating move to file system job for video %s.', video.uuid, { tags: [ video.uuid ] })
+
+  try {
+    await JobQueue.Instance.createJob(await buildMoveJob({ video, previousVideoState, isNewVideo, type: 'move-to-file-system' }))
+
+    return true
+  } catch (err) {
+    logger.error('Cannot add move to file system job', { err })
+
+    return false
+  }
+}
+
+// ---------------------------------------------------------------------------
+
 function moveToFailedTranscodingState (video: MVideo) {
   if (video.state === VideoState.TRANSCODING_FAILED) return
 
@@ -112,11 +144,19 @@ function moveToFailedMoveToObjectStorageState (video: MVideo) {
   return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
 }
 
+function moveToFailedMoveToFileSystemState (video: MVideo) {
+  if (video.state === VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED) return
+
+  return video.setNewState(VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED, false, undefined)
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   buildNextVideoState,
+  moveToFailedMoveToFileSystemState,
   moveToExternalStorageState,
+  moveToFileSystemState,
   moveToFailedTranscodingState,
   moveToFailedMoveToObjectStorageState,
   moveToNextState

server/core/lib/video.ts

@@ -21,7 +21,7 @@ import { CreateJobArgument, JobQueue } from './job-queue/job-queue.js'
 import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js'
 import { moveFilesIfPrivacyChanged } from './video-privacy.js'
 
-function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
+export function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
   return {
     name: videoInfo.name,
     remote: false,
@@ -42,7 +42,7 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil
   }
 }
 
-async function buildVideoThumbnailsFromReq (options: {
+export async function buildVideoThumbnailsFromReq (options: {
   video: MVideoThumbnail
   files: UploadFiles
   fallback: (type: ThumbnailType_Type) => Promise<MThumbnail>
@@ -79,7 +79,7 @@ async function buildVideoThumbnailsFromReq (options: {
 
 // ---------------------------------------------------------------------------
 
-async function setVideoTags (options: {
+export async function setVideoTags (options: {
   video: MVideoTag
   tags: string[]
   transaction?: Transaction
@@ -95,17 +95,18 @@ async function setVideoTags (options: {
 
 // ---------------------------------------------------------------------------
 
-async function buildMoveToObjectStorageJob (options: {
+export async function buildMoveJob (options: {
   video: MVideoUUID
   previousVideoState: VideoStateType
+  type: 'move-to-object-storage' | 'move-to-file-system'
   isNewVideo?: boolean // Default true
 }) {
-  const { video, previousVideoState, isNewVideo = true } = options
+  const { video, previousVideoState, isNewVideo = true, type } = options
 
   await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
 
   return {
-    type: 'move-to-object-storage' as 'move-to-object-storage',
+    type,
     payload: {
       videoUUID: video.uuid,
       isNewVideo,
@@ -116,7 +117,7 @@ async function buildMoveToObjectStorageJob (options: {
 
 // ---------------------------------------------------------------------------
 
-async function getVideoDuration (videoId: number | string) {
+export async function getVideoDuration (videoId: number | string) {
   const video = await VideoModel.load(videoId)
 
   const duration = video.isLive
@@ -126,7 +127,7 @@ async function getVideoDuration (videoId: number | string) {
   return { duration, isLive: video.isLive }
 }
 
-const getCachedVideoDuration = memoizee(getVideoDuration, {
+export const getCachedVideoDuration = memoizee(getVideoDuration, {
   promise: true,
   max: MEMOIZE_LENGTH.VIDEO_DURATION,
   maxAge: MEMOIZE_TTL.VIDEO_DURATION
@@ -134,7 +135,7 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
 
 // ---------------------------------------------------------------------------
 
-async function addVideoJobsAfterUpdate (options: {
+export async function addVideoJobsAfterUpdate (options: {
   video: MVideoFullLight
   isNewVideo: boolean
 
@@ -188,14 +189,3 @@ async function addVideoJobsAfterUpdate (options: {
 
   return JobQueue.Instance.createSequentialJobFlow(...jobs)
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  buildLocalVideoFromReq,
-  buildVideoThumbnailsFromReq,
-  setVideoTags,
-  buildMoveToObjectStorageJob,
-  addVideoJobsAfterUpdate,
-  getCachedVideoDuration
-}

server/core/middlewares/validators/videos/shared/video-validators.ts

@@ -89,6 +89,7 @@ export function checkVideoFileCanBeEdited (video: MVideo, res: express.Response)
   const validStates = new Set<VideoStateType>([
     VideoState.PUBLISHED,
     VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED,
+    VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED,
     VideoState.TRANSCODING_FAILED
   ])
 

server/scripts/create-move-video-storage-job.ts

@@ -3,21 +3,23 @@ import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
 import { CONFIG } from '@server/initializers/config.js'
 import { initDatabaseModels } from '@server/initializers/database.js'
 import { JobQueue } from '@server/lib/job-queue/index.js'
-import { moveToExternalStorageState } from '@server/lib/video-state.js'
+import { moveToExternalStorageState, moveToFileSystemState } from '@server/lib/video-state.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { VideoState, VideoStorage } from '@peertube/peertube-models'
+import { MStreamingPlaylist, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
 
 program
   .description('Move videos to another storage.')
   .option('-o, --to-object-storage', 'Move videos in object storage')
+  .option('-f, --to-file-system', 'Move videos to file system')
   .option('-v, --video [videoUUID]', 'Move a specific video')
   .option('-a, --all-videos', 'Migrate all videos')
   .parse(process.argv)
 
 const options = program.opts()
 
-if (!options['toObjectStorage']) {
+if (!options['toObjectStorage'] && !options['toFileSystem']) {
-  console.error('You need to choose where to send video files.')
+  console.error('You need to choose where to send video files using --to-object-storage or --to-file-system.')
   process.exit(-1)
 }
 
@@ -63,8 +65,8 @@ async function run () {
       process.exit(-1)
     }
 
-    if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
+    if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE || video.state === VideoState.TO_MOVE_TO_FILE_SYSTEM) {
-      console.error('This video is already being moved to external storage')
+      console.error('This video is already being moved to external storage/file system')
       process.exit(-1)
     }
 
@@ -75,25 +77,58 @@ async function run () {
 
   for (const id of ids) {
     const videoFull = await VideoModel.loadFull(id)
-
     if (videoFull.isLive) continue
 
-    const files = videoFull.VideoFiles || []
+    if (options['toObjectStorage']) {
-    const hls = videoFull.getHLSPlaylist()
+      await createMoveJobIfNeeded({
+        video: videoFull,
+        type: 'to object storage',
+        canProcessVideo: (files, hls) => {
+          return files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM
+        },
+        handler: () => moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined })
+      })
 
-    if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) {
+      continue
-      console.log('Processing video %s.', videoFull.name)
-
-      const success = await moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined })
-
-      if (!success) {
-        console.error(
-          'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video',
-          videoFull.name
-        )
-      }
     }
 
-    console.log(`Created move-to-object-storage job for ${videoFull.name}.`)
+    if (options['toFileSystem']) {
+      await createMoveJobIfNeeded({
+        video: videoFull,
+        type: 'to file system',
+
+        canProcessVideo: (files, hls) => {
+          return files.some(f => f.storage === VideoStorage.OBJECT_STORAGE) || hls?.storage === VideoStorage.OBJECT_STORAGE
+        },
+        handler: () => moveToFileSystemState({ video: videoFull, isNewVideo: false, transaction: undefined })
+      })
+    }
+  }
+}
+
+async function createMoveJobIfNeeded (options: {
+  video: MVideoFullLight
+  type: 'to object storage' | 'to file system'
+
+  canProcessVideo: (files: MVideoFile[], hls: MStreamingPlaylist) => boolean
+  handler: () => Promise<any>
+}) {
+  const { video, type, canProcessVideo, handler } = options
+
+  const files = video.VideoFiles || []
+  const hls = video.getHLSPlaylist()
+
+  if (canProcessVideo(files, hls)) {
+    console.log(`Moving ${type} video ${video.name}`)
+
+    const success = await handler()
+
+    if (!success) {
+      console.error(
+        `Cannot create move ${type} for ${video.name}: job creation may have failed or there may be pending transcoding jobs for this video`
+      )
+    } else {
+      console.log(`Created job ${type} for ${video.name}.`)
+    }
   }
 }

server/tsconfig.json

@@ -18,5 +18,8 @@
   ],
   "include": [
     "./**/*.ts"
+  ],
+  "exclude": [
+    "./dist/**/*.ts"
   ]
 }