diff --git a/client/src/app/+admin/overview/videos/video-list.component.html b/client/src/app/+admin/overview/videos/video-list.component.html index 6e4fb4c6f..738bcedee 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.html +++ b/client/src/app/+admin/overview/videos/video-list.component.html @@ -56,8 +56,8 @@ diff --git a/client/src/app/+admin/overview/videos/video-list.component.ts b/client/src/app/+admin/overview/videos/video-list.component.ts index 3c21adb44..4aed5221b 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.ts +++ b/client/src/app/+admin/overview/videos/video-list.component.ts @@ -40,7 +40,8 @@ export class VideoListComponent extends RestTable implements OnInit { duplicate: true, mute: true, liveInfo: false, - removeFiles: true + removeFiles: true, + transcoding: true } loading = true @@ -89,16 +90,28 @@ export class VideoListComponent extends RestTable implements OnInit { } ], [ + { + label: $localize`Run HLS transcoding`, + handler: videos => this.runTranscoding(videos, 'hls'), + isDisplayed: videos => videos.every(v => v.canRunTranscoding(this.authUser)), + iconName: 'cog' + }, + { + label: $localize`Run WebTorrent transcoding`, + handler: videos => this.runTranscoding(videos, 'webtorrent'), + isDisplayed: videos => videos.every(v => v.canRunTranscoding(this.authUser)), + iconName: 'cog' + }, { label: $localize`Delete HLS files`, handler: videos => this.removeVideoFiles(videos, 'hls'), - isDisplayed: videos => this.authUser.hasRight(UserRight.MANAGE_VIDEO_FILES) && videos.every(v => v.hasHLS() && v.hasWebTorrent()), + isDisplayed: videos => videos.every(v => v.canRemoveFiles(this.authUser)), iconName: 'delete' }, { label: $localize`Delete WebTorrent files`, handler: videos => this.removeVideoFiles(videos, 'webtorrent'), - isDisplayed: videos => this.authUser.hasRight(UserRight.MANAGE_VIDEO_FILES) && videos.every(v => v.hasHLS() && v.hasWebTorrent()), + isDisplayed: videos => videos.every(v => v.canRemoveFiles(this.authUser)), iconName: 'delete' } ] @@ -226,4 +239,17 @@ export class VideoListComponent extends RestTable implements OnInit { error: err => this.notifier.error(err.message) }) } + + private runTranscoding (videos: Video[], type: 'hls' | 'webtorrent') { + this.videoService.runTranscoding(videos.map(v => v.id), type) + .subscribe({ + next: () => { + this.notifier.success($localize`Transcoding jobs created.`) + + this.reloadData() + }, + + error: err => this.notifier.error(err.message) + }) + } } diff --git a/client/src/app/shared/shared-main/video/video.model.ts b/client/src/app/shared/shared-main/video/video.model.ts index 4203ff1c0..eefa90489 100644 --- a/client/src/app/shared/shared-main/video/video.model.ts +++ b/client/src/app/shared/shared-main/video/video.model.ts @@ -220,6 +220,18 @@ export class Video implements VideoServerModel { return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO)) } + canRemoveFiles (user: AuthUser) { + return user.hasRight(UserRight.MANAGE_VIDEO_FILES) && + this.state.id !== VideoState.TO_TRANSCODE && + this.hasHLS() && + this.hasWebTorrent() + } + + canRunTranscoding (user: AuthUser) { + return user.hasRight(UserRight.RUN_VIDEO_TRANSCODING) && + this.state.id !== VideoState.TO_TRANSCODE + } + hasHLS () { return this.streamingPlaylists?.some(p => p.type === VideoStreamingPlaylistType.HLS) } diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts index d135a27dc..9bfa397f8 100644 --- a/client/src/app/shared/shared-main/video/video.service.ts +++ b/client/src/app/shared/shared-main/video/video.service.ts @@ -21,6 +21,7 @@ import { VideoInclude, VideoPrivacy, VideoSortField, + VideoTranscodingCreate, VideoUpdate } from '@shared/models' import { environment } from '../../../../environments/environment' @@ -308,6 +309,17 @@ export class VideoService { ) } + runTranscoding (videoIds: (number | string)[], type: 'hls' | 'webtorrent') { + const body: VideoTranscodingCreate = { transcodingType: type } + + return from(videoIds) + .pipe( + concatMap(id => this.authHttp.post(VideoService.BASE_VIDEO_URL + '/' + id + '/transcoding', body)), + toArray(), + catchError(err => this.restExtractor.handleError(err)) + ) + } + loadCompleteDescription (descriptionPath: string) { return this.authHttp .get<{ description: string }>(environment.apiUrl + descriptionPath) diff --git a/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts b/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts index 82c084791..2ab9f4739 100644 --- a/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts @@ -2,7 +2,7 @@ import { Component, EventEmitter, Input, OnChanges, Output, ViewChild } from '@a import { AuthService, ConfirmService, Notifier, ScreenService } from '@app/core' import { BlocklistService, VideoBlockComponent, VideoBlockService, VideoReportComponent } from '@app/shared/shared-moderation' import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap' -import { UserRight, VideoCaption } from '@shared/models' +import { UserRight, VideoCaption, VideoState } from '@shared/models' import { Actor, DropdownAction, @@ -28,6 +28,7 @@ export type VideoActionsDisplayType = { mute?: boolean liveInfo?: boolean removeFiles?: boolean + transcoding?: boolean } @Component({ @@ -56,7 +57,9 @@ export class VideoActionsDropdownComponent implements OnChanges { report: true, duplicate: true, mute: true, - liveInfo: false + liveInfo: false, + removeFiles: false, + transcoding: false } @Input() placement = 'left' @@ -71,6 +74,7 @@ export class VideoActionsDropdownComponent implements OnChanges { @Output() videoUnblocked = new EventEmitter() @Output() videoBlocked = new EventEmitter() @Output() videoAccountMuted = new EventEmitter() + @Output() transcodingCreated = new EventEmitter() @Output() modalOpened = new EventEmitter() videoActions: DropdownAction<{ video: Video }>[][] = [] @@ -177,7 +181,11 @@ export class VideoActionsDropdownComponent implements OnChanges { } canRemoveVideoFiles () { - return this.user.hasRight(UserRight.MANAGE_VIDEO_FILES) && this.video.hasHLS() && this.video.hasWebTorrent() + return this.video.canRemoveFiles(this.user) + } + + canRunTranscoding () { + return this.video.canRunTranscoding(this.user) } /* Action handlers */ @@ -268,6 +276,18 @@ export class VideoActionsDropdownComponent implements OnChanges { }) } + runTranscoding (video: Video, type: 'hls' | 'webtorrent') { + this.videoService.runTranscoding([ video.id ], type) + .subscribe({ + next: () => { + this.notifier.success($localize`Transcoding jobs created for ${video.name}.`) + this.transcodingCreated.emit() + }, + + error: err => this.notifier.error(err.message) + }) + } + onVideoBlocked () { this.videoBlocked.emit() } @@ -341,6 +361,18 @@ export class VideoActionsDropdownComponent implements OnChanges { } ], [ + { + label: $localize`Run HLS transcoding`, + handler: ({ video }) => this.runTranscoding(video, 'hls'), + isDisplayed: () => this.displayOptions.transcoding && this.canRunTranscoding(), + iconName: 'cog' + }, + { + label: $localize`Run WebTorrent transcoding`, + handler: ({ video }) => this.runTranscoding(video, 'webtorrent'), + isDisplayed: () => this.displayOptions.transcoding && this.canRunTranscoding(), + iconName: 'cog' + }, { label: $localize`Delete HLS files`, handler: ({ video }) => this.removeVideoFiles(video, 'hls'), diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts index 29c398822..244c38fcd 100755 --- a/scripts/create-transcoding-job.ts +++ b/scripts/create-transcoding-job.ts @@ -5,7 +5,7 @@ import { program } from 'commander' import { VideoModel } from '../server/models/video/video' import { initDatabaseModels } from '../server/initializers/database' import { JobQueue } from '../server/lib/job-queue' -import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils' +import { computeLowerResolutionsToTranscode } from '@server/helpers/ffprobe-utils' import { VideoState, VideoTranscodingPayload } from '@shared/models' import { CONFIG } from '@server/initializers/config' import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc' @@ -50,13 +50,13 @@ async function run () { if (!video) throw new Error('Video not found.') const dataInput: VideoTranscodingPayload[] = [] - const resolution = video.getMaxQualityFile().resolution + const maxResolution = video.getMaxQualityFile().resolution // Generate HLS files if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { const resolutionsEnabled = options.resolution ? [ parseInt(options.resolution) ] - : computeResolutionsToTranscode(resolution, 'vod').concat([ resolution ]) + : computeLowerResolutionsToTranscode(maxResolution, 'vod').concat([ maxResolution ]) for (const resolution of resolutionsEnabled) { dataInput.push({ @@ -66,7 +66,8 @@ async function run () { isPortraitMode: false, copyCodecs: false, isNewVideo: false, - isMaxQuality: false + isMaxQuality: maxResolution === resolution, + autoDeleteWebTorrentIfNeeded: false }) } } else { diff --git a/server/controllers/api/videos/files.ts b/server/controllers/api/videos/files.ts index 2fe4b5a3f..a8b32411d 100644 --- a/server/controllers/api/videos/files.ts +++ b/server/controllers/api/videos/files.ts @@ -3,10 +3,11 @@ import toInt from 'validator/lib/toInt' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { VideoFileModel } from '@server/models/video/video-file' -import { HttpStatusCode } from '@shared/models' +import { HttpStatusCode, UserRight } from '@shared/models' import { asyncMiddleware, authenticate, + ensureUserHasRight, videoFileMetadataGetValidator, videoFilesDeleteHLSValidator, videoFilesDeleteWebTorrentValidator @@ -22,12 +23,14 @@ filesRouter.get('/:id/metadata/:videoFileId', filesRouter.delete('/:id/hls', authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), asyncMiddleware(videoFilesDeleteHLSValidator), asyncMiddleware(removeHLSPlaylist) ) filesRouter.delete('/:id/webtorrent', authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), asyncMiddleware(videoFilesDeleteWebTorrentValidator), asyncMiddleware(removeWebTorrentFiles) ) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 2d088a73e..fc1bcc73d 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -40,6 +40,7 @@ import { videoImportsRouter } from './import' import { liveRouter } from './live' import { ownershipVideoRouter } from './ownership' import { rateVideoRouter } from './rate' +import { transcodingRouter } from './transcoding' import { updateRouter } from './update' import { uploadRouter } from './upload' import { watchingRouter } from './watching' @@ -58,6 +59,7 @@ videosRouter.use('/', liveRouter) videosRouter.use('/', uploadRouter) videosRouter.use('/', updateRouter) videosRouter.use('/', filesRouter) +videosRouter.use('/', transcodingRouter) videosRouter.get('/categories', openapiOperationDoc({ operationId: 'getCategories' }), diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts new file mode 100644 index 000000000..dd6fbd3de --- /dev/null +++ b/server/controllers/api/videos/transcoding.ts @@ -0,0 +1,62 @@ +import express from 'express' +import { computeLowerResolutionsToTranscode } from '@server/helpers/ffprobe-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { addTranscodingJob } from '@server/lib/video' +import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models' +import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares' + +const lTags = loggerTagsFactory('api', 'video') +const transcodingRouter = express.Router() + +transcodingRouter.post('/:videoId/transcoding', + authenticate, + ensureUserHasRight(UserRight.RUN_VIDEO_TRANSCODING), + asyncMiddleware(createTranscodingValidator), + asyncMiddleware(createTranscoding) +) + +// --------------------------------------------------------------------------- + +export { + transcodingRouter +} + +// --------------------------------------------------------------------------- + +async function createTranscoding (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + logger.info('Creating %s transcoding job for %s.', req.body.type, video.url, lTags()) + + const body: VideoTranscodingCreate = req.body + + const { resolution: maxResolution, isPortraitMode } = await video.getMaxQualityResolution() + const resolutions = computeLowerResolutionsToTranscode(maxResolution, 'vod').concat([ maxResolution ]) + + video.state = VideoState.TO_TRANSCODE + await video.save() + + for (const resolution of resolutions) { + if (body.transcodingType === 'hls') { + await addTranscodingJob({ + type: 'new-resolution-to-hls', + videoUUID: video.uuid, + resolution, + isPortraitMode, + copyCodecs: false, + isNewVideo: false, + autoDeleteWebTorrentIfNeeded: false, + isMaxQuality: maxResolution === resolution + }) + } else if (body.transcodingType === 'webtorrent') { + await addTranscodingJob({ + type: 'new-resolution-to-webtorrent', + videoUUID: video.uuid, + isNewVideo: false, + resolution: resolution, + isPortraitMode + }) + } + } + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/controllers/download.ts b/server/controllers/download.ts index 8da710669..43d525f83 100644 --- a/server/controllers/download.ts +++ b/server/controllers/download.ts @@ -85,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { return res.redirect(videoFile.getObjectStorageUrl()) } - await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => { + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}` return res.download(path, filename) @@ -119,7 +119,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response return res.redirect(videoFile.getObjectStorageUrl()) } - await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => { + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` return res.download(path, filename) diff --git a/server/helpers/custom-validators/video-transcoding.ts b/server/helpers/custom-validators/video-transcoding.ts new file mode 100644 index 000000000..cf792f996 --- /dev/null +++ b/server/helpers/custom-validators/video-transcoding.ts @@ -0,0 +1,12 @@ +import { exists } from './misc' + +function isValidCreateTranscodingType (value: any) { + return exists(value) && + (value === 'hls' || value === 'webtorrent') +} + +// --------------------------------------------------------------------------- + +export { + isValidCreateTranscodingType +} diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts index 907f13651..e15628e2a 100644 --- a/server/helpers/ffprobe-utils.ts +++ b/server/helpers/ffprobe-utils.ts @@ -206,7 +206,7 @@ async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData return metadata.streams.find(s => s.codec_type === 'video') || null } -function computeResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') { +function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') { const configResolutions = type === 'vod' ? CONFIG.TRANSCODING.RESOLUTIONS : CONFIG.LIVE.TRANSCODING.RESOLUTIONS @@ -214,7 +214,7 @@ function computeResolutionsToTranscode (videoFileResolution: number, type: 'vod' const resolutionsEnabled: number[] = [] // Put in the order we want to proceed jobs - const resolutions = [ + const resolutions: VideoResolution[] = [ VideoResolution.H_NOVIDEO, VideoResolution.H_480P, VideoResolution.H_360P, @@ -327,7 +327,7 @@ export { getVideoFileFPS, ffprobePromise, getClosestFramerateStandard, - computeResolutionsToTranscode, + computeLowerResolutionsToTranscode, getVideoFileBitrate, canDoQuickTranscode, canDoQuickVideoTranscode, diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 5e1ea6198..c75c058e4 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -100,7 +100,7 @@ function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlayli urlList: buildUrlList(video, videoFile) } - return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => { + return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), async videoPath => { const torrentContent = await createTorrentPromise(videoPath, options) const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 8160e7949..d969549b8 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -37,7 +37,7 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl for (const file of playlist.VideoFiles) { const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => { + await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { const size = await getVideoStreamSize(videoFilePath) const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) @@ -69,10 +69,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP // For all the resolutions available for this video for (const file of playlist.VideoFiles) { const rangeHashes: { [range: string]: string } = {} + const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => { + await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => { + return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { const playlistContent = await readFile(resolutionPlaylistPath) const ranges = getRangesFromPlaylist(playlistContent.toString()) diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index 4beca3d75..54a7c566b 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts @@ -56,16 +56,17 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) { async function moveHLSFiles (video: MVideoWithAllFiles) { for (const playlist of video.VideoStreamingPlaylists) { + const playlistWithVideo = playlist.withVideo(video) for (const file of playlist.VideoFiles) { if (file.storage !== VideoStorage.FILE_SYSTEM) continue // Resolution playlist const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - await storeHLSFile(playlist, video, playlistFilename) + await storeHLSFile(playlistWithVideo, playlistFilename) // Resolution fragmented file - const fileUrl = await storeHLSFile(playlist, video, file.filename) + const fileUrl = await storeHLSFile(playlistWithVideo, file.filename) const oldPath = join(getHLSDirectory(video), file.filename) @@ -78,10 +79,12 @@ async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) { for (const playlist of video.VideoStreamingPlaylists) { if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue + const playlistWithVideo = playlist.withVideo(video) + // Master playlist - playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename) + playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename) // Sha256 segments file - playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename) + playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename) playlist.storage = VideoStorage.OBJECT_STORAGE diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 904ef2e3c..2d0798e12 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -14,7 +14,7 @@ import { VideoTranscodingPayload } from '../../../../shared' import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils' +import { computeLowerResolutionsToTranscode } from '../../../helpers/ffprobe-utils' import { logger, loggerTagsFactory } from '../../../helpers/logger' import { CONFIG } from '../../../initializers/config' import { VideoModel } from '../../../models/video/video' @@ -81,7 +81,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() - await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => { + await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { return generateHlsPlaylistResolution({ video, videoInputPath, @@ -135,7 +135,7 @@ async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodi // --------------------------------------------------------------------------- async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) { - if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { + if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { // Remove webtorrent files if not enabled for (const file of video.VideoFiles) { await video.removeWebTorrentFileAndTorrent(file) @@ -232,6 +232,7 @@ async function createHlsJobIfEnabled (user: MUserId, payload: { isPortraitMode: payload.isPortraitMode, copyCodecs: payload.copyCodecs, isMaxQuality: payload.isMaxQuality, + autoDeleteWebTorrentIfNeeded: true, isNewVideo: payload.isNewVideo } @@ -261,7 +262,7 @@ async function createLowerResolutionsJobs (options: { const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod') + const resolutionsEnabled = computeLowerResolutionsToTranscode(videoFileResolution, 'vod') const resolutionCreated: string[] = [] for (const resolution of resolutionsEnabled) { @@ -288,6 +289,7 @@ async function createLowerResolutionsJobs (options: { isPortraitMode, copyCodecs: false, isMaxQuality: false, + autoDeleteWebTorrentIfNeeded: true, isNewVideo } diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 2562edb75..b3bf5a999 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -3,7 +3,7 @@ import { readFile } from 'fs-extra' import { createServer, Server } from 'net' import { createServer as createServerTLS, Server as ServerTLS } from 'tls' import { - computeResolutionsToTranscode, + computeLowerResolutionsToTranscode, ffprobePromise, getVideoFileBitrate, getVideoFileFPS, @@ -402,7 +402,7 @@ class LiveManager { private buildAllResolutionsToTranscode (originResolution: number) { const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED - ? computeResolutionsToTranscode(originResolution, 'live') + ? computeLowerResolutionsToTranscode(originResolution, 'live') : [] return resolutionsEnabled.concat([ originResolution ]) diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts index 12acb3aec..4f17073f4 100644 --- a/server/lib/object-storage/keys.ts +++ b/server/lib/object-storage/keys.ts @@ -1,12 +1,12 @@ import { join } from 'path' -import { MStreamingPlaylist, MVideoUUID } from '@server/types/models' +import { MStreamingPlaylistVideo } from '@server/types/models' -function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { - return join(generateHLSObjectBaseStorageKey(playlist, video), filename) +function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { + return join(generateHLSObjectBaseStorageKey(playlist), filename) } -function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { - return join(playlist.getStringType(), video.uuid) +function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { + return join(playlist.getStringType(), playlist.Video.uuid) } function generateWebTorrentObjectStorageKey (filename: string) { diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index 15b8f58d5..8988f3e2a 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts @@ -1,17 +1,17 @@ import { join } from 'path' import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models' +import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' import { getHLSDirectory } from '../paths' import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' -function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) { - const baseHlsDirectory = getHLSDirectory(video) +function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) { + const baseHlsDirectory = getHLSDirectory(playlist.Video) return storeObject({ inputPath: join(baseHlsDirectory, filename), - objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename), + objectStorageKey: generateHLSObjectStorageKey(playlist, filename), bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS }) } @@ -24,16 +24,16 @@ function storeWebTorrentFile (filename: string) { }) } -function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) { - return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { + return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) } function removeWebTorrentObjectStorage (videoFile: MVideoFile) { return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) } -async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) { - const key = generateHLSObjectStorageKey(playlist, video, filename) +async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { + const key = generateHLSObjectStorageKey(playlist, filename) logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index d2384f53c..36270e5c1 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts @@ -115,7 +115,7 @@ function generateVideoMiniature (options: { }) { const { video, videoFile, type } = options - return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => { + return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => { const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) const thumbnailCreator = videoFile.isAudio() diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index 250a678eb..d0db05216 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts @@ -35,7 +35,7 @@ function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVid const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' - return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => { + return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => { const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) @@ -81,7 +81,7 @@ function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: V const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const extname = '.mp4' - return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => { + return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => { const newVideoFile = new VideoFileModel({ resolution, extname, @@ -134,7 +134,7 @@ function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolutio const inputVideoFile = video.getMinQualityFile() - return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => { + return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => { const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) // If the user updates the video preview during transcoding diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts index 4c5d0c89d..27058005c 100644 --- a/server/lib/video-path-manager.ts +++ b/server/lib/video-path-manager.ts @@ -3,7 +3,14 @@ import { extname, join } from 'path' import { buildUUID } from '@server/helpers/uuid' import { extractVideo } from '@server/helpers/video' import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' +import { + MStreamingPlaylistVideo, + MVideo, + MVideoFile, + MVideoFileStreamingPlaylistVideo, + MVideoFileVideo, + MVideoUUID +} from '@server/types/models' import { VideoStorage } from '@shared/models' import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' @@ -43,10 +50,10 @@ class VideoPathManager { return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename) } - async makeAvailableVideoFile (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB) { + async makeAvailableVideoFile (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { if (videoFile.storage === VideoStorage.FILE_SYSTEM) { return this.makeAvailableFactory( - () => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile), + () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile), false, cb ) @@ -55,10 +62,10 @@ class VideoPathManager { const destination = this.buildTMPDestination(videoFile.filename) if (videoFile.isHLS()) { - const video = extractVideo(videoOrPlaylist) + const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist return this.makeAvailableFactory( - () => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination), + () => makeHLSFileAvailable(playlist, videoFile.filename, destination), true, cb ) @@ -71,19 +78,20 @@ class VideoPathManager { ) } - async makeAvailableResolutionPlaylistFile (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB) { + async makeAvailableResolutionPlaylistFile (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { const filename = getHlsResolutionPlaylistFilename(videoFile.filename) if (videoFile.storage === VideoStorage.FILE_SYSTEM) { return this.makeAvailableFactory( - () => join(getHLSDirectory(playlist.Video), filename), + () => join(getHLSDirectory(videoFile.getVideo()), filename), false, cb ) } + const playlist = videoFile.VideoStreamingPlaylist return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), + () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), true, cb ) @@ -99,7 +107,7 @@ class VideoPathManager { } return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)), + () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), true, cb ) diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts index 0b51f5c6b..bf6dd4bc8 100644 --- a/server/lib/video-state.ts +++ b/server/lib/video-state.ts @@ -80,6 +80,8 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b } function moveToFailedTranscodingState (video: MVideoFullLight) { + if (video.state === VideoState.TRANSCODING_FAILED) return + return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) } diff --git a/server/lib/video.ts b/server/lib/video.ts index 0a2b93cc0..1cfe4f27c 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -105,7 +105,7 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF return addTranscodingJob(dataInput, jobOptions) } -async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) { +async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) { await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options) diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts index fd1d58093..f365d8ee1 100644 --- a/server/middlewares/validators/videos/index.ts +++ b/server/middlewares/validators/videos/index.ts @@ -9,4 +9,5 @@ export * from './video-ownership-changes' export * from './video-watch' export * from './video-rates' export * from './video-shares' +export * from './video-transcoding' export * from './videos' diff --git a/server/middlewares/validators/videos/video-files.ts b/server/middlewares/validators/videos/video-files.ts index 282594ab6..c1fa77502 100644 --- a/server/middlewares/validators/videos/video-files.ts +++ b/server/middlewares/validators/videos/video-files.ts @@ -1,6 +1,6 @@ import express from 'express' -import { MUser, MVideo } from '@server/types/models' -import { HttpStatusCode, UserRight } from '../../../../shared' +import { MVideo } from '@server/types/models' +import { HttpStatusCode } from '../../../../shared' import { logger } from '../../../helpers/logger' import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' @@ -14,9 +14,7 @@ const videoFilesDeleteWebTorrentValidator = [ if (!await doesVideoExist(req.params.id, res)) return const video = res.locals.videoAll - const user = res.locals.oauth.token.User - if (!checkUserCanDeleteFiles(user, res)) return if (!checkLocalVideo(video, res)) return if (!video.hasWebTorrentFiles()) { @@ -47,9 +45,7 @@ const videoFilesDeleteHLSValidator = [ if (!await doesVideoExist(req.params.id, res)) return const video = res.locals.videoAll - const user = res.locals.oauth.token.User - if (!checkUserCanDeleteFiles(user, res)) return if (!checkLocalVideo(video, res)) return if (!video.getHLSPlaylist()) { @@ -89,16 +85,3 @@ function checkLocalVideo (video: MVideo, res: express.Response) { return true } - -function checkUserCanDeleteFiles (user: MUser, res: express.Response) { - if (user.hasRight(UserRight.MANAGE_VIDEO_FILES) !== true) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'User cannot update video files' - }) - - return false - } - - return true -} diff --git a/server/middlewares/validators/videos/video-transcoding.ts b/server/middlewares/validators/videos/video-transcoding.ts new file mode 100644 index 000000000..34f231d45 --- /dev/null +++ b/server/middlewares/validators/videos/video-transcoding.ts @@ -0,0 +1,55 @@ +import express from 'express' +import { body } from 'express-validator' +import { isValidCreateTranscodingType } from '@server/helpers/custom-validators/video-transcoding' +import { CONFIG } from '@server/initializers/config' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { HttpStatusCode } from '@shared/models' +import { logger } from '../../../helpers/logger' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' + +const createTranscodingValidator = [ + isValidVideoIdParam('videoId'), + + body('transcodingType') + .custom(isValidCreateTranscodingType).withMessage('Should have a valid transcoding type'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking createTranscodingValidator parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'all')) return + + const video = res.locals.videoAll + + if (video.remote) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot run transcoding job on a remote video' + }) + } + + if (CONFIG.TRANSCODING.ENABLED !== true) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot run transcoding job because transcoding is disabled on this instance' + }) + } + + // Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state + const info = await VideoJobInfoModel.load(video.id) + if (info && info.pendingTranscode !== 0) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'This video is already being transcoded' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + createTranscodingValidator +} diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 461e296df..fd4da68ed 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -2,8 +2,7 @@ import { uuidToShort } from '@server/helpers/uuid' import { generateMagnetUri } from '@server/helpers/webtorrent' import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' import { VideoViews } from '@server/lib/video-views' -import { VideosCommonQueryAfterSanitize } from '@shared/models' -import { VideoFile } from '@shared/models/videos/video-file.model' +import { VideoFile, VideosCommonQueryAfterSanitize } from '@shared/models' import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos' import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model' diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 106f9602b..87311c0ed 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -25,7 +25,7 @@ import { logger } from '@server/helpers/logger' import { extractVideo } from '@server/helpers/video' import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' import { getFSTorrentFilePath } from '@server/lib/paths' -import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { VideoStorage } from '@shared/models' import { @@ -536,4 +536,10 @@ export class VideoFileModel extends Model (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) ) } + + withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { + if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist }) + + return Object.assign(this, { Video: videoOrPlaylist }) + } } diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts index 7c1fe6734..cb1f3f2f0 100644 --- a/server/models/video/video-job-info.ts +++ b/server/models/video/video-job-info.ts @@ -49,7 +49,7 @@ export class VideoJobInfoModel extends Model>> { const file = this.getMaxQualityFile() const videoOrPlaylist = file.getVideoOrStreamingPlaylist() - return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => { + return VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(videoOrPlaylist), originalFilePath => { return getVideoFileResolution(originalFilePath) }) } @@ -1742,7 +1741,7 @@ export class VideoModel extends Model>> { ) if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - await removeHLSObjectStorage(streamingPlaylist, this) + await removeHLSObjectStorage(streamingPlaylist.withVideo(this)) } } } diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index ff7dc4abb..e052296db 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -15,6 +15,7 @@ import './plugins' import './redundancy' import './search' import './services' +import './transcoding' import './upload-quota' import './user-notifications' import './user-subscriptions' diff --git a/server/tests/api/check-params/transcoding.ts b/server/tests/api/check-params/transcoding.ts new file mode 100644 index 000000000..a8daafe3e --- /dev/null +++ b/server/tests/api/check-params/transcoding.ts @@ -0,0 +1,104 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' +import { HttpStatusCode, UserRole } from '@shared/models' + +describe('Test transcoding API validators', function () { + let servers: PeerTubeServer[] + + let userToken: string + let moderatorToken: string + + let remoteId: string + let validId: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) + moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) + + { + const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) + remoteId = uuid + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) + validId = uuid + } + + await waitJobs(servers) + + await servers[0].config.enableTranscoding() + }) + + it('Should not run transcoding of a unknown video', async function () { + await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'hls', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'webtorrent', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not run transcoding of a remote video', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'hls', expectedStatus }) + await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'webtorrent', expectedStatus }) + }) + + it('Should not run transcoding by a non admin user', async function () { + const expectedStatus = HttpStatusCode.FORBIDDEN_403 + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', token: userToken, expectedStatus }) + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', token: moderatorToken, expectedStatus }) + }) + + it('Should not run transcoding without transcoding type', async function () { + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not run transcoding with an incorrect transcoding type', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'toto' as any, expectedStatus }) + }) + + it('Should not run transcoding if the instance disabled it', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].config.disableTranscoding() + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', expectedStatus }) + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', expectedStatus }) + }) + + it('Should run transcoding', async function () { + this.timeout(120_000) + + await servers[0].config.enableTranscoding() + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls' }) + await waitJobs(servers) + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent' }) + await waitJobs(servers) + }) + + it('Should not run transcoding on a video that is already being transcoded', async function () { + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent' }) + + const expectedStatus = HttpStatusCode.CONFLICT_409 + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', expectedStatus }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts index 48b10d2b5..61936d562 100644 --- a/server/tests/api/check-params/video-files.ts +++ b/server/tests/api/check-params/video-files.ts @@ -1,16 +1,19 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' +import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' import { HttpStatusCode, UserRole } from '@shared/models' describe('Test videos files', function () { let servers: PeerTubeServer[] + let webtorrentId: string let hlsId: string let remoteId: string + let userToken: string let moderatorToken: string + let validId1: string let validId2: string @@ -22,9 +25,16 @@ describe('Test videos files', function () { servers = await createMultipleServers(2) await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) + { + const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) + remoteId = uuid + } + { await servers[0].config.enableTranscoding(true, true) @@ -58,6 +68,11 @@ describe('Test videos files', function () { await waitJobs(servers) }) + it('Should not delete files of a unknown video', async function () { + await servers[0].videos.removeHLSFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await servers[0].videos.removeWebTorrentFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + it('Should not delete files of a remote video', async function () { await servers[0].videos.removeHLSFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) await servers[0].videos.removeWebTorrentFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index f92e339e7..bedb9b8b6 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -6,6 +6,7 @@ import './video-captions' import './video-change-ownership' import './video-channels' import './video-comments' +import './video-create-transcoding' import './video-description' import './video-files' import './video-hls' diff --git a/server/tests/api/videos/video-create-transcoding.ts b/server/tests/api/videos/video-create-transcoding.ts new file mode 100644 index 000000000..bae06ac6c --- /dev/null +++ b/server/tests/api/videos/video-create-transcoding.ts @@ -0,0 +1,156 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { + areObjectStorageTestsDisabled, + cleanupTests, + createMultipleServers, + doubleFollow, + expectStartWith, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@shared/extra-utils' +import { HttpStatusCode, VideoDetails } from '@shared/models' + +const expect = chai.expect + +async function checkFilesInObjectStorage (video: VideoDetails) { + for (const file of video.files) { + expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) + } + + for (const file of video.streamingPlaylists[0].files) { + expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) + await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) + } +} + +async function expectNoFailedTranscodingJob (server: PeerTubeServer) { + const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' }) + expect(data).to.have.lengthOf(0) +} + +function runTests (objectStorage: boolean) { + let servers: PeerTubeServer[] = [] + let videoUUID: string + let publishedAt: string + + before(async function () { + this.timeout(120000) + + const config = objectStorage + ? ObjectStorageCommand.getDefaultConfig() + : {} + + // Run server 2 to have transcoding enabled + servers = await createMultipleServers(2, config) + await setAccessTokensToServers(servers) + + await servers[0].config.disableTranscoding() + + await doubleFollow(servers[0], servers[1]) + + if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() + + const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = shortUUID + + const video = await servers[0].videos.get({ id: videoUUID }) + publishedAt = video.publishedAt as string + + await servers[0].config.enableTranscoding() + + await waitJobs(servers) + }) + + it('Should generate HLS', async function () { + this.timeout(60000) + + await servers[0].videos.runTranscoding({ + videoId: videoUUID, + transcodingType: 'hls' + }) + + await waitJobs(servers) + await expectNoFailedTranscodingJob(servers[0]) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (objectStorage) await checkFilesInObjectStorage(videoDetails) + } + }) + + it('Should generate WebTorrent', async function () { + this.timeout(60000) + + await servers[0].videos.runTranscoding({ + videoId: videoUUID, + transcodingType: 'webtorrent' + }) + + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(5) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (objectStorage) await checkFilesInObjectStorage(videoDetails) + } + }) + + it('Should generate WebTorrent from HLS only video', async function () { + this.timeout(60000) + + await servers[0].videos.removeWebTorrentFiles({ videoId: videoUUID }) + await waitJobs(servers) + + await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' }) + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(5) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (objectStorage) await checkFilesInObjectStorage(videoDetails) + } + }) + + it('Should not have updated published at attributes', async function () { + const video = await servers[0].videos.get({ id: videoUUID }) + + expect(video.publishedAt).to.equal(publishedAt) + }) + + after(async function () { + await cleanupTests(servers) + }) +} + +describe('Test create transcoding jobs from API', function () { + + describe('On filesystem', function () { + runTests(false) + }) + + describe('On object storage', function () { + if (areObjectStorageTestsDisabled()) return + + runTests(true) + }) +}) diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts index f28397816..6636e7e4d 100644 --- a/shared/extra-utils/server/jobs-command.ts +++ b/shared/extra-utils/server/jobs-command.ts @@ -36,6 +36,21 @@ export class JobsCommand extends AbstractCommand { }) } + listFailed (options: OverrideCommandOptions & { + jobType?: JobType + }) { + const path = this.buildJobsUrl('failed') + + return this.getRequestBody>({ + ...options, + + path, + query: { start: 0, count: 50 }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + private buildJobsUrl (state?: JobState) { let path = '/api/v1/jobs' diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 13a7d0e1c..7ec9c3647 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts @@ -18,7 +18,8 @@ import { VideoDetails, VideoFileMetadata, VideoPrivacy, - VideosCommonQuery + VideosCommonQuery, + VideoTranscodingCreate } from '@shared/models' import { buildAbsoluteFixturePath, wait } from '../miscs' import { unwrapBody } from '../requests' @@ -630,6 +631,24 @@ export class VideosCommand extends AbstractCommand { }) } + runTranscoding (options: OverrideCommandOptions & { + videoId: number | string + transcodingType: 'hls' | 'webtorrent' + }) { + const path = '/api/v1/videos/' + options.videoId + '/transcoding' + + const fields: VideoTranscodingCreate = pick(options, [ 'transcodingType' ]) + + return this.postBodyRequest({ + ...options, + + path, + fields, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + // --------------------------------------------------------------------------- private buildListQuery (options: VideosCommonQuery) { diff --git a/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts b/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts index a0422a460..b6fb46ba0 100644 --- a/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts +++ b/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts @@ -1,4 +1,4 @@ -import { EncoderOptionsBuilder } from '../../../videos/video-transcoding.model' +import { EncoderOptionsBuilder } from '../../../videos/transcoding' export interface PluginTranscodingManager { addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder): boolean diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 6da2753b3..ecc960da5 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -1,5 +1,5 @@ import { ContextType } from '../activitypub/context' -import { VideoResolution } from '../videos/video-resolution.enum' +import { VideoResolution } from '../videos/file/video-resolution.enum' import { SendEmailOptions } from './emailer.model' export type JobState = 'active' | 'completed' | 'failed' | 'waiting' | 'delayed' | 'paused' @@ -106,6 +106,8 @@ export interface HLSTranscodingPayload extends BaseTranscodingPayload { isPortraitMode?: boolean resolution: VideoResolution copyCodecs: boolean + + autoDeleteWebTorrentIfNeeded: boolean isMaxQuality: boolean } diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index 96bccaf2f..6415ca6f2 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts @@ -40,5 +40,6 @@ export const enum UserRight { MANAGE_VIDEOS_REDUNDANCIES, - MANAGE_VIDEO_FILES + MANAGE_VIDEO_FILES, + RUN_VIDEO_TRANSCODING } diff --git a/shared/models/videos/file/index.ts b/shared/models/videos/file/index.ts new file mode 100644 index 000000000..78a784a3c --- /dev/null +++ b/shared/models/videos/file/index.ts @@ -0,0 +1,3 @@ +export * from './video-file-metadata.model' +export * from './video-file.model' +export * from './video-resolution.enum' diff --git a/shared/models/videos/video-file-metadata.model.ts b/shared/models/videos/file/video-file-metadata.model.ts similarity index 100% rename from shared/models/videos/video-file-metadata.model.ts rename to shared/models/videos/file/video-file-metadata.model.ts diff --git a/shared/models/videos/video-file.model.ts b/shared/models/videos/file/video-file.model.ts similarity index 88% rename from shared/models/videos/video-file.model.ts rename to shared/models/videos/file/video-file.model.ts index 28fce0aaf..0ea857e7a 100644 --- a/shared/models/videos/video-file.model.ts +++ b/shared/models/videos/file/video-file.model.ts @@ -1,4 +1,4 @@ -import { VideoConstant } from './video-constant.model' +import { VideoConstant } from '../video-constant.model' import { VideoFileMetadata } from './video-file-metadata.model' import { VideoResolution } from './video-resolution.enum' diff --git a/shared/models/videos/video-resolution.enum.ts b/shared/models/videos/file/video-resolution.enum.ts similarity index 100% rename from shared/models/videos/video-resolution.enum.ts rename to shared/models/videos/file/video-resolution.enum.ts diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index 3d3eedcc6..67614efc9 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -4,9 +4,11 @@ export * from './change-ownership' export * from './channel' export * from './comment' export * from './live' +export * from './file' export * from './import' export * from './playlist' export * from './rate' +export * from './transcoding' export * from './nsfw-policy.type' @@ -15,14 +17,10 @@ export * from './thumbnail.type' export * from './video-constant.model' export * from './video-create.model' -export * from './video-file-metadata.model' -export * from './video-file.model' - export * from './video-privacy.enum' export * from './video-filter.type' export * from './video-include.enum' export * from './video-rate.type' -export * from './video-resolution.enum' export * from './video-schedule-update.model' export * from './video-sort-field.type' @@ -32,9 +30,6 @@ export * from './video-storage.enum' export * from './video-streaming-playlist.model' export * from './video-streaming-playlist.type' -export * from './video-transcoding.model' -export * from './video-transcoding-fps.model' - export * from './video-update.model' export * from './video.model' export * from './video-create-result.model' diff --git a/shared/models/videos/transcoding/index.ts b/shared/models/videos/transcoding/index.ts new file mode 100644 index 000000000..14472d900 --- /dev/null +++ b/shared/models/videos/transcoding/index.ts @@ -0,0 +1,3 @@ +export * from './video-transcoding-create.model' +export * from './video-transcoding-fps.model' +export * from './video-transcoding.model' diff --git a/shared/models/videos/transcoding/video-transcoding-create.model.ts b/shared/models/videos/transcoding/video-transcoding-create.model.ts new file mode 100644 index 000000000..aeb393e57 --- /dev/null +++ b/shared/models/videos/transcoding/video-transcoding-create.model.ts @@ -0,0 +1,3 @@ +export interface VideoTranscodingCreate { + transcodingType: 'hls' | 'webtorrent' +} diff --git a/shared/models/videos/video-transcoding-fps.model.ts b/shared/models/videos/transcoding/video-transcoding-fps.model.ts similarity index 100% rename from shared/models/videos/video-transcoding-fps.model.ts rename to shared/models/videos/transcoding/video-transcoding-fps.model.ts diff --git a/shared/models/videos/video-transcoding.model.ts b/shared/models/videos/transcoding/video-transcoding.model.ts similarity index 94% rename from shared/models/videos/video-transcoding.model.ts rename to shared/models/videos/transcoding/video-transcoding.model.ts index 83b8e98a0..3a7fb6472 100644 --- a/shared/models/videos/video-transcoding.model.ts +++ b/shared/models/videos/transcoding/video-transcoding.model.ts @@ -1,4 +1,4 @@ -import { VideoResolution } from './video-resolution.enum' +import { VideoResolution } from '../file/video-resolution.enum' // Types used by plugins and ffmpeg-utils diff --git a/shared/models/videos/video-streaming-playlist.model.ts b/shared/models/videos/video-streaming-playlist.model.ts index b547a0ac7..11919a4ee 100644 --- a/shared/models/videos/video-streaming-playlist.model.ts +++ b/shared/models/videos/video-streaming-playlist.model.ts @@ -1,5 +1,5 @@ import { VideoStreamingPlaylistType } from './video-streaming-playlist.type' -import { VideoFile } from './video-file.model' +import { VideoFile } from './file' export interface VideoStreamingPlaylist { id: number diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index 8d223cded..f98eed012 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts @@ -1,7 +1,7 @@ import { Account, AccountSummary } from '../actors' import { VideoChannel, VideoChannelSummary } from './channel/video-channel.model' +import { VideoFile } from './file' import { VideoConstant } from './video-constant.model' -import { VideoFile } from './video-file.model' import { VideoPrivacy } from './video-privacy.enum' import { VideoScheduleUpdate } from './video-schedule-update.model' import { VideoState } from './video-state.enum' diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 88a089fc7..cfba7b361 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -267,6 +267,10 @@ tags: description: Like/dislike a video. - name: Video Playlists description: Operations dealing with playlists of videos. Playlists are bound to users and/or channels. + - name: Video Files + description: Operations on video files + - name: Video Transcoding + description: Video transcoding related operations - name: Feeds description: Server syndication feeds - name: Search @@ -309,6 +313,8 @@ x-tagGroups: - Video Playlists - Video Ownership Change - Video Mirroring + - Video Files + - Video Transcoding - Live Videos - Feeds - name: Search @@ -3568,6 +3574,69 @@ paths: '404': description: video does not exist + '/videos/{id}/hls': + delete: + summary: Delete video HLS files + security: + - OAuth2: + - admin + tags: + - Video Files + operationId: delVideoHLS + parameters: + - $ref: '#/components/parameters/idOrUUID' + responses: + '204': + description: successful operation + '404': + description: video does not exist + '/videos/{id}/webtorrent': + delete: + summary: Delete video WebTorrent files + security: + - OAuth2: + - admin + tags: + - Video Files + operationId: delVideoWebTorrent + parameters: + - $ref: '#/components/parameters/idOrUUID' + responses: + '204': + description: successful operation + '404': + description: video does not exist + + '/videos/{id}/transcoding': + post: + summary: Create a transcoding job + security: + - OAuth2: + - admin + tags: + - Video Transcoding + operationId: createVideoTranscoding + parameters: + - $ref: '#/components/parameters/idOrUUID' + requestBody: + content: + application/json: + schema: + type: object + properties: + transcodingType: + type: string + enum: + - hls + - webtorrent + required: + - transcodingType + responses: + '204': + description: successful operation + '404': + description: video does not exist + /search/videos: get: tags: