forks
/
peertube
1
0
Fork 0
Code Releases Activity

Add ability to run transcoding jobs

This commit is contained in:
Chocobozzz 2021-11-18 14:35:08 +01:00
parent b46cf4b920
commit ad5db1044c
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
54 changed files with 715 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client/src/app/+admin/overview/videos/video-list.component.html
View File

@ -56,8 +56,8 @@
<td class="action-cell">
<my-video-actions-dropdown
placement="bottom auto" buttonDirection="horizontal" [buttonStyled]="true" [video]="video"
[displayOptions]="videoActionsOptions" (videoRemoved)="reloadData()" (videoFilesRemoved)="reloadData()"
placement="bottom auto" buttonDirection="horizontal" [buttonStyled]="true" [video]="video" [displayOptions]="videoActionsOptions"
(videoRemoved)="reloadData()" (videoFilesRemoved)="reloadData()" (transcodingCreated)="reloadData()"
></my-video-actions-dropdown>
</td>

32
client/src/app/+admin/overview/videos/video-list.component.ts
View File

@ -40,7 +40,8 @@ export class VideoListComponent extends RestTable implements OnInit {
duplicate: true,
mute: true,
liveInfo: false,
removeFiles: true
removeFiles: true,
transcoding: true
}
loading = true
@ -89,16 +90,28 @@ export class VideoListComponent extends RestTable implements OnInit {
}
],
[
{
label: $localize`Run HLS transcoding`,
handler: videos => this.runTranscoding(videos, 'hls'),
isDisplayed: videos => videos.every(v => v.canRunTranscoding(this.authUser)),
iconName: 'cog'
},
{
label: $localize`Run WebTorrent transcoding`,
handler: videos => this.runTranscoding(videos, 'webtorrent'),
isDisplayed: videos => videos.every(v => v.canRunTranscoding(this.authUser)),
iconName: 'cog'
},
{
label: $localize`Delete HLS files`,
handler: videos => this.removeVideoFiles(videos, 'hls'),
isDisplayed: videos => this.authUser.hasRight(UserRight.MANAGE_VIDEO_FILES) && videos.every(v => v.hasHLS() && v.hasWebTorrent()),
isDisplayed: videos => videos.every(v => v.canRemoveFiles(this.authUser)),
iconName: 'delete'
},
{
label: $localize`Delete WebTorrent files`,
handler: videos => this.removeVideoFiles(videos, 'webtorrent'),
isDisplayed: videos => this.authUser.hasRight(UserRight.MANAGE_VIDEO_FILES) && videos.every(v => v.hasHLS() && v.hasWebTorrent()),
isDisplayed: videos => videos.every(v => v.canRemoveFiles(this.authUser)),
iconName: 'delete'
}
]
@ -226,4 +239,17 @@ export class VideoListComponent extends RestTable implements OnInit {
error: err => this.notifier.error(err.message)
})
}
private runTranscoding (videos: Video[], type: 'hls' | 'webtorrent') {
this.videoService.runTranscoding(videos.map(v => v.id), type)
.subscribe({
next: () => {
this.notifier.success($localize`Transcoding jobs created.`)
this.reloadData()
},
error: err => this.notifier.error(err.message)
})
}
}

12
client/src/app/shared/shared-main/video/video.model.ts
View File

@ -220,6 +220,18 @@ export class Video implements VideoServerModel {
return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO))
}
canRemoveFiles (user: AuthUser) {
return user.hasRight(UserRight.MANAGE_VIDEO_FILES) &&
this.state.id !== VideoState.TO_TRANSCODE &&
this.hasHLS() &&
this.hasWebTorrent()
}
canRunTranscoding (user: AuthUser) {
return user.hasRight(UserRight.RUN_VIDEO_TRANSCODING) &&
this.state.id !== VideoState.TO_TRANSCODE
}
hasHLS () {
return this.streamingPlaylists?.some(p => p.type === VideoStreamingPlaylistType.HLS)
}

12
client/src/app/shared/shared-main/video/video.service.ts
View File

@ -21,6 +21,7 @@ import {
VideoInclude,
VideoPrivacy,
VideoSortField,
VideoTranscodingCreate,
VideoUpdate
} from '@shared/models'
import { environment } from '../../../../environments/environment'
@ -308,6 +309,17 @@ export class VideoService {
)
}
runTranscoding (videoIds: (number | string)[], type: 'hls' | 'webtorrent') {
const body: VideoTranscodingCreate = { transcodingType: type }
return from(videoIds)
.pipe(
concatMap(id => this.authHttp.post(VideoService.BASE_VIDEO_URL + '/' + id + '/transcoding', body)),
toArray(),
catchError(err => this.restExtractor.handleError(err))
)
}
loadCompleteDescription (descriptionPath: string) {
return this.authHttp
.get<{ description: string }>(environment.apiUrl + descriptionPath)

38
client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts
View File

@ -2,7 +2,7 @@ import { Component, EventEmitter, Input, OnChanges, Output, ViewChild } from '@a
import { AuthService, ConfirmService, Notifier, ScreenService } from '@app/core'
import { BlocklistService, VideoBlockComponent, VideoBlockService, VideoReportComponent } from '@app/shared/shared-moderation'
import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap'
import { UserRight, VideoCaption } from '@shared/models'
import { UserRight, VideoCaption, VideoState } from '@shared/models'
import {
Actor,
DropdownAction,
@ -28,6 +28,7 @@ export type VideoActionsDisplayType = {
mute?: boolean
liveInfo?: boolean
removeFiles?: boolean
transcoding?: boolean
}
@Component({
@ -56,7 +57,9 @@ export class VideoActionsDropdownComponent implements OnChanges {
report: true,
duplicate: true,
mute: true,
liveInfo: false
liveInfo: false,
removeFiles: false,
transcoding: false
}
@Input() placement = 'left'
@ -71,6 +74,7 @@ export class VideoActionsDropdownComponent implements OnChanges {
@Output() videoUnblocked = new EventEmitter()
@Output() videoBlocked = new EventEmitter()
@Output() videoAccountMuted = new EventEmitter()
@Output() transcodingCreated = new EventEmitter()
@Output() modalOpened = new EventEmitter()
videoActions: DropdownAction<{ video: Video }>[][] = []
@ -177,7 +181,11 @@ export class VideoActionsDropdownComponent implements OnChanges {
}
canRemoveVideoFiles () {
return this.user.hasRight(UserRight.MANAGE_VIDEO_FILES) && this.video.hasHLS() && this.video.hasWebTorrent()
return this.video.canRemoveFiles(this.user)
}
canRunTranscoding () {
return this.video.canRunTranscoding(this.user)
}
/* Action handlers */
@ -268,6 +276,18 @@ export class VideoActionsDropdownComponent implements OnChanges {
})
}
runTranscoding (video: Video, type: 'hls' | 'webtorrent') {
this.videoService.runTranscoding([ video.id ], type)
.subscribe({
next: () => {
this.notifier.success($localize`Transcoding jobs created for ${video.name}.`)
this.transcodingCreated.emit()
},
error: err => this.notifier.error(err.message)
})
}
onVideoBlocked () {
this.videoBlocked.emit()
}
@ -341,6 +361,18 @@ export class VideoActionsDropdownComponent implements OnChanges {
}
],
[
{
label: $localize`Run HLS transcoding`,
handler: ({ video }) => this.runTranscoding(video, 'hls'),
isDisplayed: () => this.displayOptions.transcoding && this.canRunTranscoding(),
iconName: 'cog'
},
{
label: $localize`Run WebTorrent transcoding`,
handler: ({ video }) => this.runTranscoding(video, 'webtorrent'),
isDisplayed: () => this.displayOptions.transcoding && this.canRunTranscoding(),
iconName: 'cog'
},
{
label: $localize`Delete HLS files`,
handler: ({ video }) => this.removeVideoFiles(video, 'hls'),

9
scripts/create-transcoding-job.ts
View File

@ -5,7 +5,7 @@ import { program } from 'commander'
import { VideoModel } from '../server/models/video/video'
import { initDatabaseModels } from '../server/initializers/database'
import { JobQueue } from '../server/lib/job-queue'
import { computeResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
import { computeLowerResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
import { VideoState, VideoTranscodingPayload } from '@shared/models'
import { CONFIG } from '@server/initializers/config'
import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc'
@ -50,13 +50,13 @@ async function run () {
if (!video) throw new Error('Video not found.')
const dataInput: VideoTranscodingPayload[] = []
const resolution = video.getMaxQualityFile().resolution
const maxResolution = video.getMaxQualityFile().resolution
// Generate HLS files
if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
const resolutionsEnabled = options.resolution
? [ parseInt(options.resolution) ]
: computeResolutionsToTranscode(resolution, 'vod').concat([ resolution ])
: computeLowerResolutionsToTranscode(maxResolution, 'vod').concat([ maxResolution ])
for (const resolution of resolutionsEnabled) {
dataInput.push({
@ -66,7 +66,8 @@ async function run () {
isPortraitMode: false,
copyCodecs: false,
isNewVideo: false,
isMaxQuality: false
isMaxQuality: maxResolution === resolution,
autoDeleteWebTorrentIfNeeded: false
})
}
} else {

5
server/controllers/api/videos/files.ts
View File

@ -3,10 +3,11 @@ import toInt from 'validator/lib/toInt'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { VideoFileModel } from '@server/models/video/video-file'
import { HttpStatusCode } from '@shared/models'
import { HttpStatusCode, UserRight } from '@shared/models'
import {
asyncMiddleware,
authenticate,
ensureUserHasRight,
videoFileMetadataGetValidator,
videoFilesDeleteHLSValidator,
videoFilesDeleteWebTorrentValidator
@ -22,12 +23,14 @@ filesRouter.get('/:id/metadata/:videoFileId',
filesRouter.delete('/:id/hls',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
asyncMiddleware(videoFilesDeleteHLSValidator),
asyncMiddleware(removeHLSPlaylist)
)
filesRouter.delete('/:id/webtorrent',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
asyncMiddleware(videoFilesDeleteWebTorrentValidator),
asyncMiddleware(removeWebTorrentFiles)
)

2
server/controllers/api/videos/index.ts
View File

@ -40,6 +40,7 @@ import { videoImportsRouter } from './import'
import { liveRouter } from './live'
import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { transcodingRouter } from './transcoding'
import { updateRouter } from './update'
import { uploadRouter } from './upload'
import { watchingRouter } from './watching'
@ -58,6 +59,7 @@ videosRouter.use('/', liveRouter)
videosRouter.use('/', uploadRouter)
videosRouter.use('/', updateRouter)
videosRouter.use('/', filesRouter)
videosRouter.use('/', transcodingRouter)
videosRouter.get('/categories',
openapiOperationDoc({ operationId: 'getCategories' }),

62
server/controllers/api/videos/transcoding.ts Normal file
View File

@ -0,0 +1,62 @@
import express from 'express'
import { computeLowerResolutionsToTranscode } from '@server/helpers/ffprobe-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { addTranscodingJob } from '@server/lib/video'
import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models'
import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares'
const lTags = loggerTagsFactory('api', 'video')
const transcodingRouter = express.Router()
transcodingRouter.post('/:videoId/transcoding',
authenticate,
ensureUserHasRight(UserRight.RUN_VIDEO_TRANSCODING),
asyncMiddleware(createTranscodingValidator),
asyncMiddleware(createTranscoding)
)
// ---------------------------------------------------------------------------
export {
transcodingRouter
}
// ---------------------------------------------------------------------------
async function createTranscoding (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
logger.info('Creating %s transcoding job for %s.', req.body.type, video.url, lTags())
const body: VideoTranscodingCreate = req.body
const { resolution: maxResolution, isPortraitMode } = await video.getMaxQualityResolution()
const resolutions = computeLowerResolutionsToTranscode(maxResolution, 'vod').concat([ maxResolution ])
video.state = VideoState.TO_TRANSCODE
await video.save()
for (const resolution of resolutions) {
if (body.transcodingType === 'hls') {
await addTranscodingJob({
type: 'new-resolution-to-hls',
videoUUID: video.uuid,
resolution,
isPortraitMode,
copyCodecs: false,
isNewVideo: false,
autoDeleteWebTorrentIfNeeded: false,
isMaxQuality: maxResolution === resolution
})
} else if (body.transcodingType === 'webtorrent') {
await addTranscodingJob({
type: 'new-resolution-to-webtorrent',
videoUUID: video.uuid,
isNewVideo: false,
resolution: resolution,
isPortraitMode
})
}
}
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}

4
server/controllers/download.ts
View File

@ -85,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
return res.redirect(videoFile.getObjectStorageUrl())
}
await VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, path => {
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
const filename = `${video.name}-${videoFile.resolution}p${videoFile.extname}`
return res.download(path, filename)
@ -119,7 +119,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
return res.redirect(videoFile.getObjectStorageUrl())
}
await VideoPathManager.Instance.makeAvailableVideoFile(streamingPlaylist, videoFile, path => {
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
const filename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
return res.download(path, filename)

12
server/helpers/custom-validators/video-transcoding.ts Normal file
View File

@ -0,0 +1,12 @@
import { exists } from './misc'
function isValidCreateTranscodingType (value: any) {
return exists(value) &&
(value === 'hls' || value === 'webtorrent')
}
// ---------------------------------------------------------------------------
export {
isValidCreateTranscodingType
}

6
server/helpers/ffprobe-utils.ts
View File

@ -206,7 +206,7 @@ async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData
return metadata.streams.find(s => s.codec_type === 'video') || null
}
function computeResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') {
function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') {
const configResolutions = type === 'vod'
? CONFIG.TRANSCODING.RESOLUTIONS
: CONFIG.LIVE.TRANSCODING.RESOLUTIONS
@ -214,7 +214,7 @@ function computeResolutionsToTranscode (videoFileResolution: number, type: 'vod'
const resolutionsEnabled: number[] = []
// Put in the order we want to proceed jobs
const resolutions = [
const resolutions: VideoResolution[] = [
VideoResolution.H_NOVIDEO,
VideoResolution.H_480P,
VideoResolution.H_360P,
@ -327,7 +327,7 @@ export {
getVideoFileFPS,
ffprobePromise,
getClosestFramerateStandard,
computeResolutionsToTranscode,
computeLowerResolutionsToTranscode,
getVideoFileBitrate,
canDoQuickTranscode,
canDoQuickVideoTranscode,

2
server/helpers/webtorrent.ts
View File

@ -100,7 +100,7 @@ function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlayli
urlList: buildUrlList(video, videoFile)
}
return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => {
return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), async videoPath => {
const torrentContent = await createTorrentPromise(videoPath, options)
const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)

7
server/lib/hls.ts
View File

@ -37,7 +37,7 @@ async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlayl
for (const file of playlist.VideoFiles) {
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, async videoFilePath => {
await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
const size = await getVideoStreamSize(videoFilePath)
const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
@ -69,10 +69,11 @@ async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingP
// For all the resolutions available for this video
for (const file of playlist.VideoFiles) {
const rangeHashes: { [range: string]: string } = {}
const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
await VideoPathManager.Instance.makeAvailableVideoFile(playlist, file, videoPath => {
await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(playlist, file, async resolutionPlaylistPath => {
return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
const playlistContent = await readFile(resolutionPlaylistPath)
const ranges = getRangesFromPlaylist(playlistContent.toString())

11
server/lib/job-queue/handlers/move-to-object-storage.ts
View File

@ -56,16 +56,17 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
async function moveHLSFiles (video: MVideoWithAllFiles) {
for (const playlist of video.VideoStreamingPlaylists) {
const playlistWithVideo = playlist.withVideo(video)
for (const file of playlist.VideoFiles) {
if (file.storage !== VideoStorage.FILE_SYSTEM) continue
// Resolution playlist
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
await storeHLSFile(playlist, video, playlistFilename)
await storeHLSFile(playlistWithVideo, playlistFilename)
// Resolution fragmented file
const fileUrl = await storeHLSFile(playlist, video, file.filename)
const fileUrl = await storeHLSFile(playlistWithVideo, file.filename)
const oldPath = join(getHLSDirectory(video), file.filename)
@ -78,10 +79,12 @@ async function doAfterLastJob (video: MVideoWithAllFiles, isNewVideo: boolean) {
for (const playlist of video.VideoStreamingPlaylists) {
if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
const playlistWithVideo = playlist.withVideo(video)
// Master playlist
playlist.playlistUrl = await storeHLSFile(playlist, video, playlist.playlistFilename)
playlist.playlistUrl = await storeHLSFile(playlistWithVideo, playlist.playlistFilename)
// Sha256 segments file
playlist.segmentsSha256Url = await storeHLSFile(playlist, video, playlist.segmentsSha256Filename)
playlist.segmentsSha256Url = await storeHLSFile(playlistWithVideo, playlist.segmentsSha256Filename)
playlist.storage = VideoStorage.OBJECT_STORAGE

10
server/lib/job-queue/handlers/video-transcoding.ts
View File

@ -14,7 +14,7 @@ import {
VideoTranscodingPayload
} from '../../../../shared'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { computeResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
import { computeLowerResolutionsToTranscode } from '../../../helpers/ffprobe-utils'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import { CONFIG } from '../../../initializers/config'
import { VideoModel } from '../../../models/video/video'
@ -81,7 +81,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
await VideoPathManager.Instance.makeAvailableVideoFile(videoOrStreamingPlaylist, videoFileInput, videoInputPath => {
await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
return generateHlsPlaylistResolution({
video,
videoInputPath,
@ -135,7 +135,7 @@ async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodi
// ---------------------------------------------------------------------------
async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) {
if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
// Remove webtorrent files if not enabled
for (const file of video.VideoFiles) {
await video.removeWebTorrentFileAndTorrent(file)
@ -232,6 +232,7 @@ async function createHlsJobIfEnabled (user: MUserId, payload: {
isPortraitMode: payload.isPortraitMode,
copyCodecs: payload.copyCodecs,
isMaxQuality: payload.isMaxQuality,
autoDeleteWebTorrentIfNeeded: true,
isNewVideo: payload.isNewVideo
}
@ -261,7 +262,7 @@ async function createLowerResolutionsJobs (options: {
const { video, user, videoFileResolution, isPortraitMode, isNewVideo, type } = options
// Create transcoding jobs if there are enabled resolutions
const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
const resolutionsEnabled = computeLowerResolutionsToTranscode(videoFileResolution, 'vod')
const resolutionCreated: string[] = []
for (const resolution of resolutionsEnabled) {
@ -288,6 +289,7 @@ async function createLowerResolutionsJobs (options: {
isPortraitMode,
copyCodecs: false,
isMaxQuality: false,
autoDeleteWebTorrentIfNeeded: true,
isNewVideo
}

4
server/lib/live/live-manager.ts
View File

@ -3,7 +3,7 @@ import { readFile } from 'fs-extra'
import { createServer, Server } from 'net'
import { createServer as createServerTLS, Server as ServerTLS } from 'tls'
import {
computeResolutionsToTranscode,
computeLowerResolutionsToTranscode,
ffprobePromise,
getVideoFileBitrate,
getVideoFileFPS,
@ -402,7 +402,7 @@ class LiveManager {
private buildAllResolutionsToTranscode (originResolution: number) {
const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
? computeResolutionsToTranscode(originResolution, 'live')
? computeLowerResolutionsToTranscode(originResolution, 'live')
: []
return resolutionsEnabled.concat([ originResolution ])

10
server/lib/object-storage/keys.ts
View File

@ -1,12 +1,12 @@
import { join } from 'path'
import { MStreamingPlaylist, MVideoUUID } from '@server/types/models'
import { MStreamingPlaylistVideo } from '@server/types/models'
function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
return join(generateHLSObjectBaseStorageKey(playlist, video), filename)
function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
return join(generateHLSObjectBaseStorageKey(playlist), filename)
}
function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) {
return join(playlist.getStringType(), video.uuid)
function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
return join(playlist.getStringType(), playlist.Video.uuid)
}
function generateWebTorrentObjectStorageKey (filename: string) {

16
server/lib/object-storage/videos.ts
View File

@ -1,17 +1,17 @@
import { join } from 'path'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { MStreamingPlaylist, MVideoFile, MVideoUUID } from '@server/types/models'
import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
import { getHLSDirectory } from '../paths'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
function storeHLSFile (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string) {
const baseHlsDirectory = getHLSDirectory(video)
function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string) {
const baseHlsDirectory = getHLSDirectory(playlist.Video)
return storeObject({
inputPath: join(baseHlsDirectory, filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, video, filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
})
}
@ -24,16 +24,16 @@ function storeWebTorrentFile (filename: string) {
})
}
function removeHLSObjectStorage (playlist: MStreamingPlaylist, video: MVideoUUID) {
return removePrefix(generateHLSObjectBaseStorageKey(playlist, video), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
}
async function makeHLSFileAvailable (playlist: MStreamingPlaylist, video: MVideoUUID, filename: string, destination: string) {
const key = generateHLSObjectStorageKey(playlist, video, filename)
async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
const key = generateHLSObjectStorageKey(playlist, filename)
logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())

2
server/lib/thumbnail.ts
View File

@ -115,7 +115,7 @@ function generateVideoMiniature (options: {
}) {
const { video, videoFile, type } = options
return VideoPathManager.Instance.makeAvailableVideoFile(video, videoFile, input => {
return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => {
const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
const thumbnailCreator = videoFile.isAudio()

6
server/lib/transcoding/video-transcoding.ts
View File

@ -35,7 +35,7 @@ function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVid
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async videoInputPath => {
return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
@ -81,7 +81,7 @@ function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: V
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const extname = '.mp4'
return VideoPathManager.Instance.makeAvailableVideoFile(video, video.getMaxQualityFile(), async videoInputPath => {
return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => {
const newVideoFile = new VideoFileModel({
resolution,
extname,
@ -134,7 +134,7 @@ function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolutio
const inputVideoFile = video.getMinQualityFile()
return VideoPathManager.Instance.makeAvailableVideoFile(video, inputVideoFile, async audioInputPath => {
return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
// If the user updates the video preview during transcoding

26
server/lib/video-path-manager.ts
View File

@ -3,7 +3,14 @@ import { extname, join } from 'path'
import { buildUUID } from '@server/helpers/uuid'
import { extractVideo } from '@server/helpers/video'
import { CONFIG } from '@server/initializers/config'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
import {
MStreamingPlaylistVideo,
MVideo,
MVideoFile,
MVideoFileStreamingPlaylistVideo,
MVideoFileVideo,
MVideoUUID
} from '@server/types/models'
import { VideoStorage } from '@shared/models'
import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
@ -43,10 +50,10 @@ class VideoPathManager {
return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
}
async makeAvailableVideoFile <T> (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
return this.makeAvailableFactory(
() => this.getFSVideoFileOutputPath(videoOrPlaylist, videoFile),
() => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile),
false,
cb
)
@ -55,10 +62,10 @@ class VideoPathManager {
const destination = this.buildTMPDestination(videoFile.filename)
if (videoFile.isHLS()) {
const video = extractVideo(videoOrPlaylist)
const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
return this.makeAvailableFactory(
() => makeHLSFileAvailable(videoOrPlaylist as MStreamingPlaylistVideo, video, videoFile.filename, destination),
() => makeHLSFileAvailable(playlist, videoFile.filename, destination),
true,
cb
)
@ -71,19 +78,20 @@ class VideoPathManager {
)
}
async makeAvailableResolutionPlaylistFile <T> (playlist: MStreamingPlaylistVideo, videoFile: MVideoFile, cb: MakeAvailableCB<T>) {
async makeAvailableResolutionPlaylistFile <T> (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
const filename = getHlsResolutionPlaylistFilename(videoFile.filename)
if (videoFile.storage === VideoStorage.FILE_SYSTEM) {
return this.makeAvailableFactory(
() => join(getHLSDirectory(playlist.Video), filename),
() => join(getHLSDirectory(videoFile.getVideo()), filename),
false,
cb
)
}
const playlist = videoFile.VideoStreamingPlaylist
return this.makeAvailableFactory(
() => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
() => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)),
true,
cb
)
@ -99,7 +107,7 @@ class VideoPathManager {
}
return this.makeAvailableFactory(
() => makeHLSFileAvailable(playlist, playlist.Video, filename, this.buildTMPDestination(filename)),
() => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)),
true,
cb
)

2
server/lib/video-state.ts
View File

@ -80,6 +80,8 @@ async function moveToExternalStorageState (video: MVideoFullLight, isNewVideo: b
}
function moveToFailedTranscodingState (video: MVideoFullLight) {
if (video.state === VideoState.TRANSCODING_FAILED) return
return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
}

2
server/lib/video.ts
View File

@ -105,7 +105,7 @@ async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoF
return addTranscodingJob(dataInput, jobOptions)
}
async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions) {
async function addTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) {
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: payload }, options)

1
server/middlewares/validators/videos/index.ts
View File

@ -9,4 +9,5 @@ export * from './video-ownership-changes'
export * from './video-watch'
export * from './video-rates'
export * from './video-shares'
export * from './video-transcoding'
export * from './videos'

21
server/middlewares/validators/videos/video-files.ts
View File

@ -1,6 +1,6 @@
import express from 'express'
import { MUser, MVideo } from '@server/types/models'
import { HttpStatusCode, UserRight } from '../../../../shared'
import { MVideo } from '@server/types/models'
import { HttpStatusCode } from '../../../../shared'
import { logger } from '../../../helpers/logger'
import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
@ -14,9 +14,7 @@ const videoFilesDeleteWebTorrentValidator = [
if (!await doesVideoExist(req.params.id, res)) return
const video = res.locals.videoAll
const user = res.locals.oauth.token.User
if (!checkUserCanDeleteFiles(user, res)) return
if (!checkLocalVideo(video, res)) return
if (!video.hasWebTorrentFiles()) {
@ -47,9 +45,7 @@ const videoFilesDeleteHLSValidator = [
if (!await doesVideoExist(req.params.id, res)) return
const video = res.locals.videoAll
const user = res.locals.oauth.token.User
if (!checkUserCanDeleteFiles(user, res)) return
if (!checkLocalVideo(video, res)) return
if (!video.getHLSPlaylist()) {
@ -89,16 +85,3 @@ function checkLocalVideo (video: MVideo, res: express.Response) {
return true
}
function checkUserCanDeleteFiles (user: MUser, res: express.Response) {
if (user.hasRight(UserRight.MANAGE_VIDEO_FILES) !== true) {
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'User cannot update video files'
})
return false
}
return true
}

55
server/middlewares/validators/videos/video-transcoding.ts Normal file
View File

@ -0,0 +1,55 @@
import express from 'express'
import { body } from 'express-validator'
import { isValidCreateTranscodingType } from '@server/helpers/custom-validators/video-transcoding'
import { CONFIG } from '@server/initializers/config'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { HttpStatusCode } from '@shared/models'
import { logger } from '../../../helpers/logger'
import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
const createTranscodingValidator = [
isValidVideoIdParam('videoId'),
body('transcodingType')
.custom(isValidCreateTranscodingType).withMessage('Should have a valid transcoding type'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking createTranscodingValidator parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.videoId, res, 'all')) return
const video = res.locals.videoAll
if (video.remote) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Cannot run transcoding job on a remote video'
})
}
if (CONFIG.TRANSCODING.ENABLED !== true) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Cannot run transcoding job because transcoding is disabled on this instance'
})
}
// Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state
const info = await VideoJobInfoModel.load(video.id)
if (info && info.pendingTranscode !== 0) {
return res.fail({
status: HttpStatusCode.CONFLICT_409,
message: 'This video is already being transcoded'
})
}
return next()
}
]
// ---------------------------------------------------------------------------
export {
createTranscodingValidator
}

3
server/models/video/formatter/video-format-utils.ts
View File

@ -2,8 +2,7 @@ import { uuidToShort } from '@server/helpers/uuid'
import { generateMagnetUri } from '@server/helpers/webtorrent'
import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls'
import { VideoViews } from '@server/lib/video-views'
import { VideosCommonQueryAfterSanitize } from '@shared/models'
import { VideoFile } from '@shared/models/videos/video-file.model'
import { VideoFile, VideosCommonQueryAfterSanitize } from '@shared/models'
import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects'
import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos'
import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model'

8
server/models/video/video-file.ts
View File

@ -25,7 +25,7 @@ import { logger } from '@server/helpers/logger'
import { extractVideo } from '@server/helpers/video'
import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
import { getFSTorrentFilePath } from '@server/lib/paths'
import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
import { AttributesOnly } from '@shared/core-utils'
import { VideoStorage } from '@shared/models'
import {
@ -536,4 +536,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
(this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId)
)
}
withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist })
return Object.assign(this, { Video: videoOrPlaylist })
}
}

2
server/models/video/video-job-info.ts
View File

@ -49,7 +49,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo
})
Video: VideoModel
static load (videoId: number, transaction: Transaction) {
static load (videoId: number, transaction?: Transaction) {
const where = {
videoId
}

4
server/models/video/video-streaming-playlist.ts
View File

@ -239,6 +239,10 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
this.videoId === other.videoId
}
withVideo (video: MVideo) {
return Object.assign(this, { Video: video })
}
private getMasterPlaylistStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
}

9
server/models/video/video.ts
View File

@ -33,9 +33,8 @@ import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { getServerActor } from '@server/models/application/application'
import { ModelCache } from '@server/models/model-cache'
import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
import { VideoInclude } from '@shared/models'
import { VideoFile } from '@shared/models/videos/video-file.model'
import { AttributesOnly, buildVideoEmbedPath, buildVideoWatchPath, isThisWeek, pick } from '@shared/core-utils'
import { VideoFile, VideoInclude } from '@shared/models'
import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
import { VideoObject } from '../../../shared/models/activitypub/objects'
import { Video, VideoDetails, VideoRateType, VideoStorage } from '../../../shared/models/videos'
@ -1673,7 +1672,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
const file = this.getMaxQualityFile()
const videoOrPlaylist = file.getVideoOrStreamingPlaylist()
return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, file, originalFilePath => {
return VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(videoOrPlaylist), originalFilePath => {
return getVideoFileResolution(originalFilePath)
})
}
@ -1742,7 +1741,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
)
if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
await removeHLSObjectStorage(streamingPlaylist, this)
await removeHLSObjectStorage(streamingPlaylist.withVideo(this))
}
}
}

1
server/tests/api/check-params/index.ts
View File

@ -15,6 +15,7 @@ import './plugins'
import './redundancy'
import './search'
import './services'
import './transcoding'
import './upload-quota'
import './user-notifications'
import './user-subscriptions'

104
server/tests/api/check-params/transcoding.ts Normal file
View File

@ -0,0 +1,104 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
import { HttpStatusCode, UserRole } from '@shared/models'
describe('Test transcoding API validators', function () {
let servers: PeerTubeServer[]
let userToken: string
let moderatorToken: string
let remoteId: string
let validId: string
// ---------------------------------------------------------------
before(async function () {
this.timeout(60000)
servers = await createMultipleServers(2)
await setAccessTokensToServers(servers)
await doubleFollow(servers[0], servers[1])
userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
{
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
remoteId = uuid
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
validId = uuid
}
await waitJobs(servers)
await servers[0].config.enableTranscoding()
})
it('Should not run transcoding of a unknown video', async function () {
await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'hls', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'webtorrent', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not run transcoding of a remote video', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'hls', expectedStatus })
await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'webtorrent', expectedStatus })
})
it('Should not run transcoding by a non admin user', async function () {
const expectedStatus = HttpStatusCode.FORBIDDEN_403
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', token: userToken, expectedStatus })
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', token: moderatorToken, expectedStatus })
})
it('Should not run transcoding without transcoding type', async function () {
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should not run transcoding with an incorrect transcoding type', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'toto' as any, expectedStatus })
})
it('Should not run transcoding if the instance disabled it', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].config.disableTranscoding()
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', expectedStatus })
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', expectedStatus })
})
it('Should run transcoding', async function () {
this.timeout(120_000)
await servers[0].config.enableTranscoding()
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls' })
await waitJobs(servers)
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent' })
await waitJobs(servers)
})
it('Should not run transcoding on a video that is already being transcoded', async function () {
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent' })
const expectedStatus = HttpStatusCode.CONFLICT_409
await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'webtorrent', expectedStatus })
})
after(async function () {
await cleanupTests(servers)
})
})

17
server/tests/api/check-params/video-files.ts
View File

@ -1,16 +1,19 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
import { HttpStatusCode, UserRole } from '@shared/models'
describe('Test videos files', function () {
let servers: PeerTubeServer[]
let webtorrentId: string
let hlsId: string
let remoteId: string
let userToken: string
let moderatorToken: string
let validId1: string
let validId2: string
@ -22,9 +25,16 @@ describe('Test videos files', function () {
servers = await createMultipleServers(2)
await setAccessTokensToServers(servers)
await doubleFollow(servers[0], servers[1])
userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
{
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
remoteId = uuid
}
{
await servers[0].config.enableTranscoding(true, true)
@ -58,6 +68,11 @@ describe('Test videos files', function () {
await waitJobs(servers)
})
it('Should not delete files of a unknown video', async function () {
await servers[0].videos.removeHLSFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeWebTorrentFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not delete files of a remote video', async function () {
await servers[0].videos.removeHLSFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeWebTorrentFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })

1
server/tests/api/videos/index.ts
View File

@ -6,6 +6,7 @@ import './video-captions'
import './video-change-ownership'
import './video-channels'
import './video-comments'
import './video-create-transcoding'
import './video-description'
import './video-files'
import './video-hls'

156
server/tests/api/videos/video-create-transcoding.ts Normal file
View File

@ -0,0 +1,156 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import {
areObjectStorageTestsDisabled,
cleanupTests,
createMultipleServers,
doubleFollow,
expectStartWith,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
} from '@shared/extra-utils'
import { HttpStatusCode, VideoDetails } from '@shared/models'
const expect = chai.expect
async function checkFilesInObjectStorage (video: VideoDetails) {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
}
}
async function expectNoFailedTranscodingJob (server: PeerTubeServer) {
const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' })
expect(data).to.have.lengthOf(0)
}
function runTests (objectStorage: boolean) {
let servers: PeerTubeServer[] = []
let videoUUID: string
let publishedAt: string
before(async function () {
this.timeout(120000)
const config = objectStorage
? ObjectStorageCommand.getDefaultConfig()
: {}
// Run server 2 to have transcoding enabled
servers = await createMultipleServers(2, config)
await setAccessTokensToServers(servers)
await servers[0].config.disableTranscoding()
await doubleFollow(servers[0], servers[1])
if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets()
const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
videoUUID = shortUUID
const video = await servers[0].videos.get({ id: videoUUID })
publishedAt = video.publishedAt as string
await servers[0].config.enableTranscoding()
await waitJobs(servers)
})
it('Should generate HLS', async function () {
this.timeout(60000)
await servers[0].videos.runTranscoding({
videoId: videoUUID,
transcodingType: 'hls'
})
await waitJobs(servers)
await expectNoFailedTranscodingJob(servers[0])
for (const server of servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
expect(videoDetails.files).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
}
})
it('Should generate WebTorrent', async function () {
this.timeout(60000)
await servers[0].videos.runTranscoding({
videoId: videoUUID,
transcodingType: 'webtorrent'
})
await waitJobs(servers)
for (const server of servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
expect(videoDetails.files).to.have.lengthOf(5)
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
}
})
it('Should generate WebTorrent from HLS only video', async function () {
this.timeout(60000)
await servers[0].videos.removeWebTorrentFiles({ videoId: videoUUID })
await waitJobs(servers)
await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' })
await waitJobs(servers)
for (const server of servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
expect(videoDetails.files).to.have.lengthOf(5)
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
if (objectStorage) await checkFilesInObjectStorage(videoDetails)
}
})
it('Should not have updated published at attributes', async function () {
const video = await servers[0].videos.get({ id: videoUUID })
expect(video.publishedAt).to.equal(publishedAt)
})
after(async function () {
await cleanupTests(servers)
})
}
describe('Test create transcoding jobs from API', function () {
describe('On filesystem', function () {
runTests(false)
})
describe('On object storage', function () {
if (areObjectStorageTestsDisabled()) return
runTests(true)
})
})

15
shared/extra-utils/server/jobs-command.ts
View File

@ -36,6 +36,21 @@ export class JobsCommand extends AbstractCommand {
})
}
listFailed (options: OverrideCommandOptions & {
jobType?: JobType
}) {
const path = this.buildJobsUrl('failed')
return this.getRequestBody<ResultList<Job>>({
...options,
path,
query: { start: 0, count: 50 },
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.OK_200
})
}
private buildJobsUrl (state?: JobState) {
let path = '/api/v1/jobs'

21
shared/extra-utils/videos/videos-command.ts
View File

@ -18,7 +18,8 @@ import {
VideoDetails,
VideoFileMetadata,
VideoPrivacy,
VideosCommonQuery
VideosCommonQuery,
VideoTranscodingCreate
} from '@shared/models'
import { buildAbsoluteFixturePath, wait } from '../miscs'
import { unwrapBody } from '../requests'
@ -630,6 +631,24 @@ export class VideosCommand extends AbstractCommand {
})
}
runTranscoding (options: OverrideCommandOptions & {
videoId: number | string
transcodingType: 'hls' | 'webtorrent'
}) {
const path = '/api/v1/videos/' + options.videoId + '/transcoding'
const fields: VideoTranscodingCreate = pick(options, [ 'transcodingType' ])
return this.postBodyRequest({
...options,
path,
fields,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
// ---------------------------------------------------------------------------
private buildListQuery (options: VideosCommonQuery) {

2
shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts
View File

@ -1,4 +1,4 @@
import { EncoderOptionsBuilder } from '../../../videos/video-transcoding.model'
import { EncoderOptionsBuilder } from '../../../videos/transcoding'
export interface PluginTranscodingManager {
addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder): boolean

4
shared/models/server/job.model.ts
View File

@ -1,5 +1,5 @@
import { ContextType } from '../activitypub/context'
import { VideoResolution } from '../videos/video-resolution.enum'
import { VideoResolution } from '../videos/file/video-resolution.enum'
import { SendEmailOptions } from './emailer.model'
export type JobState = 'active' | 'completed' | 'failed' | 'waiting' | 'delayed' | 'paused'
@ -106,6 +106,8 @@ export interface HLSTranscodingPayload extends BaseTranscodingPayload {
isPortraitMode?: boolean
resolution: VideoResolution
copyCodecs: boolean
autoDeleteWebTorrentIfNeeded: boolean
isMaxQuality: boolean
}

3
shared/models/users/user-right.enum.ts
View File

@ -40,5 +40,6 @@ export const enum UserRight {
MANAGE_VIDEOS_REDUNDANCIES,
MANAGE_VIDEO_FILES
MANAGE_VIDEO_FILES,
RUN_VIDEO_TRANSCODING
}

3
shared/models/videos/file/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './video-file-metadata.model'
export * from './video-file.model'
export * from './video-resolution.enum'

0
shared/models/videos/video-file-metadata.model.ts → shared/models/videos/file/video-file-metadata.model.ts
View File

2
shared/models/videos/video-file.model.ts → shared/models/videos/file/video-file.model.ts
View File

@ -1,4 +1,4 @@
import { VideoConstant } from './video-constant.model'
import { VideoConstant } from '../video-constant.model'
import { VideoFileMetadata } from './video-file-metadata.model'
import { VideoResolution } from './video-resolution.enum'

0
shared/models/videos/video-resolution.enum.ts → shared/models/videos/file/video-resolution.enum.ts
View File

9
shared/models/videos/index.ts
View File

@ -4,9 +4,11 @@ export * from './change-ownership'
export * from './channel'
export * from './comment'
export * from './live'
export * from './file'
export * from './import'
export * from './playlist'
export * from './rate'
export * from './transcoding'
export * from './nsfw-policy.type'
@ -15,14 +17,10 @@ export * from './thumbnail.type'
export * from './video-constant.model'
export * from './video-create.model'
export * from './video-file-metadata.model'
export * from './video-file.model'
export * from './video-privacy.enum'
export * from './video-filter.type'
export * from './video-include.enum'
export * from './video-rate.type'
export * from './video-resolution.enum'
export * from './video-schedule-update.model'
export * from './video-sort-field.type'
@ -32,9 +30,6 @@ export * from './video-storage.enum'
export * from './video-streaming-playlist.model'
export * from './video-streaming-playlist.type'
export * from './video-transcoding.model'
export * from './video-transcoding-fps.model'
export * from './video-update.model'
export * from './video.model'
export * from './video-create-result.model'

3
shared/models/videos/transcoding/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './video-transcoding-create.model'
export * from './video-transcoding-fps.model'
export * from './video-transcoding.model'

3
shared/models/videos/transcoding/video-transcoding-create.model.ts Normal file
View File

@ -0,0 +1,3 @@
export interface VideoTranscodingCreate {
transcodingType: 'hls' | 'webtorrent'
}

0
shared/models/videos/video-transcoding-fps.model.ts → shared/models/videos/transcoding/video-transcoding-fps.model.ts
View File

2
shared/models/videos/video-transcoding.model.ts → shared/models/videos/transcoding/video-transcoding.model.ts
View File

@ -1,4 +1,4 @@
import { VideoResolution } from './video-resolution.enum'
import { VideoResolution } from '../file/video-resolution.enum'
// Types used by plugins and ffmpeg-utils

2
shared/models/videos/video-streaming-playlist.model.ts
View File

@ -1,5 +1,5 @@
import { VideoStreamingPlaylistType } from './video-streaming-playlist.type'
import { VideoFile } from './video-file.model'
import { VideoFile } from './file'
export interface VideoStreamingPlaylist {
id: number

2
shared/models/videos/video.model.ts
View File

@ -1,7 +1,7 @@
import { Account, AccountSummary } from '../actors'
import { VideoChannel, VideoChannelSummary } from './channel/video-channel.model'
import { VideoFile } from './file'
import { VideoConstant } from './video-constant.model'
import { VideoFile } from './video-file.model'
import { VideoPrivacy } from './video-privacy.enum'
import { VideoScheduleUpdate } from './video-schedule-update.model'
import { VideoState } from './video-state.enum'

69
support/doc/api/openapi.yaml
View File

@ -267,6 +267,10 @@ tags:
description: Like/dislike a video.
- name: Video Playlists
description: Operations dealing with playlists of videos. Playlists are bound to users and/or channels.
- name: Video Files
description: Operations on video files
- name: Video Transcoding
description: Video transcoding related operations
- name: Feeds
description: Server syndication feeds
- name: Search
@ -309,6 +313,8 @@ x-tagGroups:
- Video Playlists
- Video Ownership Change
- Video Mirroring
- Video Files
- Video Transcoding
- Live Videos
- Feeds
- name: Search
@ -3568,6 +3574,69 @@ paths:
'404':
description: video does not exist
'/videos/{id}/hls':
delete:
summary: Delete video HLS files
security:
- OAuth2:
- admin
tags:
- Video Files
operationId: delVideoHLS
parameters:
- $ref: '#/components/parameters/idOrUUID'
responses:
'204':
description: successful operation
'404':
description: video does not exist
'/videos/{id}/webtorrent':
delete:
summary: Delete video WebTorrent files
security:
- OAuth2:
- admin
tags:
- Video Files
operationId: delVideoWebTorrent
parameters:
- $ref: '#/components/parameters/idOrUUID'
responses:
'204':
description: successful operation
'404':
description: video does not exist
'/videos/{id}/transcoding':
post:
summary: Create a transcoding job
security:
- OAuth2:
- admin
tags:
- Video Transcoding
operationId: createVideoTranscoding
parameters:
- $ref: '#/components/parameters/idOrUUID'
requestBody:
content:
application/json:
schema:
type: object
properties:
transcodingType:
type: string
enum:
- hls
- webtorrent
required:
- transcodingType
responses:
'204':
description: successful operation
'404':
description: video does not exist
/search/videos:
get:
tags: