From 5c7d650827cc471a03e7fa18362bcbcbe5d30838 Mon Sep 17 00:00:00 2001 From: frankdelange Date: Fri, 1 Nov 2019 02:06:19 +0100 Subject: [PATCH] Add audio-only option to transcoders and player This patch adds an audio-only option to PeerTube by means of a new transcoding configuration which creates mp4 files which only contain an audio stream. This new transcoder has a resolution of '0' and is presented in the preferences and in the player resolution menu as 'Audio-only' (localised). When playing such streams the player shows the file thumbnail as background and disables controls autohide. Audio-only files can be shared and streamed just like any other file. They can be downloaded as well, the resulting file will be an mp4 container with a single audio stream. This patch is a proof of concept to show the feasibility of 'true' audio-only support. There are better ways of doing this which also enable multiple audio streams for a given video stream (e.g. DASH) but as this would entail a fundamental change in the way PeerTube works it is a bridge too far for a simple proof of concept. --- .../edit-custom-config.component.ts | 4 ++ .../resolution-menu-button.ts | 2 +- .../player/webtorrent/webtorrent-plugin.ts | 16 +++++-- config/default.yaml | 1 + config/production.yaml.example | 1 + config/test.yaml | 1 + server/controllers/api/config.ts | 2 + server/helpers/ffmpeg-utils.ts | 45 +++++++++++++++--- server/initializers/config.ts | 1 + server/lib/video-transcoding.ts | 46 +++++++++++++++++-- server/middlewares/validators/config.ts | 1 + server/tests/api/check-params/config.ts | 1 + server/tests/api/server/config.ts | 1 + shared/extra-utils/server/config.ts | 1 + shared/models/server/custom-config.model.ts | 1 + shared/models/videos/video-resolution.enum.ts | 5 ++ 16 files changed, 115 insertions(+), 14 deletions(-) diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index 8411c4f4f..5f23c80a2 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -36,6 +36,10 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { super() this.resolutions = [ + { + id: '0p', + label: this.i18n('Audio-only') + }, { id: '240p', label: this.i18n('240p') diff --git a/client/src/assets/player/videojs-components/resolution-menu-button.ts b/client/src/assets/player/videojs-components/resolution-menu-button.ts index aeb48888f..445b14b2b 100644 --- a/client/src/assets/player/videojs-components/resolution-menu-button.ts +++ b/client/src/assets/player/videojs-components/resolution-menu-button.ts @@ -79,7 +79,7 @@ class ResolutionMenuButton extends MenuButton { this.player_, { id: d.id, - label: d.label, + label: d.id == 0 ? this.player .localize('Audio-only') : d.label, selected: d.selected, callback: data.qualitySwitchCallback }) diff --git a/client/src/assets/player/webtorrent/webtorrent-plugin.ts b/client/src/assets/player/webtorrent/webtorrent-plugin.ts index 4a0b38703..007fc58cc 100644 --- a/client/src/assets/player/webtorrent/webtorrent-plugin.ts +++ b/client/src/assets/player/webtorrent/webtorrent-plugin.ts @@ -181,20 +181,29 @@ class WebTorrentPlugin extends Plugin { const currentTime = this.player.currentTime() const isPaused = this.player.paused() - // Remove poster to have black background - this.playerElement.poster = '' - // Hide bigPlayButton if (!isPaused) { this.player.bigPlayButton.hide() } + // Audio-only (resolutionId == 0) gets special treatment + if (resolutionId > 0) { + // Hide poster to have black background + this.player.removeClass('vjs-playing-audio-only-content') + this.player.posterImage.hide() + } else { + // Audio-only: show poster, do not auto-hide controls + this.player.addClass('vjs-playing-audio-only-content') + this.player.posterImage.show() + } + const newVideoFile = this.videoFiles.find(f => f.resolution.id === resolutionId) const options = { forcePlay: false, delay, seek: currentTime + (delay / 1000) } + this.updateVideoFile(newVideoFile, options) } @@ -327,6 +336,7 @@ class WebTorrentPlugin extends Plugin { this.player.posterImage.show() this.player.removeClass('vjs-has-autoplay') this.player.removeClass('vjs-has-big-play-button-clicked') + this.player.removeClass('vjs-playing-audio-only-content') return done() }) diff --git a/config/default.yaml b/config/default.yaml index 9d102f760..07fd4d24f 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -203,6 +203,7 @@ transcoding: allow_audio_files: true threads: 1 resolutions: # Only created if the original video has a higher resolution, uses more storage! + 0p: false # audio-only (creates mp4 without video stream) 240p: false 360p: false 480p: false diff --git a/config/production.yaml.example b/config/production.yaml.example index 68ae22944..d7bbc39cf 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -217,6 +217,7 @@ transcoding: allow_audio_files: true threads: 1 resolutions: # Only created if the original video has a higher resolution, uses more storage! + 0p: false # audio-only (creates mp4 without video stream) 240p: false 360p: false 480p: false diff --git a/config/test.yaml b/config/test.yaml index 8843bb2dc..eedd28537 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -72,6 +72,7 @@ transcoding: allow_audio_files: false threads: 2 resolutions: + 0p: true 240p: true 360p: true 480p: true diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 70e8aa970..8a00f9835 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -300,6 +300,7 @@ function customConfig (): CustomConfig { allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES, threads: CONFIG.TRANSCODING.THREADS, resolutions: { + '0p': CONFIG.TRANSCODING.RESOLUTIONS[ '0p' ], '240p': CONFIG.TRANSCODING.RESOLUTIONS[ '240p' ], '360p': CONFIG.TRANSCODING.RESOLUTIONS[ '360p' ], '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], @@ -356,6 +357,7 @@ function convertCustomConfigBody (body: CustomConfig) { function keyConverter (k: string) { // Transcoding resolutions exception if (/^\d{3,4}p$/.exec(k)) return k + if (/^0p$/.exec(k)) return k return snakeCase(k) } diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 7a4ac0970..2d9ce2bfa 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -14,6 +14,7 @@ function computeResolutionsToTranscode (videoFileHeight: number) { // Put in the order we want to proceed jobs const resolutions = [ + VideoResolution.H_NOVIDEO, VideoResolution.H_480P, VideoResolution.H_360P, VideoResolution.H_720P, @@ -34,10 +35,15 @@ function computeResolutionsToTranscode (videoFileHeight: number) { async function getVideoFileSize (path: string) { const videoStream = await getVideoStreamFromFile(path) - return { - width: videoStream.width, - height: videoStream.height - } + return videoStream == null + ? { + width: 0, + height: 0 + } + : { + width: videoStream.width, + height: videoStream.height + } } async function getVideoFileResolution (path: string) { @@ -52,6 +58,10 @@ async function getVideoFileResolution (path: string) { async function getVideoFileFPS (path: string) { const videoStream = await getVideoStreamFromFile(path) + if (videoStream == null) { + return 0 + } + for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { const valuesText: string = videoStream[key] if (!valuesText) continue @@ -118,7 +128,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima } } -type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' +type TranscodeOptionsType = 'hls' | 'quick-transcode' | 'video' | 'merge-audio' | 'split-audio' interface BaseTranscodeOptions { type: TranscodeOptionsType @@ -149,7 +159,11 @@ interface MergeAudioTranscodeOptions extends BaseTranscodeOptions { audioPath: string } -type TranscodeOptions = HLSTranscodeOptions | VideoTranscodeOptions | MergeAudioTranscodeOptions | QuickTranscodeOptions +interface SplitAudioTranscodeOptions extends BaseTranscodeOptions { + type: 'split-audio' +} + +type TranscodeOptions = HLSTranscodeOptions | VideoTranscodeOptions | MergeAudioTranscodeOptions | SplitAudioTranscodeOptions | QuickTranscodeOptions function transcode (options: TranscodeOptions) { return new Promise(async (res, rej) => { @@ -163,6 +177,8 @@ function transcode (options: TranscodeOptions) { command = await buildHLSCommand(command, options) } else if (options.type === 'merge-audio') { command = await buildAudioMergeCommand(command, options) + } else if (options.type === 'split-audio') { + command = await buildAudioSplitCommand(command, options) } else { command = await buildx264Command(command, options) } @@ -198,6 +214,7 @@ async function canDoQuickTranscode (path: string): Promise { const resolution = await getVideoFileResolution(path) // check video params + if (videoStream == null) return false if (videoStream[ 'codec_name' ] !== 'h264') return false if (videoStream[ 'pix_fmt' ] !== 'yuv420p') return false if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false @@ -276,6 +293,12 @@ async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: M return command } +async function buildAudioSplitCommand (command: ffmpeg.FfmpegCommand, options: SplitAudioTranscodeOptions) { + command = await presetAudioSplit(command) + + return command +} + async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) { command = await presetCopy(command) @@ -327,7 +350,7 @@ function getVideoStreamFromFile (path: string) { if (err) return rej(err) const videoStream = metadata.streams.find(s => s.codec_type === 'video') - if (!videoStream) return rej(new Error('Cannot find video stream of ' + path)) + //if (!videoStream) return rej(new Error('Cannot find video stream of ' + path)) return res(videoStream) }) @@ -482,3 +505,11 @@ async function presetCopy (command: ffmpeg.FfmpegCommand): Promise { + return command + .format('mp4') + .audioCodec('copy') + .noVideo() +} diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 6d5d55487..c6e478f57 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -168,6 +168,7 @@ const CONFIG = { get ALLOW_AUDIO_FILES () { return config.get('transcoding.allow_audio_files') }, get THREADS () { return config.get('transcoding.threads') }, RESOLUTIONS: { + get '0p' () { return config.get('transcoding.resolutions.0p') }, get '240p' () { return config.get('transcoding.resolutions.240p') }, get '360p' () { return config.get('transcoding.resolutions.360p') }, get '480p' () { return config.get('transcoding.resolutions.480p') }, diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 9243d1742..9dd54837f 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -81,12 +81,52 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR const videoOutputPath = getVideoFilePath(video, newVideoFile) const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile)) + const transcodeOptions = resolution === VideoResolution.H_NOVIDEO + ? { + type: 'split-audio' as 'split-audio', + inputPath: videoInputPath, + outputPath: videoTranscodedPath, + resolution, + } + : { + type: 'video' as 'video', + inputPath: videoInputPath, + outputPath: videoTranscodedPath, + resolution, + isPortraitMode: isPortrait + } + + await transcode(transcodeOptions) + + return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) +} + +/** + * Extract audio into a separate audio-only mp4. + */ +async function splitAudioFile (video: MVideoWithFile) { + const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const extname = '.mp4' + const resolution = VideoResolution.H_NOVIDEO + + // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed + const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) + + const newVideoFile = new VideoFileModel({ + resolution, + extname, + size: 0, + videoId: video.id + }) + const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) + const videoTranscodedPath = join(transcodeDirectory, video.getVideoFilename(newVideoFile)) + const transcodeOptions = { - type: 'video' as 'video', + type: 'split-audio' as 'split-audio', inputPath: videoInputPath, outputPath: videoTranscodedPath, - resolution, - isPortraitMode: isPortrait + resolution } await transcode(transcodeOptions) diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 1db907f91..d86fa700b 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -37,6 +37,7 @@ const customConfigUpdateValidator = [ body('transcoding.enabled').isBoolean().withMessage('Should have a valid transcoding enabled boolean'), body('transcoding.allowAdditionalExtensions').isBoolean().withMessage('Should have a valid additional extensions boolean'), body('transcoding.threads').isInt().withMessage('Should have a valid transcoding threads number'), + body('transcoding.resolutions.0p').isBoolean().withMessage('Should have a valid transcoding 0p resolution enabled boolean'), body('transcoding.resolutions.240p').isBoolean().withMessage('Should have a valid transcoding 240p resolution enabled boolean'), body('transcoding.resolutions.360p').isBoolean().withMessage('Should have a valid transcoding 360p resolution enabled boolean'), body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 3c558d4ea..443fbcb60 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -85,6 +85,7 @@ describe('Test config API validators', function () { allowAudioFiles: true, threads: 1, resolutions: { + '0p': false, '240p': false, '360p': true, '480p': true, diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index a494858b3..cf99e5c0a 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -274,6 +274,7 @@ describe('Test config', function () { allowAudioFiles: true, threads: 1, resolutions: { + '0p': false, '240p': false, '360p': true, '480p': true, diff --git a/shared/extra-utils/server/config.ts b/shared/extra-utils/server/config.ts index ada173313..35b08477f 100644 --- a/shared/extra-utils/server/config.ts +++ b/shared/extra-utils/server/config.ts @@ -111,6 +111,7 @@ function updateCustomSubConfig (url: string, token: string, newConfig: DeepParti allowAudioFiles: true, threads: 1, resolutions: { + '0p': false, '240p': false, '360p': true, '480p': true, diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 97972b759..032b91a29 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -75,6 +75,7 @@ export interface CustomConfig { threads: number resolutions: { + '0p': boolean '240p': boolean '360p': boolean '480p': boolean diff --git a/shared/models/videos/video-resolution.enum.ts b/shared/models/videos/video-resolution.enum.ts index fa26fc3cc..dc53294f6 100644 --- a/shared/models/videos/video-resolution.enum.ts +++ b/shared/models/videos/video-resolution.enum.ts @@ -1,6 +1,7 @@ import { VideoTranscodingFPS } from './video-transcoding-fps.model' export enum VideoResolution { + H_NOVIDEO = 0, H_240P = 240, H_360P = 360, H_480P = 480, @@ -18,6 +19,10 @@ export enum VideoResolution { */ function getBaseBitrate (resolution: VideoResolution) { switch (resolution) { + case VideoResolution.H_NOVIDEO: + // audio-only + return 64 * 1000 + case VideoResolution.H_240P: // quality according to Google Live Encoder: 300 - 700 Kbps // Quality according to YouTube Video Info: 186 Kbps