1
0
Fork 0

Add ability to delete a specific video file

This commit is contained in:
Chocobozzz 2022-07-29 14:50:41 +02:00
parent 12d84abeca
commit 1bb4c9ab2e
No known key found for this signature in database
GPG key ID: 583A612D890159BE
23 changed files with 678 additions and 209 deletions

View file

@ -107,6 +107,11 @@
<ul> <ul>
<li *ngFor="let file of video.files"> <li *ngFor="let file of video.files">
{{ file.resolution.label }}: {{ file.size | bytes: 1 }} {{ file.resolution.label }}: {{ file.size | bytes: 1 }}
<my-global-icon
i18n-ngbTooltip ngbTooltip="Delete this file" iconName="delete" role="button"
(click)="removeVideoFile(video, file, 'webtorrent')"
></my-global-icon>
</li> </li>
</ul> </ul>
</div> </div>
@ -117,6 +122,11 @@
<ul> <ul>
<li *ngFor="let file of video.streamingPlaylists[0].files"> <li *ngFor="let file of video.streamingPlaylists[0].files">
{{ file.resolution.label }}: {{ file.size | bytes: 1 }} {{ file.resolution.label }}: {{ file.size | bytes: 1 }}
<my-global-icon
i18n-ngbTooltip ngbTooltip="Delete this file" iconName="delete" role="button"
(click)="removeVideoFile(video, file, 'hls')"
></my-global-icon>
</li> </li>
</ul> </ul>
</div> </div>

View file

@ -13,6 +13,13 @@ my-embed {
.video-info > div { .video-info > div {
display: flex; display: flex;
my-global-icon {
width: 16px;
margin-left: 3px;
position: relative;
top: -2px;
}
} }
.loading { .loading {

View file

@ -8,7 +8,7 @@ import { AdvancedInputFilter } from '@app/shared/shared-forms'
import { DropdownAction, Video, VideoService } from '@app/shared/shared-main' import { DropdownAction, Video, VideoService } from '@app/shared/shared-main'
import { VideoBlockComponent, VideoBlockService } from '@app/shared/shared-moderation' import { VideoBlockComponent, VideoBlockService } from '@app/shared/shared-moderation'
import { VideoActionsDisplayType } from '@app/shared/shared-video-miniature' import { VideoActionsDisplayType } from '@app/shared/shared-video-miniature'
import { UserRight, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models' import { UserRight, VideoFile, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models'
import { VideoAdminService } from './video-admin.service' import { VideoAdminService } from './video-admin.service'
@Component({ @Component({
@ -196,6 +196,22 @@ export class VideoListComponent extends RestTable implements OnInit {
}) })
} }
async removeVideoFile (video: Video, file: VideoFile, type: 'hls' | 'webtorrent') {
const message = $localize`Are you sure you want to delete this ${file.resolution.label} file?`
const res = await this.confirmService.confirm(message, $localize`Delete file`)
if (res === false) return
this.videoService.removeFile(video.uuid, file.id, type)
.subscribe({
next: () => {
this.notifier.success($localize`File removed.`)
this.reloadData()
},
error: err => this.notifier.error(err.message)
})
}
private async removeVideos (videos: Video[]) { private async removeVideos (videos: Video[]) {
const message = prepareIcu($localize`Are you sure you want to delete {count, plural, =1 {this video} other {these {count} videos}}?`)( const message = prepareIcu($localize`Are you sure you want to delete {count, plural, =1 {this video} other {these {count} videos}}?`)(
{ count: videos.length }, { count: videos.length },

View file

@ -305,6 +305,11 @@ export class VideoService {
) )
} }
removeFile (videoId: number | string, fileId: number, type: 'hls' | 'webtorrent') {
return this.authHttp.delete(VideoService.BASE_VIDEO_URL + '/' + videoId + '/' + type + '/' + fileId)
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
runTranscoding (videoIds: (number | string)[], type: 'hls' | 'webtorrent') { runTranscoding (videoIds: (number | string)[], type: 'hls' | 'webtorrent') {
const body: VideoTranscodingCreate = { transcodingType: type } const body: VideoTranscodingCreate = { transcodingType: type }

View file

@ -150,6 +150,7 @@
"node-media-server": "^2.1.4", "node-media-server": "^2.1.4",
"nodemailer": "^6.0.0", "nodemailer": "^6.0.0",
"opentelemetry-instrumentation-sequelize": "^0.29.0", "opentelemetry-instrumentation-sequelize": "^0.29.0",
"p-queue": "^6",
"parse-torrent": "^9.1.0", "parse-torrent": "^9.1.0",
"password-generator": "^2.0.2", "password-generator": "^2.0.2",
"pg": "^8.2.1", "pg": "^8.2.1",

View file

@ -2,6 +2,7 @@ import express from 'express'
import toInt from 'validator/lib/toInt' import toInt from 'validator/lib/toInt'
import { logger, loggerTagsFactory } from '@server/helpers/logger' import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { removeAllWebTorrentFiles, removeHLSFile, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
import { VideoFileModel } from '@server/models/video/video-file' import { VideoFileModel } from '@server/models/video/video-file'
import { HttpStatusCode, UserRight } from '@shared/models' import { HttpStatusCode, UserRight } from '@shared/models'
import { import {
@ -9,10 +10,13 @@ import {
authenticate, authenticate,
ensureUserHasRight, ensureUserHasRight,
videoFileMetadataGetValidator, videoFileMetadataGetValidator,
videoFilesDeleteHLSFileValidator,
videoFilesDeleteHLSValidator, videoFilesDeleteHLSValidator,
videoFilesDeleteWebTorrentFileValidator,
videoFilesDeleteWebTorrentValidator, videoFilesDeleteWebTorrentValidator,
videosGetValidator videosGetValidator
} from '../../../middlewares' } from '../../../middlewares'
import { updatePlaylistAfterFileChange } from '@server/lib/hls'
const lTags = loggerTagsFactory('api', 'video') const lTags = loggerTagsFactory('api', 'video')
const filesRouter = express.Router() const filesRouter = express.Router()
@ -27,14 +31,26 @@ filesRouter.delete('/:id/hls',
authenticate, authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
asyncMiddleware(videoFilesDeleteHLSValidator), asyncMiddleware(videoFilesDeleteHLSValidator),
asyncMiddleware(removeHLSPlaylist) asyncMiddleware(removeHLSPlaylistController)
)
filesRouter.delete('/:id/hls/:videoFileId',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
asyncMiddleware(videoFilesDeleteHLSFileValidator),
asyncMiddleware(removeHLSFileController)
) )
filesRouter.delete('/:id/webtorrent', filesRouter.delete('/:id/webtorrent',
authenticate, authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
asyncMiddleware(videoFilesDeleteWebTorrentValidator), asyncMiddleware(videoFilesDeleteWebTorrentValidator),
asyncMiddleware(removeWebTorrentFiles) asyncMiddleware(removeAllWebTorrentFilesController)
)
filesRouter.delete('/:id/webtorrent/:videoFileId',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES),
asyncMiddleware(videoFilesDeleteWebTorrentFileValidator),
asyncMiddleware(removeWebTorrentFileController)
) )
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -51,33 +67,53 @@ async function getVideoFileMetadata (req: express.Request, res: express.Response
return res.json(videoFile.metadata) return res.json(videoFile.metadata)
} }
async function removeHLSPlaylist (req: express.Request, res: express.Response) { // ---------------------------------------------------------------------------
async function removeHLSPlaylistController (req: express.Request, res: express.Response) {
const video = res.locals.videoAll const video = res.locals.videoAll
logger.info('Deleting HLS playlist of %s.', video.url, lTags(video.uuid)) logger.info('Deleting HLS playlist of %s.', video.url, lTags(video.uuid))
await removeHLSPlaylist(video)
const hls = video.getHLSPlaylist()
await video.removeStreamingPlaylistFiles(hls)
await hls.destroy()
video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id)
await federateVideoIfNeeded(video, false, undefined) await federateVideoIfNeeded(video, false, undefined)
return res.sendStatus(HttpStatusCode.NO_CONTENT_204) return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
} }
async function removeWebTorrentFiles (req: express.Request, res: express.Response) { async function removeHLSFileController (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const videoFileId = +req.params.videoFileId
logger.info('Deleting HLS file %d of %s.', videoFileId, video.url, lTags(video.uuid))
const playlist = await removeHLSFile(video, videoFileId)
if (playlist) await updatePlaylistAfterFileChange(video, playlist)
await federateVideoIfNeeded(video, false, undefined)
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
// ---------------------------------------------------------------------------
async function removeAllWebTorrentFilesController (req: express.Request, res: express.Response) {
const video = res.locals.videoAll const video = res.locals.videoAll
logger.info('Deleting WebTorrent files of %s.', video.url, lTags(video.uuid)) logger.info('Deleting WebTorrent files of %s.', video.url, lTags(video.uuid))
for (const file of video.VideoFiles) { await removeAllWebTorrentFiles(video)
await video.removeWebTorrentFileAndTorrent(file) await federateVideoIfNeeded(video, false, undefined)
await file.destroy()
} return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
video.VideoFiles = []
async function removeWebTorrentFileController (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const videoFileId = +req.params.videoFileId
logger.info('Deleting WebTorrent file %d of %s.', videoFileId, video.url, lTags(video.uuid))
await removeWebTorrentFile(video, videoFileId)
await federateVideoIfNeeded(video, false, undefined) await federateVideoIfNeeded(video, false, undefined)
return res.sendStatus(HttpStatusCode.NO_CONTENT_204) return res.sendStatus(HttpStatusCode.NO_CONTENT_204)

View file

@ -1,15 +1,15 @@
import { FfprobeData } from 'fluent-ffmpeg' import { FfprobeData } from 'fluent-ffmpeg'
import { getMaxBitrate } from '@shared/core-utils' import { getMaxBitrate } from '@shared/core-utils'
import { import {
buildFileMetadata,
ffprobePromise, ffprobePromise,
getAudioStream, getAudioStream,
getVideoStreamDuration,
getMaxAudioBitrate, getMaxAudioBitrate,
buildFileMetadata,
getVideoStreamBitrate,
getVideoStreamFPS,
getVideoStream, getVideoStream,
getVideoStreamBitrate,
getVideoStreamDimensionsInfo, getVideoStreamDimensionsInfo,
getVideoStreamDuration,
getVideoStreamFPS,
hasAudioStream hasAudioStream
} from '@shared/extra-utils/ffprobe' } from '@shared/extra-utils/ffprobe'
import { VideoResolution, VideoTranscodingFPS } from '@shared/models' import { VideoResolution, VideoTranscodingFPS } from '@shared/models'

View file

@ -1,4 +1,4 @@
import { Transaction } from 'sequelize/types' import { CreationAttributes, Transaction } from 'sequelize/types'
import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
import { logger, LoggerTagsFn } from '@server/helpers/logger' import { logger, LoggerTagsFn } from '@server/helpers/logger'
import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
@ -7,7 +7,15 @@ import { VideoCaptionModel } from '@server/models/video/video-caption'
import { VideoFileModel } from '@server/models/video/video-file' import { VideoFileModel } from '@server/models/video/video-file'
import { VideoLiveModel } from '@server/models/video/video-live' import { VideoLiveModel } from '@server/models/video/video-live'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { MStreamingPlaylistFilesVideo, MThumbnail, MVideoCaption, MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models' import {
MStreamingPlaylistFiles,
MStreamingPlaylistFilesVideo,
MThumbnail,
MVideoCaption,
MVideoFile,
MVideoFullLight,
MVideoThumbnail
} from '@server/types/models'
import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models'
import { getOrCreateAPActor } from '../../actors' import { getOrCreateAPActor } from '../../actors'
import { checkUrlsSameHost } from '../../url' import { checkUrlsSameHost } from '../../url'
@ -125,38 +133,39 @@ export abstract class APVideoAbstractBuilder {
// Remove video playlists that do not exist anymore // Remove video playlists that do not exist anymore
await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)
const oldPlaylists = video.VideoStreamingPlaylists
video.VideoStreamingPlaylists = [] video.VideoStreamingPlaylists = []
for (const playlistAttributes of streamingPlaylistAttributes) { for (const playlistAttributes of streamingPlaylistAttributes) {
const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
streamingPlaylistModel.Video = video streamingPlaylistModel.Video = video
await this.setStreamingPlaylistFiles(video, streamingPlaylistModel, playlistAttributes.tagAPObject, t) await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t)
video.VideoStreamingPlaylists.push(streamingPlaylistModel) video.VideoStreamingPlaylists.push(streamingPlaylistModel)
} }
} }
private async insertOrReplaceStreamingPlaylist (attributes: VideoStreamingPlaylistModel['_creationAttributes'], t: Transaction) { private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes<VideoStreamingPlaylistModel>, t: Transaction) {
const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t })
return streamingPlaylist as MStreamingPlaylistFilesVideo return streamingPlaylist as MStreamingPlaylistFilesVideo
} }
private getStreamingPlaylistFiles (video: MVideoFullLight, type: VideoStreamingPlaylistType) { private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType) {
const playlist = video.VideoStreamingPlaylists.find(s => s.type === type) const playlist = oldPlaylists.find(s => s.type === type)
if (!playlist) return [] if (!playlist) return []
return playlist.VideoFiles return playlist.VideoFiles
} }
private async setStreamingPlaylistFiles ( private async setStreamingPlaylistFiles (
video: MVideoFullLight, oldPlaylists: MStreamingPlaylistFiles[],
playlistModel: MStreamingPlaylistFilesVideo, playlistModel: MStreamingPlaylistFilesVideo,
tagObjects: ActivityTagObject[], tagObjects: ActivityTagObject[],
t: Transaction t: Transaction
) { ) {
const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(video, playlistModel.type) const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type)
const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))

View file

@ -1,7 +1,8 @@
import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
import { flatten, uniq } from 'lodash' import { flatten, uniq } from 'lodash'
import PQueue from 'p-queue'
import { basename, dirname, join } from 'path' import { basename, dirname, join } from 'path'
import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
import { sha256 } from '@shared/extra-utils' import { sha256 } from '@shared/extra-utils'
import { VideoStorage } from '@shared/models' import { VideoStorage } from '@shared/models'
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
@ -14,7 +15,7 @@ import { sequelizeTypescript } from '../initializers/database'
import { VideoFileModel } from '../models/video/video-file' import { VideoFileModel } from '../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
import { storeHLSFile } from './object-storage' import { storeHLSFile } from './object-storage'
import { getHlsResolutionPlaylistFilename } from './paths' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
import { VideoPathManager } from './video-path-manager' import { VideoPathManager } from './video-path-manager'
async function updateStreamingPlaylistsInfohashesIfNeeded () { async function updateStreamingPlaylistsInfohashesIfNeeded () {
@ -33,80 +34,123 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
} }
} }
async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) {
const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist)
playlistWithFiles = await updateSha256VODSegments(video, playlist)
for (const file of playlist.VideoFiles) { // Refresh playlist, operations can take some time
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)
playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
await playlistWithFiles.save()
await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { video.setHLSPlaylist(playlistWithFiles)
const size = await getVideoStreamDimensionsInfo(videoFilePath) }
const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) // ---------------------------------------------------------------------------
const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` // Avoid concurrency issues when updating streaming playlist files
if (file.fps) line += ',FRAME-RATE=' + file.fps const playlistFilesQueue = new PQueue({ concurrency: 1 })
const codecs = await Promise.all([ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
getVideoStreamCodec(videoFilePath), return playlistFilesQueue.add(async () => {
getAudioStreamCodec(videoFilePath) const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
])
line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
masterPlaylists.push(line) for (const file of playlist.VideoFiles) {
masterPlaylists.push(playlistFilename) const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
})
}
await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => { await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
const size = await getVideoStreamDimensionsInfo(videoFilePath)
const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
if (file.fps) line += ',FRAME-RATE=' + file.fps
const codecs = await Promise.all([
getVideoStreamCodec(videoFilePath),
getAudioStreamCodec(videoFilePath)
])
line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
masterPlaylists.push(line)
masterPlaylists.push(playlistFilename)
})
}
if (playlist.playlistFilename) {
await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename)
}
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
if (playlist.storage === VideoStorage.OBJECT_STORAGE) { if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath) playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename)
await remove(masterPlaylistPath)
} }
return playlist.save()
}) })
} }
async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { // ---------------------------------------------------------------------------
const json: { [filename: string]: { [range: string]: string } } = {}
// For all the resolutions available for this video async function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
for (const file of playlist.VideoFiles) { return playlistFilesQueue.add(async () => {
const rangeHashes: { [range: string]: string } = {} const json: { [filename: string]: { [range: string]: string } } = {}
const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { // For all the resolutions available for this video
const playlistContent = await readFile(resolutionPlaylistPath) for (const file of playlist.VideoFiles) {
const ranges = getRangesFromPlaylist(playlistContent.toString()) const rangeHashes: { [range: string]: string } = {}
const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
const fd = await open(videoPath, 'r') await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
for (const range of ranges) {
const buf = Buffer.alloc(range.length)
await read(fd, buf, 0, range.length, range.offset)
rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
} const playlistContent = await readFile(resolutionPlaylistPath)
await close(fd) const ranges = getRangesFromPlaylist(playlistContent.toString())
const videoFilename = file.filename const fd = await open(videoPath, 'r')
json[videoFilename] = rangeHashes for (const range of ranges) {
const buf = Buffer.alloc(range.length)
await read(fd, buf, 0, range.length, range.offset)
rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
}
await close(fd)
const videoFilename = file.filename
json[videoFilename] = rangeHashes
})
}) })
}) }
}
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) if (playlist.segmentsSha256Filename) {
await outputJSON(outputPath, json) await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename)
}
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
if (playlist.storage === VideoStorage.OBJECT_STORAGE) { const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
await storeHLSFile(playlist, playlist.segmentsSha256Filename) await outputJSON(outputPath, json)
await remove(outputPath)
} if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename)
await remove(outputPath)
}
return playlist.save()
})
} }
// ---------------------------------------------------------------------------
async function buildSha256Segment (segmentPath: string) { async function buildSha256Segment (segmentPath: string) {
const buf = await readFile(segmentPath) const buf = await readFile(segmentPath)
return sha256(buf) return sha256(buf)
@ -190,7 +234,8 @@ export {
updateSha256VODSegments, updateSha256VODSegments,
buildSha256Segment, buildSha256Segment,
downloadPlaylistSegments, downloadPlaylistSegments,
updateStreamingPlaylistsInfohashesIfNeeded updateStreamingPlaylistsInfohashesIfNeeded,
updatePlaylistAfterFileChange
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View file

@ -55,7 +55,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
if (currentVideoFile) { if (currentVideoFile) {
// Remove old file and old torrent // Remove old file and old torrent
await video.removeWebTorrentFileAndTorrent(currentVideoFile) await video.removeWebTorrentFile(currentVideoFile)
// Remove the old video file from the array // Remove the old video file from the array
video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)

View file

@ -9,6 +9,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles'
import { isAbleToUploadVideo } from '@server/lib/user' import { isAbleToUploadVideo } from '@server/lib/user'
import { addOptimizeOrMergeAudioJob } from '@server/lib/video' import { addOptimizeOrMergeAudioJob } from '@server/lib/video'
import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file'
import { VideoPathManager } from '@server/lib/video-path-manager' import { VideoPathManager } from '@server/lib/video-path-manager'
import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio'
import { UserModel } from '@server/models/user/user' import { UserModel } from '@server/models/user/user'
@ -27,12 +28,12 @@ import {
} from '@shared/extra-utils' } from '@shared/extra-utils'
import { import {
VideoStudioEditionPayload, VideoStudioEditionPayload,
VideoStudioTaskPayload, VideoStudioTask,
VideoStudioTaskCutPayload, VideoStudioTaskCutPayload,
VideoStudioTaskIntroPayload, VideoStudioTaskIntroPayload,
VideoStudioTaskOutroPayload, VideoStudioTaskOutroPayload,
VideoStudioTaskWatermarkPayload, VideoStudioTaskPayload,
VideoStudioTask VideoStudioTaskWatermarkPayload
} from '@shared/models' } from '@shared/models'
import { logger, loggerTagsFactory } from '../../../helpers/logger' import { logger, loggerTagsFactory } from '../../../helpers/logger'
@ -89,7 +90,6 @@ async function processVideoStudioEdition (job: Job) {
await move(editionResultPath, outputPath) await move(editionResultPath, outputPath)
await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
await removeAllFiles(video, newFile) await removeAllFiles(video, newFile)
await newFile.save() await newFile.save()
@ -197,18 +197,12 @@ async function buildNewFile (video: MVideoId, path: string) {
} }
async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) {
const hls = video.getHLSPlaylist() await removeHLSPlaylist(video)
if (hls) {
await video.removeStreamingPlaylistFiles(hls)
await hls.destroy()
}
for (const file of video.VideoFiles) { for (const file of video.VideoFiles) {
if (file.id === webTorrentFileException.id) continue if (file.id === webTorrentFileException.id) continue
await video.removeWebTorrentFileAndTorrent(file) await removeWebTorrentFile(video, file.id)
await file.destroy()
} }
} }

View file

@ -149,7 +149,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
// Remove webtorrent files if not enabled // Remove webtorrent files if not enabled
for (const file of video.VideoFiles) { for (const file of video.VideoFiles) {
await video.removeWebTorrentFileAndTorrent(file) await video.removeWebTorrentFile(file)
await file.destroy() await file.destroy()
} }

View file

@ -5,9 +5,8 @@ import { toEven } from '@server/helpers/core-utils'
import { retryTransactionWrapper } from '@server/helpers/database-utils' import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { sequelizeTypescript } from '@server/initializers/database' import { sequelizeTypescript } from '@server/initializers/database'
import { MStreamingPlaylistFilesVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { VideoResolution, VideoStorage } from '../../../shared/models/videos' import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { import {
buildFileMetadata, buildFileMetadata,
canDoQuickTranscode, canDoQuickTranscode,
@ -18,17 +17,10 @@ import {
TranscodeVODOptionsType TranscodeVODOptionsType
} from '../../helpers/ffmpeg' } from '../../helpers/ffmpeg'
import { CONFIG } from '../../initializers/config' import { CONFIG } from '../../initializers/config'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
import { VideoFileModel } from '../../models/video/video-file' import { VideoFileModel } from '../../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' import { updatePlaylistAfterFileChange } from '../hls'
import { import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths'
generateHLSMasterPlaylistFilename,
generateHlsSha256SegmentsFilename,
generateHLSVideoFilename,
generateWebTorrentVideoFilename,
getHlsResolutionPlaylistFilename
} from '../paths'
import { VideoPathManager } from '../video-path-manager' import { VideoPathManager } from '../video-path-manager'
import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
@ -260,7 +252,7 @@ async function onWebTorrentVideoFileTranscoding (
await createTorrentAndSetInfoHash(video, videoFile) await createTorrentAndSetInfoHash(video, videoFile)
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebTorrentFileAndTorrent(oldFile) if (oldFile) await video.removeWebTorrentFile(oldFile)
await VideoFileModel.customUpsert(videoFile, 'video', undefined) await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles') video.VideoFiles = await video.$get('VideoFiles')
@ -314,35 +306,15 @@ async function generateHlsPlaylistCommon (options: {
await transcodeVOD(transcodeOptions) await transcodeVOD(transcodeOptions)
// Create or update the playlist // Create or update the playlist
const { playlist, oldPlaylistFilename, oldSegmentsSha256Filename } = await retryTransactionWrapper(() => { const playlist = await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async transaction => { return sequelizeTypescript.transaction(async transaction => {
const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction)
const oldPlaylistFilename = playlist.playlistFilename
const oldSegmentsSha256Filename = playlist.segmentsSha256Filename
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
playlist.p2pMediaLoaderInfohashes = []
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
playlist.type = VideoStreamingPlaylistType.HLS
await playlist.save({ transaction })
return { playlist, oldPlaylistFilename, oldSegmentsSha256Filename }
}) })
}) })
if (oldPlaylistFilename) await video.removeStreamingPlaylistFile(playlist, oldPlaylistFilename)
if (oldSegmentsSha256Filename) await video.removeStreamingPlaylistFile(playlist, oldSegmentsSha256Filename)
// Build the new playlist file
const extname = extnameUtil(videoFilename)
const newVideoFile = new VideoFileModel({ const newVideoFile = new VideoFileModel({
resolution, resolution,
extname, extname: extnameUtil(videoFilename),
size: 0, size: 0,
filename: videoFilename, filename: videoFilename,
fps: -1, fps: -1,
@ -350,8 +322,6 @@ async function generateHlsPlaylistCommon (options: {
}) })
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
// Move files from tmp transcoded directory to the appropriate place
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
// Move playlist file // Move playlist file
@ -369,21 +339,14 @@ async function generateHlsPlaylistCommon (options: {
await createTorrentAndSetInfoHash(playlist, newVideoFile) await createTorrentAndSetInfoHash(playlist, newVideoFile)
const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution }) const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution })
if (oldFile) await video.removeStreamingPlaylistVideoFile(playlist, oldFile) if (oldFile) {
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
await oldFile.destroy()
}
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo await updatePlaylistAfterFileChange(video, playlist)
playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
playlist.storage = VideoStorage.FILE_SYSTEM
await playlist.save()
video.setHLSPlaylist(playlist)
await updateMasterHLSPlaylist(video, playlistWithFiles)
await updateSha256VODSegments(video, playlistWithFiles)
return { resolutionPlaylistPath, videoFile: savedVideoFile } return { resolutionPlaylistPath, videoFile: savedVideoFile }
} }

69
server/lib/video-file.ts Normal file
View file

@ -0,0 +1,69 @@
import { logger } from '@server/helpers/logger'
import { MVideoWithAllFiles } from '@server/types/models'
import { lTags } from './object-storage/shared'
async function removeHLSPlaylist (video: MVideoWithAllFiles) {
const hls = video.getHLSPlaylist()
if (!hls) return
await video.removeStreamingPlaylistFiles(hls)
await hls.destroy()
video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id)
}
async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) {
logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid))
const hls = video.getHLSPlaylist()
const files = hls.VideoFiles
if (files.length === 1) {
await removeHLSPlaylist(video)
return undefined
}
const toDelete = files.find(f => f.id === fileToDeleteId)
await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete)
await toDelete.destroy()
hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id)
return hls
}
// ---------------------------------------------------------------------------
async function removeAllWebTorrentFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
await video.removeWebTorrentFile(file)
await file.destroy()
}
video.VideoFiles = []
return video
}
async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId: number) {
const files = video.VideoFiles
if (files.length === 1) {
return removeAllWebTorrentFiles(video)
}
const toDelete = files.find(f => f.id === fileToDeleteId)
await video.removeWebTorrentFile(toDelete)
await toDelete.destroy()
video.VideoFiles = files.filter(f => f.id !== toDelete.id)
return video
}
export {
removeHLSPlaylist,
removeHLSFile,
removeAllWebTorrentFiles,
removeWebTorrentFile
}

View file

@ -3,6 +3,8 @@ import { MVideo } from '@server/types/models'
import { HttpStatusCode } from '@shared/models' import { HttpStatusCode } from '@shared/models'
import { logger } from '../../../helpers/logger' import { logger } from '../../../helpers/logger'
import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared'
import { isIdValid } from '@server/helpers/custom-validators/misc'
import { param } from 'express-validator'
const videoFilesDeleteWebTorrentValidator = [ const videoFilesDeleteWebTorrentValidator = [
isValidVideoIdParam('id'), isValidVideoIdParam('id'),
@ -35,6 +37,43 @@ const videoFilesDeleteWebTorrentValidator = [
} }
] ]
const videoFilesDeleteWebTorrentFileValidator = [
isValidVideoIdParam('id'),
param('videoFileId')
.custom(isIdValid).withMessage('Should have a valid file id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoFilesDeleteWebTorrentFile parameters', { parameters: req.params })
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.id, res)) return
const video = res.locals.videoAll
if (!checkLocalVideo(video, res)) return
const files = video.VideoFiles
if (!files.find(f => f.id === +req.params.videoFileId)) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
message: 'This video does not have this WebTorrent file id'
})
}
if (files.length === 1 && !video.getHLSPlaylist()) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Cannot delete WebTorrent files since this video does not have HLS playlist'
})
}
return next()
}
]
// ---------------------------------------------------------------------------
const videoFilesDeleteHLSValidator = [ const videoFilesDeleteHLSValidator = [
isValidVideoIdParam('id'), isValidVideoIdParam('id'),
@ -66,9 +105,55 @@ const videoFilesDeleteHLSValidator = [
} }
] ]
const videoFilesDeleteHLSFileValidator = [
isValidVideoIdParam('id'),
param('videoFileId')
.custom(isIdValid).withMessage('Should have a valid file id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoFilesDeleteHLSFile parameters', { parameters: req.params })
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.id, res)) return
const video = res.locals.videoAll
if (!checkLocalVideo(video, res)) return
if (!video.getHLSPlaylist()) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'This video does not have HLS files'
})
}
const hlsFiles = video.getHLSPlaylist().VideoFiles
if (!hlsFiles.find(f => f.id === +req.params.videoFileId)) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
message: 'This HLS playlist does not have this file id'
})
}
// Last file to delete
if (hlsFiles.length === 1 && !video.hasWebTorrentFiles()) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Cannot delete last HLS playlist file since this video does not have WebTorrent files'
})
}
return next()
}
]
export { export {
videoFilesDeleteWebTorrentValidator, videoFilesDeleteWebTorrentValidator,
videoFilesDeleteHLSValidator videoFilesDeleteWebTorrentFileValidator,
videoFilesDeleteHLSValidator,
videoFilesDeleteHLSFileValidator
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View file

@ -162,7 +162,7 @@ export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedu
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier) logger.info('Removing duplicated video file %s.', logIdentifier)
videoFile.Video.removeWebTorrentFileAndTorrent(videoFile, true) videoFile.Video.removeWebTorrentFile(videoFile, true)
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
} }

View file

@ -16,8 +16,9 @@ import {
UpdatedAt UpdatedAt
} from 'sequelize-typescript' } from 'sequelize-typescript'
import { getHLSPublicFileUrl } from '@server/lib/object-storage' import { getHLSPublicFileUrl } from '@server/lib/object-storage'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
import { VideoFileModel } from '@server/models/video/video-file' import { VideoFileModel } from '@server/models/video/video-file'
import { MStreamingPlaylist, MVideo } from '@server/types/models' import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
import { sha1 } from '@shared/extra-utils' import { sha1 } from '@shared/extra-utils'
import { VideoStorage } from '@shared/models' import { VideoStorage } from '@shared/models'
import { AttributesOnly } from '@shared/typescript-utils' import { AttributesOnly } from '@shared/typescript-utils'
@ -167,6 +168,22 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return VideoStreamingPlaylistModel.findAll(query) return VideoStreamingPlaylistModel.findAll(query)
} }
static loadWithVideoAndFiles (id: number) {
const options = {
include: [
{
model: VideoModel.unscoped(),
required: true
},
{
model: VideoFileModel.unscoped()
}
]
}
return VideoStreamingPlaylistModel.findByPk<MStreamingPlaylistFilesVideo>(id, options)
}
static loadWithVideo (id: number) { static loadWithVideo (id: number) {
const options = { const options = {
include: [ include: [
@ -194,9 +211,22 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
static async loadOrGenerate (video: MVideo, transaction?: Transaction) { static async loadOrGenerate (video: MVideo, transaction?: Transaction) {
let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id, transaction) let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id, transaction)
if (!playlist) playlist = new VideoStreamingPlaylistModel()
return Object.assign(playlist, { videoId: video.id, Video: video }) if (!playlist) {
playlist = new VideoStreamingPlaylistModel({
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
type: VideoStreamingPlaylistType.HLS,
storage: VideoStorage.FILE_SYSTEM,
p2pMediaLoaderInfohashes: [],
playlistFilename: generateHLSMasterPlaylistFilename(video.isLive),
segmentsSha256Filename: generateHlsSha256SegmentsFilename(video.isLive),
videoId: video.id
})
await playlist.save({ transaction })
}
return Object.assign(playlist, { Video: video })
} }
static doesOwnedHLSPlaylistExist (videoUUID: string) { static doesOwnedHLSPlaylistExist (videoUUID: string) {

View file

@ -28,7 +28,7 @@ import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation
import { LiveManager } from '@server/lib/live/live-manager' import { LiveManager } from '@server/lib/live/live-manager'
import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage'
import { tracer } from '@server/lib/opentelemetry/tracing' import { tracer } from '@server/lib/opentelemetry/tracing'
import { getHLSDirectory, getHLSRedundancyDirectory } from '@server/lib/paths' import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager' import { VideoPathManager } from '@server/lib/video-path-manager'
import { getServerActor } from '@server/models/application/application' import { getServerActor } from '@server/models/application/application'
import { ModelCache } from '@server/models/model-cache' import { ModelCache } from '@server/models/model-cache'
@ -769,7 +769,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
// Remove physical files and torrents // Remove physical files and torrents
instance.VideoFiles.forEach(file => { instance.VideoFiles.forEach(file => {
tasks.push(instance.removeWebTorrentFileAndTorrent(file)) tasks.push(instance.removeWebTorrentFile(file))
}) })
// Remove playlists file // Remove playlists file
@ -1783,7 +1783,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
.concat(toAdd) .concat(toAdd)
} }
removeWebTorrentFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { removeWebTorrentFile (videoFile: MVideoFile, isRedundancy = false) {
const filePath = isRedundancy const filePath = isRedundancy
? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile)
: VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile)
@ -1829,8 +1829,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
await videoFile.removeTorrent() await videoFile.removeTorrent()
await remove(filePath) await remove(filePath)
const resolutionFilename = getHlsResolutionPlaylistFilename(videoFile.filename)
await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename))
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename) await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), videoFile.filename)
await removeHLSFileObjectStorage(streamingPlaylist.withVideo(this), resolutionFilename)
} }
} }

View file

@ -24,6 +24,12 @@ describe('Test videos files', function () {
let validId1: string let validId1: string
let validId2: string let validId2: string
let hlsFileId: number
let webtorrentFileId: number
let remoteHLSFileId: number
let remoteWebtorrentFileId: number
// --------------------------------------------------------------- // ---------------------------------------------------------------
before(async function () { before(async function () {
@ -39,7 +45,12 @@ describe('Test videos files', function () {
{ {
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
remoteId = uuid await waitJobs(servers)
const video = await servers[1].videos.get({ id: uuid })
remoteId = video.uuid
remoteHLSFileId = video.streamingPlaylists[0].files[0].id
remoteWebtorrentFileId = video.files[0].id
} }
{ {
@ -47,7 +58,12 @@ describe('Test videos files', function () {
{ {
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
validId1 = uuid await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
validId1 = video.uuid
hlsFileId = video.streamingPlaylists[0].files[0].id
webtorrentFileId = video.files[0].id
} }
{ {
@ -76,43 +92,67 @@ describe('Test videos files', function () {
}) })
it('Should not delete files of a unknown video', async function () { it('Should not delete files of a unknown video', async function () {
await servers[0].videos.removeHLSFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) const expectedStatus = HttpStatusCode.NOT_FOUND_404
await servers[0].videos.removeWebTorrentFiles({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
})
it('Should not delete unknown files', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
}) })
it('Should not delete files of a remote video', async function () { it('Should not delete files of a remote video', async function () {
await servers[0].videos.removeHLSFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.removeWebTorrentFiles({ videoId: remoteId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
}) })
it('Should not delete files by a non admin user', async function () { it('Should not delete files by a non admin user', async function () {
const expectedStatus = HttpStatusCode.FORBIDDEN_403 const expectedStatus = HttpStatusCode.FORBIDDEN_403
await servers[0].videos.removeHLSFiles({ videoId: validId1, token: userToken, expectedStatus }) await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeHLSFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus }) await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
}) })
it('Should not delete files if the files are not available', async function () { it('Should not delete files if the files are not available', async function () {
await servers[0].videos.removeHLSFiles({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}) })
it('Should not delete files if no both versions are available', async function () { it('Should not delete files if no both versions are available', async function () {
await servers[0].videos.removeHLSFiles({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should not delete files if no both versions are available', async function () {
await servers[0].videos.removeHLSFiles({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
}) })
it('Should delete files if both versions are available', async function () { it('Should delete files if both versions are available', async function () {
await servers[0].videos.removeHLSFiles({ videoId: validId1 }) await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
await servers[0].videos.removeWebTorrentFiles({ videoId: validId2 }) await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
}) })
after(async function () { after(async function () {

View file

@ -122,7 +122,7 @@ function runTests (objectStorage: boolean) {
it('Should generate WebTorrent from HLS only video', async function () { it('Should generate WebTorrent from HLS only video', async function () {
this.timeout(60000) this.timeout(60000)
await servers[0].videos.removeWebTorrentFiles({ videoId: videoUUID }) await servers[0].videos.removeAllWebTorrentFiles({ videoId: videoUUID })
await waitJobs(servers) await waitJobs(servers)
await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' }) await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' })
@ -142,7 +142,7 @@ function runTests (objectStorage: boolean) {
it('Should only generate WebTorrent', async function () { it('Should only generate WebTorrent', async function () {
this.timeout(60000) this.timeout(60000)
await servers[0].videos.removeHLSFiles({ videoId: videoUUID }) await servers[0].videos.removeHLSPlaylist({ videoId: videoUUID })
await waitJobs(servers) await waitJobs(servers)
await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' }) await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'webtorrent' })

View file

@ -2,10 +2,12 @@
import 'mocha' import 'mocha'
import { expect } from 'chai' import { expect } from 'chai'
import { HttpStatusCode } from '@shared/models'
import { import {
cleanupTests, cleanupTests,
createMultipleServers, createMultipleServers,
doubleFollow, doubleFollow,
makeRawRequest,
PeerTubeServer, PeerTubeServer,
setAccessTokensToServers, setAccessTokensToServers,
waitJobs waitJobs
@ -13,8 +15,6 @@ import {
describe('Test videos files', function () { describe('Test videos files', function () {
let servers: PeerTubeServer[] let servers: PeerTubeServer[]
let validId1: string
let validId2: string
// --------------------------------------------------------------- // ---------------------------------------------------------------
@ -27,48 +27,160 @@ describe('Test videos files', function () {
await doubleFollow(servers[0], servers[1]) await doubleFollow(servers[0], servers[1])
await servers[0].config.enableTranscoding(true, true) await servers[0].config.enableTranscoding(true, true)
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
validId1 = uuid
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' })
validId2 = uuid
}
await waitJobs(servers)
}) })
it('Should delete webtorrent files', async function () { describe('When deleting all files', function () {
this.timeout(30_000) let validId1: string
let validId2: string
await servers[0].videos.removeWebTorrentFiles({ videoId: validId1 }) before(async function () {
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
validId1 = uuid
}
await waitJobs(servers) {
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' })
validId2 = uuid
}
for (const server of servers) { await waitJobs(servers)
const video = await server.videos.get({ id: validId1 }) })
expect(video.files).to.have.lengthOf(0) it('Should delete webtorrent files', async function () {
expect(video.streamingPlaylists).to.have.lengthOf(1) this.timeout(30_000)
}
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1 })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: validId1 })
expect(video.files).to.have.lengthOf(0)
expect(video.streamingPlaylists).to.have.lengthOf(1)
}
})
it('Should delete HLS files', async function () {
this.timeout(30_000)
await servers[0].videos.removeHLSPlaylist({ videoId: validId2 })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: validId2 })
expect(video.files).to.have.length.above(0)
expect(video.streamingPlaylists).to.have.lengthOf(0)
}
})
}) })
it('Should delete HLS files', async function () { describe('When deleting a specific file', function () {
this.timeout(30_000) let webtorrentId: string
let hlsId: string
await servers[0].videos.removeHLSFiles({ videoId: validId2 }) before(async function () {
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
webtorrentId = uuid
}
await waitJobs(servers) {
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}
for (const server of servers) { await waitJobs(servers)
const video = await server.videos.get({ id: validId2 }) })
expect(video.files).to.have.length.above(0) it('Shoulde delete a webtorrent file', async function () {
expect(video.streamingPlaylists).to.have.lengthOf(0) const video = await servers[0].videos.get({ id: webtorrentId })
} const files = video.files
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: files[0].id })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: webtorrentId })
expect(video.files).to.have.lengthOf(files.length - 1)
expect(video.files.find(f => f.id === files[0].id)).to.not.exist
}
})
it('Should delete all webtorrent files', async function () {
const video = await servers[0].videos.get({ id: webtorrentId })
const files = video.files
for (const file of files) {
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: file.id })
}
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: webtorrentId })
expect(video.files).to.have.lengthOf(0)
}
})
it('Should delete a hls file', async function () {
const video = await servers[0].videos.get({ id: hlsId })
const files = video.streamingPlaylists[0].files
const toDelete = files[0]
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: hlsId })
expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist
const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl)
expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
}
})
it('Should delete all hls files', async function () {
const video = await servers[0].videos.get({ id: hlsId })
const files = video.streamingPlaylists[0].files
for (const file of files) {
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id })
}
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: hlsId })
expect(video.streamingPlaylists).to.have.lengthOf(0)
}
})
it('Should not delete last file of a video', async function () {
const webtorrentOnly = await servers[0].videos.get({ id: hlsId })
const hlsOnly = await servers[0].videos.get({ id: webtorrentId })
for (let i = 0; i < 4; i++) {
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentOnly.id, fileId: webtorrentOnly.files[i].id })
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id })
}
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentOnly.id, fileId: webtorrentOnly.files[4].id, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus })
})
}) })
after(async function () { after(async function () {

View file

@ -20,10 +20,10 @@ import {
VideosCommonQuery, VideosCommonQuery,
VideoTranscodingCreate VideoTranscodingCreate
} from '@shared/models' } from '@shared/models'
import { VideoSource } from '@shared/models/videos/video-source'
import { unwrapBody } from '../requests' import { unwrapBody } from '../requests'
import { waitJobs } from '../server' import { waitJobs } from '../server'
import { AbstractCommand, OverrideCommandOptions } from '../shared' import { AbstractCommand, OverrideCommandOptions } from '../shared'
import { VideoSource } from '@shared/models/videos/video-source'
export type VideoEdit = Partial<Omit<VideoCreate, 'thumbnailfile' | 'previewfile'>> & { export type VideoEdit = Partial<Omit<VideoCreate, 'thumbnailfile' | 'previewfile'>> & {
fixture?: string fixture?: string
@ -605,7 +605,7 @@ export class VideosCommand extends AbstractCommand {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
removeHLSFiles (options: OverrideCommandOptions & { removeHLSPlaylist (options: OverrideCommandOptions & {
videoId: number | string videoId: number | string
}) { }) {
const path = '/api/v1/videos/' + options.videoId + '/hls' const path = '/api/v1/videos/' + options.videoId + '/hls'
@ -619,7 +619,22 @@ export class VideosCommand extends AbstractCommand {
}) })
} }
removeWebTorrentFiles (options: OverrideCommandOptions & { removeHLSFile (options: OverrideCommandOptions & {
videoId: number | string
fileId: number
}) {
const path = '/api/v1/videos/' + options.videoId + '/hls/' + options.fileId
return this.deleteRequest({
...options,
path,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
removeAllWebTorrentFiles (options: OverrideCommandOptions & {
videoId: number | string videoId: number | string
}) { }) {
const path = '/api/v1/videos/' + options.videoId + '/webtorrent' const path = '/api/v1/videos/' + options.videoId + '/webtorrent'
@ -633,6 +648,21 @@ export class VideosCommand extends AbstractCommand {
}) })
} }
removeWebTorrentFile (options: OverrideCommandOptions & {
videoId: number | string
fileId: number
}) {
const path = '/api/v1/videos/' + options.videoId + '/webtorrent/' + options.fileId
return this.deleteRequest({
...options,
path,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
runTranscoding (options: OverrideCommandOptions & { runTranscoding (options: OverrideCommandOptions & {
videoId: number | string videoId: number | string
transcodingType: 'hls' | 'webtorrent' transcodingType: 'hls' | 'webtorrent'

View file

@ -4635,6 +4635,11 @@ eventemitter-asyncresource@^1.0.0:
resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b"
integrity sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ== integrity sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ==
eventemitter3@^4.0.4:
version "4.0.7"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
events@3.3.0, events@^3.3.0: events@3.3.0, events@^3.3.0:
version "3.3.0" version "3.3.0"
resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
@ -7122,6 +7127,14 @@ p-map@^2.1.0:
resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
p-queue@^6:
version "6.6.2"
resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==
dependencies:
eventemitter3 "^4.0.4"
p-timeout "^3.2.0"
p-timeout@^3.0.0, p-timeout@^3.1.0, p-timeout@^3.2.0: p-timeout@^3.0.0, p-timeout@^3.1.0, p-timeout@^3.2.0:
version "3.2.0" version "3.2.0"
resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"