forks/peertube
1
0
Fork 0
Code Releases Activity

Painfully debug concurrent import jobs

Browse source
This commit is contained in:
Chocobozzz 2021-02-09 11:22:42 +01:00
parent 80428d16a0
commit 44d1f7f2e8
No known key found for this signature in database
GPG key ID: 583A612D890159BE
3 changed files with 44 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
server/lib/job-queue/handlers/video-import.ts
View file

@ -1,11 +1,13 @@
import * as Bull from 'bull'
import { move, remove, stat } from 'fs-extra'
import { extname } from 'path'
import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { isPostImportVideoAccepted } from '@server/lib/moderation'
import { Hooks } from '@server/lib/plugins/hooks'
import { isAbleToUploadVideo } from '@server/lib/user'
import { addOptimizeOrMergeAudioJob } from '@server/lib/video'
import { getVideoFilePath } from '@server/lib/video-paths'
import { ThumbnailModel } from '@server/models/video/thumbnail'
import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
import {
VideoImportPayload,
@ -167,49 +169,66 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
// Process thumbnail
let thumbnailModel: MThumbnail
let thumbnailSave: object
if (options.generateThumbnail) {
thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
thumbnailSave = thumbnailModel.toJSON()
}
// Process preview
let previewModel: MThumbnail
let previewSave: object
if (options.generatePreview) {
previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
previewSave = previewModel.toJSON()
}
// Create torrent
await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => {
const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
const videoFileSave = videoFile.toJSON()
// Refresh video
const video = await VideoModel.load(videoImportToUpdate.videoId, t)
if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.')
const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async t => {
const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
const videoFileCreated = await videoFile.save({ transaction: t })
videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] })
// Refresh video
const video = await VideoModel.load(videoImportToUpdate.videoId, t)
if (!video) throw new Error('Video linked to import ' + videoImportToUpdate.videoId + ' does not exist anymore.')
// Update video DB object
video.duration = duration
video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
await video.save({ transaction: t })
const videoFileCreated = await videoFile.save({ transaction: t })
if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
// Update video DB object
video.duration = duration
video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
await video.save({ transaction: t })
// Now we can federate the video (reload from database, we need more attributes)
const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
await federateVideoIfNeeded(videoForFederation, true, t)
if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t)
if (previewModel) await video.addAndSaveThumbnail(previewModel, t)
// Update video import object
videoImportToUpdate.state = VideoImportState.SUCCESS
const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo
videoImportUpdated.Video = video
// Now we can federate the video (reload from database, we need more attributes)
const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
await federateVideoIfNeeded(videoForFederation, true, t)
logger.info('Video %s imported.', video.uuid)
// Update video import object
videoImportToUpdate.state = VideoImportState.SUCCESS
const videoImportUpdated = await videoImportToUpdate.save({ transaction: t }) as MVideoImportVideo
videoImportUpdated.Video = video
return { videoImportUpdated, video: videoForFederation }
videoImportToUpdate.Video = Object.assign(video, { VideoFiles: [ videoFileCreated ] })
logger.info('Video %s imported.', video.uuid)
return { videoImportUpdated, video: videoForFederation }
}).catch(err => {
// Reset fields
if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave)
if (previewModel) previewModel = new ThumbnailModel(previewSave)
videoFile = new VideoFileModel(videoFileSave)
throw err
})
})
Notifier.Instance.notifyOnFinishedVideoImport(videoImportUpdated, true)

3
server/models/video/video-import.ts
View file

@ -21,6 +21,7 @@ import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/cons
import { UserModel } from '../account/user'
import { getSort, throwIfNotValid } from '../utils'
import { ScopeNames as VideoModelScopeNames, VideoModel } from './video'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
@DefaultScope(() => ({
include: [
@ -113,7 +114,7 @@ export class VideoImportModel extends Model {
@AfterUpdate
static deleteVideoIfFailed (instance: VideoImportModel, options) {
if (instance.state === VideoImportState.FAILED) {
return instance.Video.destroy({ transaction: options.transaction })
return afterCommitIfTransaction(options.transaction, () => instance.Video.destroy())
}
return undefined

2
shared/extra-utils/videos/video-imports.ts
View file

@ -25,7 +25,7 @@ function getYoutubeHDRVideoUrl () {
* - 337 (2160p webm vp9.2 HDR)
* - 401 (2160p mp4 av01 HDR)
*/
return 'https://www.youtube.com/watch?v=MSJ25EqI19c'
return 'https://www.youtube.com/watch?v=qR5vOXbZsI4'
}
function getMagnetURI () {