forks/peertube
1
0
Fork 0
Code Releases Activity

Decrease time to cleanup resumable uploads

Browse source
This commit is contained in:
Chocobozzz 2021-12-03 14:00:40 +01:00
parent 390366472d
commit 6d472b4046
No known key found for this signature in database
GPG key ID: 583A612D890159BE
4 changed files with 21 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
server/controllers/api/videos/upload.ts
View file

@ -8,6 +8,7 @@ import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
import { generateWebTorrentVideoFilename } from '@server/lib/paths'
import { Redis } from '@server/lib/redis'
import { uploadx } from '@server/lib/uploadx'
import {
addMoveToObjectStorageJob,
addOptimizeOrMergeAudioJob,
@ -19,7 +20,6 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
import { buildNextVideoState } from '@server/lib/video-state'
import { openapiOperationDoc } from '@server/middlewares/doc'
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { Uploadx } from '@uploadx/core'
import { VideoCreate, VideoState } from '../../../../shared'
import { HttpStatusCode } from '../../../../shared/models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
@ -41,8 +41,8 @@ import {
authenticate,
videosAddLegacyValidator,
videosAddResumableInitValidator,
videosResumableUploadIdValidator,
videosAddResumableValidator
videosAddResumableValidator,
videosResumableUploadIdValidator
} from '../../../middlewares'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { VideoModel } from '../../../models/video/video'
@ -52,9 +52,6 @@ const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
const uploadRouter = express.Router()
const uploadx = new Uploadx({ directory: getResumableUploadPath() })
uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id
const reqVideoFileAdd = createReqFiles(
[ 'videofile', 'thumbnailfile', 'previewfile' ],
Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT),

2
server/initializers/constants.ts
View file

@ -223,7 +223,7 @@ const SCHEDULER_INTERVALS_MS = {
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 * 16 // 16 hours
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
}
// ---------------------------------------------------------------------------

35
server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
View file

@ -1,9 +1,7 @@
import { map } from 'bluebird'
import { readdir, remove, stat } from 'fs-extra'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { getResumableUploadPath } from '@server/helpers/upload'
import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants'
import { METAFILE_EXTNAME } from '@uploadx/core'
import { uploadx } from '../uploadx'
import { AbstractScheduler } from './abstract-scheduler'
const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
@ -22,36 +20,17 @@ export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
}
protected async internalExecute () {
const path = getResumableUploadPath()
const files = await readdir(path)
logger.debug('Removing dangling resumable uploads', lTags())
const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME))
if (metafiles.length === 0) return
logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags())
const now = new Date().getTime()
try {
await map(metafiles, metafile => {
return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs)
}, { concurrency: 5 })
// Remove files that were not updated since the last execution
await uploadx.storage.purge(now - this.lastExecutionTimeMs)
} catch (error) {
logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
} finally {
this.lastExecutionTimeMs = new Date().getTime()
}
}
private async deleteIfOlderThan (metafile: string, olderThan: number) {
const metafilePath = getResumableUploadPath(metafile)
const statResult = await stat(metafilePath)
// Delete uploads that started since a long time
if (statResult.ctimeMs < olderThan) {
await remove(metafilePath)
const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '')
await remove(datafile)
this.lastExecutionTimeMs = now
}
}

10
server/lib/uploadx.ts Normal file
View file

@ -0,0 +1,10 @@
import express from 'express'
import { getResumableUploadPath } from '@server/helpers/upload'
import { Uploadx } from '@uploadx/core'
const uploadx = new Uploadx({ directory: getResumableUploadPath() })
uploadx.getUserId = (_, res: express.Response) => res.locals.oauth?.token.user.id
export {
uploadx
}