1
0
Fork 0

Cleanup utils helper

This commit is contained in:
Chocobozzz 2018-08-14 15:28:30 +02:00
parent 59c76ffa8f
commit 06215f15e0
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
20 changed files with 173 additions and 166 deletions

View File

@ -4,7 +4,7 @@ import { ServerConfig, UserRight } from '../../../shared'
import { About } from '../../../shared/models/server/about.model'
import { CustomConfig } from '../../../shared/models/server/custom-config.model'
import { unlinkPromise, writeFilePromise } from '../../helpers/core-utils'
import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/utils'
import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup'
import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers'
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
import { customConfigUpdateValidator } from '../../middlewares/validators/config'

View File

@ -1,9 +1,10 @@
import * as express from 'express'
import { getFormattedObjects, resetSequelizeInstance } from '../../helpers/utils'
import { getFormattedObjects } from '../../helpers/utils'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
authenticate, commonVideosFiltersValidator,
authenticate,
commonVideosFiltersValidator,
optionalAuthenticate,
paginationValidator,
setDefaultPagination,
@ -19,7 +20,7 @@ import { videosSortValidator } from '../../middlewares/validators'
import { sendUpdateActor } from '../../lib/activitypub/send'
import { VideoChannelCreate, VideoChannelUpdate } from '../../../shared'
import { createVideoChannel } from '../../lib/video-channel'
import { createReqFiles, buildNSFWFilter } from '../../helpers/express-utils'
import { buildNSFWFilter, createReqFiles } from '../../helpers/express-utils'
import { setAsyncActorKeys } from '../../lib/activitypub'
import { AccountModel } from '../../models/account/account'
import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers'
@ -28,6 +29,7 @@ import { VideoModel } from '../../models/video/video'
import { updateAvatarValidator } from '../../middlewares/validators/avatar'
import { updateActorAvatarFile } from '../../lib/avatar'
import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger'
import { resetSequelizeInstance } from '../../helpers/database-utils'
const auditLogger = auditLoggerFactory('channels')
const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })

View File

@ -6,7 +6,7 @@ import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg
import { processImage } from '../../../helpers/image-utils'
import { logger } from '../../../helpers/logger'
import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger'
import { getFormattedObjects, getServerActor, resetSequelizeInstance } from '../../../helpers/utils'
import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
import {
CONFIG,
IMAGE_MIMETYPE_EXT,
@ -51,10 +51,11 @@ import { blacklistRouter } from './blacklist'
import { videoCommentRouter } from './comment'
import { rateVideoRouter } from './rate'
import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils'
import { buildNSFWFilter, createReqFiles } from '../../../helpers/express-utils'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { videoCaptionsRouter } from './captions'
import { videoImportsRouter } from './import'
import { resetSequelizeInstance } from '../../../helpers/database-utils'
const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()

View File

@ -14,6 +14,35 @@ import * as rimraf from 'rimraf'
import { URL } from 'url'
import { truncate } from 'lodash'
const timeTable = {
ms: 1,
second: 1000,
minute: 60000,
hour: 3600000,
day: 3600000 * 24,
week: 3600000 * 24 * 7,
month: 3600000 * 24 * 30
}
export function parseDuration (duration: number | string): number {
if (typeof duration === 'number') return duration
if (typeof duration === 'string') {
const split = duration.match(/^([\d\.,]+)\s?(\w+)$/)
if (split.length === 3) {
const len = parseFloat(split[1])
let unit = split[2].replace(/s$/i,'').toLowerCase()
if (unit === 'm') {
unit = 'ms'
}
return (len || 1) * (timeTable[unit] || 0)
}
}
throw new Error('Duration could not be properly parsed')
}
function sanitizeUrl (url: string) {
const urlObject = new URL(url)

View File

@ -1,6 +1,6 @@
import * as retry from 'async/retry'
import * as Bluebird from 'bluebird'
import { Model, Sequelize } from 'sequelize-typescript'
import { Model } from 'sequelize-typescript'
import { logger } from './logger'
function retryTransactionWrapper <T, A, B, C> (
@ -66,9 +66,17 @@ function updateInstanceWithAnother <T extends Model<T>> (instanceToUpdate: Model
}
}
function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
Object.keys(savedFields).forEach(key => {
const value = savedFields[key]
instance.set(key, value)
})
}
// ---------------------------------------------------------------------------
export {
resetSequelizeInstance,
retryTransactionWrapper,
transactionRetryer,
updateInstanceWithAnother

View File

@ -3,8 +3,9 @@ import * as multer from 'multer'
import { CONFIG, REMOTE_SCHEME } from '../initializers'
import { logger } from './logger'
import { User } from '../../shared/models/users'
import { generateRandomString } from './utils'
import { deleteFileAsync, generateRandomString } from './utils'
import { extname } from 'path'
import { isArray } from './custom-validators/misc'
function buildNSFWFilter (res: express.Response, paramNSFW?: string) {
if (paramNSFW === 'true') return true
@ -23,6 +24,24 @@ function buildNSFWFilter (res: express.Response, paramNSFW?: string) {
return null
}
function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[] }) {
const files = req.files
if (!files) return
if (isArray(files)) {
(files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
return
}
for (const key of Object.keys(files)) {
const file = files[ key ]
if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
else deleteFileAsync(file.path)
}
}
function getHostWithPort (host: string) {
const splitted = host.split(':')
@ -82,5 +101,6 @@ export {
buildNSFWFilter,
getHostWithPort,
badRequest,
createReqFiles
createReqFiles,
cleanUpReqFiles
}

View File

@ -7,6 +7,28 @@ import { processImage } from './image-utils'
import { logger } from './logger'
import { checkFFmpegEncoders } from '../initializers/checker'
function computeResolutionsToTranscode (videoFileHeight: number) {
const resolutionsEnabled: number[] = []
const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
// Put in the order we want to proceed jobs
const resolutions = [
VideoResolution.H_480P,
VideoResolution.H_360P,
VideoResolution.H_720P,
VideoResolution.H_240P,
VideoResolution.H_1080P
]
for (const resolution of resolutions) {
if (configResolutions[ resolution + 'p' ] === true && videoFileHeight > resolution) {
resolutionsEnabled.push(resolution)
}
}
return resolutionsEnabled
}
async function getVideoFileResolution (path: string) {
const videoStream = await getVideoFileStream(path)
@ -134,6 +156,7 @@ export {
generateImageFromVideoFile,
transcode,
getVideoFileFPS,
computeResolutionsToTranscode,
audio
}

59
server/helpers/signup.ts Normal file
View File

@ -0,0 +1,59 @@
import { CONFIG } from '../initializers'
import { UserModel } from '../models/account/user'
import * as ipaddr from 'ipaddr.js'
const isCidr = require('is-cidr')
async function isSignupAllowed () {
if (CONFIG.SIGNUP.ENABLED === false) {
return false
}
// No limit and signup is enabled
if (CONFIG.SIGNUP.LIMIT === -1) {
return true
}
const totalUsers = await UserModel.countTotal()
return totalUsers < CONFIG.SIGNUP.LIMIT
}
function isSignupAllowedForCurrentIP (ip: string) {
const addr = ipaddr.parse(ip)
let excludeList = [ 'blacklist' ]
let matched = ''
// if there is a valid, non-empty whitelist, we exclude all unknown adresses too
if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) {
excludeList.push('unknown')
}
if (addr.kind() === 'ipv4') {
const addrV4 = ipaddr.IPv4.parse(ip)
const rangeList = {
whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr))
.map(cidr => ipaddr.IPv4.parseCIDR(cidr)),
blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr))
.map(cidr => ipaddr.IPv4.parseCIDR(cidr))
}
matched = ipaddr.subnetMatch(addrV4, rangeList, 'unknown')
} else if (addr.kind() === 'ipv6') {
const addrV6 = ipaddr.IPv6.parse(ip)
const rangeList = {
whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr))
.map(cidr => ipaddr.IPv6.parseCIDR(cidr)),
blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr))
.map(cidr => ipaddr.IPv6.parseCIDR(cidr))
}
matched = ipaddr.subnetMatch(addrV6, rangeList, 'unknown')
}
return !excludeList.includes(matched)
}
// ---------------------------------------------------------------------------
export {
isSignupAllowed,
isSignupAllowedForCurrentIP
}

View File

@ -1,37 +1,12 @@
import { Model } from 'sequelize-typescript'
import * as ipaddr from 'ipaddr.js'
import { ResultList } from '../../shared'
import { VideoResolution } from '../../shared/models/videos'
import { CONFIG } from '../initializers'
import { UserModel } from '../models/account/user'
import { ActorModel } from '../models/activitypub/actor'
import { ApplicationModel } from '../models/application/application'
import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils'
import { logger } from './logger'
import { isArray } from './custom-validators/misc'
import { join } from 'path'
import { Instance as ParseTorrent } from 'parse-torrent'
const isCidr = require('is-cidr')
function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[] }) {
const files = req.files
if (!files) return
if (isArray(files)) {
(files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
return
}
for (const key of Object.keys(files)) {
const file = files[key]
if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
else deleteFileAsync(file.path)
}
}
function deleteFileAsync (path: string) {
unlinkPromise(path)
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
@ -60,127 +35,23 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
} as ResultList<U>
}
async function isSignupAllowed () {
if (CONFIG.SIGNUP.ENABLED === false) {
return false
}
// No limit and signup is enabled
if (CONFIG.SIGNUP.LIMIT === -1) {
return true
}
const totalUsers = await UserModel.countTotal()
return totalUsers < CONFIG.SIGNUP.LIMIT
}
function isSignupAllowedForCurrentIP (ip: string) {
const addr = ipaddr.parse(ip)
let excludeList = [ 'blacklist' ]
let matched = ''
// if there is a valid, non-empty whitelist, we exclude all unknown adresses too
if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) {
excludeList.push('unknown')
}
if (addr.kind() === 'ipv4') {
const addrV4 = ipaddr.IPv4.parse(ip)
const rangeList = {
whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr))
.map(cidr => ipaddr.IPv4.parseCIDR(cidr)),
blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr))
.map(cidr => ipaddr.IPv4.parseCIDR(cidr))
}
matched = ipaddr.subnetMatch(addrV4, rangeList, 'unknown')
} else if (addr.kind() === 'ipv6') {
const addrV6 = ipaddr.IPv6.parse(ip)
const rangeList = {
whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr))
.map(cidr => ipaddr.IPv6.parseCIDR(cidr)),
blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr))
.map(cidr => ipaddr.IPv6.parseCIDR(cidr))
}
matched = ipaddr.subnetMatch(addrV6, rangeList, 'unknown')
}
return !excludeList.includes(matched)
}
function computeResolutionsToTranscode (videoFileHeight: number) {
const resolutionsEnabled: number[] = []
const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
// Put in the order we want to proceed jobs
const resolutions = [
VideoResolution.H_480P,
VideoResolution.H_360P,
VideoResolution.H_720P,
VideoResolution.H_240P,
VideoResolution.H_1080P
]
for (const resolution of resolutions) {
if (configResolutions[ resolution + 'p' ] === true && videoFileHeight > resolution) {
resolutionsEnabled.push(resolution)
}
}
return resolutionsEnabled
}
const timeTable = {
ms: 1,
second: 1000,
minute: 60000,
hour: 3600000,
day: 3600000 * 24,
week: 3600000 * 24 * 7,
month: 3600000 * 24 * 30
}
export function parseDuration (duration: number | string): number {
if (typeof duration === 'number') return duration
if (typeof duration === 'string') {
const split = duration.match(/^([\d\.,]+)\s?(\w+)$/)
if (split.length === 3) {
const len = parseFloat(split[1])
let unit = split[2].replace(/s$/i,'').toLowerCase()
if (unit === 'm') {
unit = 'ms'
}
return (len || 1) * (timeTable[unit] || 0)
}
}
throw new Error('Duration could not be properly parsed')
}
function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
Object.keys(savedFields).forEach(key => {
const value = savedFields[key]
instance.set(key, value)
})
}
let serverActor: ActorModel
async function getServerActor () {
if (serverActor === undefined) {
if (getServerActor.serverActor === undefined) {
const application = await ApplicationModel.load()
if (!application) throw Error('Could not load Application from database.')
serverActor = application.Account.Actor
getServerActor.serverActor = application.Account.Actor
}
if (!serverActor) {
if (!getServerActor.serverActor) {
logger.error('Cannot load server actor.')
process.exit(0)
}
return Promise.resolve(serverActor)
return Promise.resolve(getServerActor.serverActor)
}
namespace getServerActor {
export let serverActor: ActorModel
}
function generateVideoTmpPath (target: string | ParseTorrent) {
@ -194,21 +65,13 @@ function getSecureTorrentName (originalName: string) {
return sha256(originalName) + '.torrent'
}
type SortType = { sortModel: any, sortValue: string }
// ---------------------------------------------------------------------------
export {
cleanUpReqFiles,
deleteFileAsync,
generateRandomString,
getFormattedObjects,
isSignupAllowed,
getSecureTorrentName,
isSignupAllowedForCurrentIP,
computeResolutionsToTranscode,
resetSequelizeInstance,
getServerActor,
SortType,
generateVideoTmpPath
}

View File

@ -1,9 +1,8 @@
import * as Bluebird from 'bluebird'
import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
import { logger } from '../../../helpers/logger'
import { resetSequelizeInstance } from '../../../helpers/utils'
import { sequelizeTypescript } from '../../../initializers'
import { AccountModel } from '../../../models/account/account'
import { ActorModel } from '../../../models/activitypub/actor'

View File

@ -1,13 +1,13 @@
import * as Bull from 'bull'
import { VideoResolution, VideoState } from '../../../../shared'
import { logger } from '../../../helpers/logger'
import { computeResolutionsToTranscode } from '../../../helpers/utils'
import { VideoModel } from '../../../models/video/video'
import { JobQueue } from '../job-queue'
import { federateVideoIfNeeded } from '../../activitypub'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { sequelizeTypescript } from '../../../initializers'
import * as Bluebird from 'bluebird'
import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
export type VideoFilePayload = {
videoUUID: string

View File

@ -1,6 +1,6 @@
import * as express from 'express'
import * as AsyncLock from 'async-lock'
import { parseDuration } from '../helpers/utils'
import { parseDuration } from '../helpers/core-utils'
import { Redis } from '../lib/redis'
import { logger } from '../helpers/logger'

View File

@ -1,6 +1,6 @@
import * as express from 'express'
import 'express-validator'
import { SortType } from '../helpers/utils'
import { SortType } from '../models/utils'
function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) {
if (!req.query.sort) req.query.sort = '-createdAt'

View File

@ -4,7 +4,7 @@ import { isAvatarFile } from '../../helpers/custom-validators/users'
import { areValidationErrors } from './utils'
import { CONSTRAINTS_FIELDS } from '../../initializers'
import { logger } from '../../helpers/logger'
import { cleanUpReqFiles } from '../../helpers/utils'
import { cleanUpReqFiles } from '../../helpers/express-utils'
const updateAvatarValidator = [
body('avatarfile').custom((value, { req }) => isAvatarFile(req.files)).withMessage(

View File

@ -16,7 +16,7 @@ import {
} from '../../helpers/custom-validators/users'
import { isVideoExist } from '../../helpers/custom-validators/videos'
import { logger } from '../../helpers/logger'
import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/utils'
import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup'
import { Redis } from '../../lib/redis'
import { UserModel } from '../../models/account/user'
import { areValidationErrors } from './utils'

View File

@ -7,7 +7,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
import { UserRight } from '../../../shared'
import { logger } from '../../helpers/logger'
import { isVideoCaptionExist, isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
import { cleanUpReqFiles } from '../../helpers/utils'
import { cleanUpReqFiles } from '../../helpers/express-utils'
const addVideoCaptionValidator = [
param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),

View File

@ -5,7 +5,7 @@ import { logger } from '../../helpers/logger'
import { areValidationErrors } from './utils'
import { getCommonVideoAttributes } from './videos'
import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../helpers/custom-validators/video-imports'
import { cleanUpReqFiles } from '../../helpers/utils'
import { cleanUpReqFiles } from '../../helpers/express-utils'
import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos'
import { CONFIG } from '../../initializers/constants'
import { CONSTRAINTS_FIELDS } from '../../initializers'

View File

@ -34,7 +34,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
import { VideoShareModel } from '../../models/video/video-share'
import { authenticate } from '../oauth'
import { areValidationErrors } from './utils'
import { cleanUpReqFiles } from '../../helpers/utils'
import { cleanUpReqFiles } from '../../helpers/express-utils'
import { VideoModel } from '../../models/video/video'
import { UserModel } from '../../models/account/user'

View File

@ -1,6 +1,8 @@
// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
import { Sequelize } from 'sequelize-typescript'
type SortType = { sortModel: any, sortValue: string }
function getSort (value: string, lastSort: string[] = [ 'id', 'ASC' ]) {
let field: any
let direction: 'ASC' | 'DESC'
@ -54,6 +56,7 @@ function createSimilarityAttribute (col: string, value: string) {
// ---------------------------------------------------------------------------
export {
SortType,
getSort,
getSortOnModel,
createSimilarityAttribute,

View File

@ -4,15 +4,15 @@ import {
AllowNull,
BelongsTo,
Column,
CreatedAt, DataType,
CreatedAt,
DataType,
ForeignKey,
Is,
Model,
Table,
UpdatedAt
} from 'sequelize-typescript'
import { SortType } from '../../helpers/utils'
import { getSortOnModel, throwIfNotValid } from '../utils'
import { getSortOnModel, SortType, throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { isVideoBlacklistReasonValid } from '../../helpers/custom-validators/video-blacklist'
import { Emailer } from '../../lib/emailer'