1
0
Fork 0

Add ability to limit videos history size

This commit is contained in:
Chocobozzz 2019-04-11 15:38:53 +02:00
parent 76062d9f96
commit 8f0bc73d7d
No known key found for this signature in database
GPG key ID: 583A612D890159BE
14 changed files with 116 additions and 17 deletions

View file

@ -111,6 +111,13 @@ tracker:
# Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers)
reject_too_many_announces: false
history:
videos:
# If you want to limit users videos history
# -1 means there is no limitations
# Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
max_age: -1
cache:
previews:
size: 500 # Max number of previews you want to cache

View file

@ -112,6 +112,12 @@ tracker:
# Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers)
reject_too_many_announces: false
history:
videos:
# If you want to limit users videos history
# -1 means there is no limitations
# Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
max_age: -1
###############################################################################
#

View file

@ -105,6 +105,7 @@ import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-
import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
import { PeerTubeSocket } from './server/lib/peertube-socket'
import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@ -240,6 +241,7 @@ async function startApplication () {
UpdateVideosScheduler.Instance.enable()
YoutubeDlUpdateScheduler.Instance.enable()
VideosRedundancyScheduler.Instance.enable()
RemoveOldHistoryScheduler.Instance.enable()
// Redis initialization
Redis.Instance.init()

View file

@ -48,7 +48,7 @@ async function removeUserHistory (req: express.Request, res: express.Response) {
const beforeDate = req.body.beforeDate || null
await sequelizeTypescript.transaction(t => {
return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t)
return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t)
})
// Do not send the delete to other instances, we delete OUR copy of this video abuse

View file

@ -40,7 +40,7 @@ const timeTable = {
month: 3600000 * 24 * 30
}
export function parseDuration (duration: number | string): number {
export function parseDurationToMs (duration: number | string): number {
if (typeof duration === 'number') return duration
if (typeof duration === 'string') {

View file

@ -5,7 +5,8 @@ import 'multer'
import * as validator from 'validator'
import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
import {
CONSTRAINTS_FIELDS, MIMETYPES,
CONSTRAINTS_FIELDS,
MIMETYPES,
VIDEO_CATEGORIES,
VIDEO_LICENCES,
VIDEO_PRIVACIES,

View file

@ -2,7 +2,7 @@ import { IConfig } from 'config'
import { dirname, join } from 'path'
import { VideosRedundancy } from '../../shared/models'
// Do not use barrels, remain constants as independent as possible
import { buildPath, parseBytes, parseDuration, root } from '../helpers/core-utils'
import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
import * as bytes from 'bytes'
@ -80,7 +80,7 @@ const CONFIG = {
},
REDUNDANCY: {
VIDEOS: {
CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')),
CHECK_INTERVAL: parseDurationToMs(config.get<string>('redundancy.videos.check_interval')),
STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
}
},
@ -94,6 +94,11 @@ const CONFIG = {
PRIVATE: config.get<boolean>('tracker.private'),
REJECT_TOO_MANY_ANNOUNCES: config.get<boolean>('tracker.reject_too_many_announces')
},
HISTORY: {
VIDEOS: {
MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
}
},
ADMIN: {
get EMAIL () { return config.get<string>('admin.email') }
},
@ -216,7 +221,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
return objs.map(obj => {
return Object.assign({}, obj, {
minLifetime: parseDuration(obj.min_lifetime),
minLifetime: parseDurationToMs(obj.min_lifetime),
size: bytes.parse(obj.size),
minViews: obj.min_views
})

View file

@ -158,12 +158,12 @@ const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
// 1 hour
let SCHEDULER_INTERVALS_MS = {
const SCHEDULER_INTERVALS_MS = {
actorFollowScores: 60000 * 60, // 1 hour
removeOldJobs: 60000 * 60, // 1 hour
updateVideos: 60000, // 1 minute
youtubeDLUpdate: 60000 * 60 * 24 // 1 day
youtubeDLUpdate: 60000 * 60 * 24, // 1 day
removeOldHistory: 60000 * 60 * 24 // 1 day
}
// ---------------------------------------------------------------------------
@ -591,6 +591,7 @@ if (isTestInstance() === true) {
SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
SCHEDULER_INTERVALS_MS.updateVideos = 5000
REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
@ -734,7 +735,7 @@ function buildVideosExtname () {
}
function loadLanguages () {
VIDEO_LANGUAGES = buildLanguages()
Object.assign(VIDEO_LANGUAGES, buildLanguages())
}
function buildLanguages () {

View file

@ -1,4 +1,5 @@
import { logger } from '../../helpers/logger'
import * as Bluebird from 'bluebird'
export abstract class AbstractScheduler {
@ -30,5 +31,5 @@ export abstract class AbstractScheduler {
}
}
protected abstract internalExecute (): Promise<any>
protected abstract internalExecute (): Promise<any> | Bluebird<any>
}

View file

@ -0,0 +1,32 @@
import { logger } from '../../helpers/logger'
import { AbstractScheduler } from './abstract-scheduler'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { UserVideoHistoryModel } from '../../models/account/user-video-history'
import { CONFIG } from '../../initializers/config'
import { isTestInstance } from '../../helpers/core-utils'
export class RemoveOldHistoryScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory
private constructor () {
super()
}
protected internalExecute () {
if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
logger.info('Removing old videos history.')
const now = new Date()
const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
return UserVideoHistoryModel.removeOldHistory(beforeDate)
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

View file

@ -1,6 +1,6 @@
import * as express from 'express'
import * as AsyncLock from 'async-lock'
import { parseDuration } from '../helpers/core-utils'
import { parseDurationToMs } from '../helpers/core-utils'
import { Redis } from '../lib/redis'
import { logger } from '../helpers/logger'
@ -24,7 +24,7 @@ function cacheRoute (lifetimeArg: string | number) {
res.send = (body) => {
if (res.statusCode >= 200 && res.statusCode < 400) {
const contentType = res.get('content-type')
const lifetime = parseDuration(lifetimeArg)
const lifetime = parseDurationToMs(lifetimeArg)
Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode)
.then(() => done())

View file

@ -67,7 +67,7 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
})
}
static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
static removeUserHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
const query: DestroyOptions = {
where: {
userId: user.id
@ -83,4 +83,16 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
return UserVideoHistoryModel.destroy(query)
}
static removeOldHistory (beforeDate: string) {
const query: DestroyOptions = {
where: {
updatedAt: {
[Op.lt]: beforeDate
}
}
}
return UserVideoHistoryModel.destroy(query)
}
}

View file

@ -7,14 +7,15 @@ import {
flushTests,
getVideosListWithToken,
getVideoWithToken,
killallServers,
killallServers, reRunServer,
runServer,
searchVideoWithToken,
ServerInfo,
setAccessTokensToServers,
updateMyUser,
uploadVideo,
userLogin
userLogin,
wait
} from '../../../../shared/utils'
import { Video, VideoDetails } from '../../../../shared/models/videos'
import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history'
@ -192,6 +193,35 @@ describe('Test videos history', function () {
expect(videos[1].name).to.equal('video 3')
})
it('Should not clean old history', async function () {
this.timeout(50000)
killallServers([ server ])
await reRunServer(server, { history: { videos: { max_age: '10 days' } } })
await wait(6000)
// Should still have history
const res = await listMyVideosHistory(server.url, server.accessToken)
expect(res.body.total).to.equal(2)
})
it('Should clean old history', async function () {
this.timeout(50000)
killallServers([ server ])
await reRunServer(server, { history: { videos: { max_age: '5 seconds' } } })
await wait(6000)
const res = await listMyVideosHistory(server.url, server.accessToken)
expect(res.body.total).to.equal(0)
})
after(async function () {
killallServers([ server ])

View file

@ -18,9 +18,11 @@ import {
} from '../'
import * as validator from 'validator'
import { VideoDetails, VideoPrivacy } from '../../models/videos'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
import { dateIsValid, webtorrentAdd } from '../miscs/miscs'
loadLanguages()
type VideoAttributes = {
name?: string
category?: number