forks/peertube
1
0
Fork 0
Code Releases Activity

Speed up overviews route

Browse source
This commit is contained in:
Chocobozzz 2018-09-14 11:52:23 +02:00
parent 2b62cccd75
commit 7348b1fd84
No known key found for this signature in database
GPG key ID: 583A612D890159BE
7 changed files with 80 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -111,6 +111,7 @@
"jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017",
"lodash": "^4.17.10",
"magnet-uri": "^5.1.4",
"memoizee": "^0.4.14",
"morgan": "^1.5.3",
"multer": "^1.1.0",
"nodemailer": "^4.4.2",
@ -158,6 +159,7 @@
"@types/lodash": "^4.14.64",
"@types/magnet-uri": "^5.1.1",
"@types/maildev": "^0.0.1",
"@types/memoizee": "^0.4.2",
"@types/mkdirp": "^0.5.1",
"@types/mocha": "^5.0.0",
"@types/morgan": "^1.7.32",

26
server/controllers/api/overviews.ts
View file

@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video'
import { asyncMiddleware } from '../../middlewares'
import { TagModel } from '../../models/video/tag'
import { VideosOverview } from '../../../shared/models/overviews'
import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
import { cacheRoute } from '../../middlewares/cache'
import * as memoizee from 'memoizee'
const overviewsRouter = express.Router()
@ -23,10 +24,17 @@ export { overviewsRouter }
// This endpoint could be quite long, but we cache it
async function getVideosOverview (req: express.Request, res: express.Response) {
const attributes = await buildSamples()
const [ categories, channels, tags ] = await Promise.all([
Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
])
const result: VideosOverview = {
categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
categories,
channels,
tags
}
// Cleanup our object
@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
return res.json(result)
}
async function buildSamples () {
const buildSamples = memoizee(async function () {
const [ categories, channels, tags ] = await Promise.all([
VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
@ -45,7 +53,7 @@ async function buildSamples () {
])
return { categories, channels, tags }
}
}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
async function getVideosByTag (tag: string, res: express.Response) {
const videos = await getVideos(res, { tagsOneOf: [ tag ] })
@ -84,14 +92,16 @@ async function getVideos (
res: express.Response,
where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
) {
const { data } = await VideoModel.listForApi(Object.assign({
const query = Object.assign({
start: 0,
count: 10,
sort: '-createdAt',
includeLocalVideos: true,
nsfw: buildNSFWFilter(res),
withFiles: false
}, where))
}, where)
const { data } = await VideoModel.listForApi(query, false)
return data.map(d => d.toFormattedJSON())
}

24
server/helpers/utils.ts
View file

@ -1,12 +1,12 @@
import { ResultList } from '../../shared'
import { CONFIG } from '../initializers'
import { ActorModel } from '../models/activitypub/actor'
import { ApplicationModel } from '../models/application/application'
import { pseudoRandomBytesPromise, sha256 } from './core-utils'
import { logger } from './logger'
import { join } from 'path'
import { Instance as ParseTorrent } from 'parse-torrent'
import { remove } from 'fs-extra'
import * as memoizee from 'memoizee'
function deleteFileAsync (path: string) {
remove(path)
@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
} as ResultList<U>
}
async function getServerActor () {
if (getServerActor.serverActor === undefined) {
const application = await ApplicationModel.load()
if (!application) throw Error('Could not load Application from database.')
const getServerActor = memoizee(async function () {
const application = await ApplicationModel.load()
if (!application) throw Error('Could not load Application from database.')
getServerActor.serverActor = application.Account.Actor
}
if (!getServerActor.serverActor) {
logger.error('Cannot load server actor.')
process.exit(0)
}
return Promise.resolve(getServerActor.serverActor)
}
namespace getServerActor {
export let serverActor: ActorModel
}
return application.Account.Actor
})
function generateVideoTmpPath (target: string | ParseTorrent) {
const id = typeof target === 'string' ? target : target.infoHash

5
server/initializers/constants.ts
View file

@ -592,6 +592,10 @@ const CACHE = {
}
}
const MEMOIZE_TTL = {
OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
}
const REDUNDANCY = {
VIDEOS: {
EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
@ -708,6 +712,7 @@ export {
VIDEO_ABUSE_STATES,
JOB_REQUEST_TIMEOUT,
USER_PASSWORD_RESET_LIFETIME,
MEMOIZE_TTL,
USER_EMAIL_VERIFY_LIFETIME,
IMAGE_MIMETYPE_EXT,
OVERVIEWS,

2
server/lib/schedulers/videos-redundancy-scheduler.ts
View file

@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
if (cache.strategy === 'recently-added') {
const minViews = (cache as RecentlyAddedStrategy).minViews
const minViews = cache.minViews
return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
}
}

27
server/models/video/video.ts
View file

@ -929,7 +929,7 @@ export class VideoModel extends Model<VideoModel> {
videoChannelId?: number,
actorId?: number
trendingDays?: number
}) {
}, countVideos = true) {
const query: IFindOptions<VideoModel> = {
offset: options.start,
limit: options.count,
@ -962,7 +962,7 @@ export class VideoModel extends Model<VideoModel> {
trendingDays
}
return VideoModel.getAvailableForApi(query, queryOptions)
return VideoModel.getAvailableForApi(query, queryOptions, countVideos)
}
static async searchAndPopulateAccountAndServer (options: {
@ -1164,7 +1164,14 @@ export class VideoModel extends Model<VideoModel> {
}
// threshold corresponds to how many video the field should have to be returned
static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
const actorId = (await getServerActor()).id
const scopeOptions = {
actorId,
includeLocalVideos: true
}
const query: IFindOptions<VideoModel> = {
attributes: [ field ],
limit: count,
@ -1172,17 +1179,11 @@ export class VideoModel extends Model<VideoModel> {
having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), {
[ Sequelize.Op.gte ]: threshold
}) as any, // FIXME: typings
where: {
[ field ]: {
[ Sequelize.Op.not ]: null
},
privacy: VideoPrivacy.PUBLIC,
state: VideoState.PUBLISHED
},
order: [ this.sequelize.random() ]
}
return VideoModel.findAll(query)
return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] })
.findAll(query)
.then(rows => rows.map(r => r[ field ]))
}
@ -1210,7 +1211,7 @@ export class VideoModel extends Model<VideoModel> {
return {}
}
private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions) {
private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions, countVideos = true) {
const idsScope = {
method: [
ScopeNames.AVAILABLE_FOR_LIST_IDS, options
@ -1227,7 +1228,7 @@ export class VideoModel extends Model<VideoModel> {
}
const [ count, rowsId ] = await Promise.all([
VideoModel.scope(countScope).count(countQuery),
countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
VideoModel.scope(idsScope).findAll(query)
])
const ids = rowsId.map(r => r.id)

38
yarn.lock
View file

@ -160,6 +160,10 @@
dependencies:
"@types/node" "*"
"@types/memoizee@^0.4.2":
version "0.4.2"
resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.2.tgz#a500158999a8144a9b46cf9a9fb49b15f1853573"
"@types/mime@*":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b"
@ -2058,7 +2062,7 @@ error@^7.0.0:
string-template "~0.2.1"
xtend "~4.0.0"
es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.9, es5-ext@~0.10.14:
es5-ext@^0.10.14, es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.9, es5-ext@~0.10.14, es5-ext@~0.10.2:
version "0.10.46"
resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.46.tgz#efd99f67c5a7ec789baa3daa7f79870388f7f572"
dependencies:
@ -2110,7 +2114,7 @@ es6-symbol@3.1.1, es6-symbol@^3.1.1, es6-symbol@~3.1.1:
d "1"
es5-ext "~0.10.14"
es6-weak-map@^2.0.1:
es6-weak-map@^2.0.1, es6-weak-map@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.2.tgz#5e3ab32251ffd1538a1f8e5ffa1357772f92d96f"
dependencies:
@ -2223,7 +2227,7 @@ etag@~1.8.1:
version "1.8.1"
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
event-emitter@~0.3.5:
event-emitter@^0.3.5, event-emitter@~0.3.5:
version "0.3.5"
resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39"
dependencies:
@ -3757,7 +3761,7 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4:
dependencies:
isobject "^3.0.1"
is-promise@^2.1.0:
is-promise@^2.1, is-promise@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa"
@ -4490,6 +4494,12 @@ lru-cache@4.1.x, lru-cache@^4.0.1:
pseudomap "^1.0.2"
yallist "^2.1.2"
lru-queue@0.1:
version "0.1.0"
resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
dependencies:
es5-ext "~0.10.2"
lru@^3.0.0, lru@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/lru/-/lru-3.1.0.tgz#ea7fb8546d83733396a13091d76cfeb4c06837d5"
@ -4594,6 +4604,19 @@ mem@^1.1.0:
dependencies:
mimic-fn "^1.0.0"
memoizee@^0.4.14:
version "0.4.14"
resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.14.tgz#07a00f204699f9a95c2d9e77218271c7cd610d57"
dependencies:
d "1"
es5-ext "^0.10.45"
es6-weak-map "^2.0.2"
event-emitter "^0.3.5"
is-promise "^2.1"
lru-queue "0.1"
next-tick "1"
timers-ext "^0.1.5"
memory-chunk-store@^1.2.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/memory-chunk-store/-/memory-chunk-store-1.3.0.tgz#ae99e7e3b58b52db43d49d94722930d39459d0c4"
@ -7201,6 +7224,13 @@ timed-out@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
timers-ext@^0.1.5:
version "0.1.5"
resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.5.tgz#77147dd4e76b660c2abb8785db96574cbbd12922"
dependencies:
es5-ext "~0.10.14"
next-tick "1"
tiny-lr@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.1.tgz#9fa547412f238fedb068ee295af8b682c98b2aab"