diff --git a/server.ts b/server.ts index 9fe741175..df56bcd82 100644 --- a/server.ts +++ b/server.ts @@ -28,7 +28,7 @@ import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-be // Do not use barrels because we don't want to load all modules here (we need to initialize database first) import { logger } from './server/helpers/logger' -import { API_VERSION, CONFIG, CACHE } from './server/initializers/constants' +import { API_VERSION, CONFIG, FILES_CACHE } from './server/initializers/constants' const missed = checkMissedConfig() if (missed.length !== 0) { @@ -82,7 +82,7 @@ migrate() import { installApplication } from './server/initializers' import { Emailer } from './server/lib/emailer' import { JobQueue } from './server/lib/job-queue' -import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache' +import { VideosPreviewCache, VideosCaptionCache } from './server/lib/files-cache' import { activityPubRouter, apiRouter, @@ -218,8 +218,8 @@ async function startApplication () { ]) // Caches initializations - VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, CACHE.PREVIEWS.MAX_AGE) - VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, CACHE.VIDEO_CAPTIONS.MAX_AGE) + VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE) + VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE) // Enable Schedulers ActorFollowScheduler.Instance.enable() diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 7b14320e4..e65c7afd3 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -8,11 +8,10 @@ import { STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' -import { VideosPreviewCache } from '../lib/cache' +import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' import { cacheRoute } from '../middlewares/cache' import { asyncMiddleware, videosGetValidator } from '../middlewares' import { VideoModel } from '../models/video/video' -import { VideosCaptionCache } from '../lib/cache/videos-caption-cache' import { UserModel } from '../models/account/user' import { VideoCommentModel } from '../models/video/video-comment' import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo' diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 7fac8a4d6..7a3ec3874 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -660,7 +660,7 @@ const EMBED_SIZE = { } // Sub folders of cache directory -const CACHE = { +const FILES_CACHE = { PREVIEWS: { DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'), MAX_AGE: 1000 * 3600 * 3 // 3 hours @@ -671,6 +671,12 @@ const CACHE = { } } +const CACHE = { + USER_TOKENS: { + MAX_SIZE: 10000 + } +} + const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') @@ -741,7 +747,7 @@ if (isTestInstance() === true) { JOB_ATTEMPTS['email'] = 1 - CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 + FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' @@ -759,7 +765,7 @@ export { ACCEPT_HEADERS, BCRYPT_SALT_SIZE, TRACKER_RATE_LIMITS, - CACHE, + FILES_CACHE, CONFIG, CONSTRAINTS_FIELDS, EMBED_SIZE, @@ -799,6 +805,7 @@ export { VIDEO_TRANSCODING_FPS, FFMPEG_NICE, VIDEO_ABUSE_STATES, + CACHE, JOB_REQUEST_TIMEOUT, USER_PASSWORD_RESET_LIFETIME, MEMOIZE_TTL, diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index cd2c942fd..07af96b68 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker-after-init' -import { CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' +import { FILES_CACHE, CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' import { sequelizeTypescript } from './database' import { remove, ensureDir } from 'fs-extra' @@ -42,8 +42,8 @@ export { // --------------------------------------------------------------------------- function removeCacheAndTmpDirectories () { - const cacheDirectories = Object.keys(CACHE) - .map(k => CACHE[k].DIRECTORY) + const cacheDirectories = Object.keys(FILES_CACHE) + .map(k => FILES_CACHE[k].DIRECTORY) const tasks: Promise[] = [] @@ -60,8 +60,8 @@ function removeCacheAndTmpDirectories () { function createDirectoriesIfNotExist () { const storage = CONFIG.STORAGE - const cacheDirectories = Object.keys(CACHE) - .map(k => CACHE[k].DIRECTORY) + const cacheDirectories = Object.keys(FILES_CACHE) + .map(k => FILES_CACHE[k].DIRECTORY) const tasks: Promise[] = [] for (const key of Object.keys(storage)) { diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts similarity index 100% rename from server/lib/cache/abstract-video-static-file-cache.ts rename to server/lib/files-cache/abstract-video-static-file-cache.ts diff --git a/server/lib/cache/actor-follow-score-cache.ts b/server/lib/files-cache/actor-follow-score-cache.ts similarity index 100% rename from server/lib/cache/actor-follow-score-cache.ts rename to server/lib/files-cache/actor-follow-score-cache.ts diff --git a/server/lib/cache/index.ts b/server/lib/files-cache/index.ts similarity index 100% rename from server/lib/cache/index.ts rename to server/lib/files-cache/index.ts diff --git a/server/lib/cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts similarity index 91% rename from server/lib/cache/videos-caption-cache.ts rename to server/lib/files-cache/videos-caption-cache.ts index f240affbc..fe5b441af 100644 --- a/server/lib/cache/videos-caption-cache.ts +++ b/server/lib/files-cache/videos-caption-cache.ts @@ -1,5 +1,5 @@ import { join } from 'path' -import { CACHE, CONFIG } from '../../initializers' +import { FILES_CACHE, CONFIG } from '../../initializers' import { VideoModel } from '../../models/video/video' import { VideoCaptionModel } from '../../models/video/video-caption' import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' @@ -42,7 +42,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache { if (!video) return undefined const remoteStaticPath = videoCaption.getCaptionStaticPath() - const destPath = join(CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) + const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) } diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts similarity index 88% rename from server/lib/cache/videos-preview-cache.ts rename to server/lib/files-cache/videos-preview-cache.ts index a5d6f5b62..01cd3647e 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts @@ -1,5 +1,5 @@ import { join } from 'path' -import { CACHE, CONFIG, STATIC_PATHS } from '../../initializers' +import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers' import { VideoModel } from '../../models/video/video' import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' @@ -31,7 +31,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) - const destPath = join(CACHE.PREVIEWS.DIRECTORY, video.getPreviewName()) + const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName()) return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) } diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 9493945ff..2b1e21c39 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -2,10 +2,9 @@ import * as Bull from 'bull' import * as Bluebird from 'bluebird' import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' -import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers' -import { ActorFollowScoreCache } from '../../cache' +import { ActorFollowScoreCache } from '../../files-cache' export type ActivitypubHttpBroadcastPayload = { uris: string[] diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 3973dcdc8..59de7119a 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' import { JOB_REQUEST_TIMEOUT } from '../../../initializers' -import { ActorFollowScoreCache } from '../../cache' +import { ActorFollowScoreCache } from '../../files-cache' export type ActivitypubHttpUnicastPayload = { uri: string diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index 2cd2ae97c..5b4a2bcf9 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts @@ -4,12 +4,12 @@ import { logger } from '../helpers/logger' import { UserModel } from '../models/account/user' import { OAuthClientModel } from '../models/oauth/oauth-client' import { OAuthTokenModel } from '../models/oauth/oauth-token' -import { CONFIG } from '../initializers/constants' +import { CONFIG, CACHE } from '../initializers/constants' import { Transaction } from 'sequelize' type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date } -const accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} -const userHavingToken: { [ userId: number ]: string } = {} +let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {} +let userHavingToken: { [ userId: number ]: string } = {} // --------------------------------------------------------------------------- @@ -43,6 +43,12 @@ function getAccessToken (bearerToken: string) { return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) .then(tokenModel => { if (tokenModel) { + // Reinit our cache + if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) { + accessTokenCache = {} + userHavingToken = {} + } + accessTokenCache[ bearerToken ] = tokenModel userHavingToken[ tokenModel.userId ] = tokenModel.accessToken } diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts index 3967be7f8..05e6bd139 100644 --- a/server/lib/schedulers/actor-follow-scheduler.ts +++ b/server/lib/schedulers/actor-follow-scheduler.ts @@ -3,7 +3,7 @@ import { logger } from '../../helpers/logger' import { ActorFollowModel } from '../../models/activitypub/actor-follow' import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' -import { ActorFollowScoreCache } from '../cache' +import { ActorFollowScoreCache } from '../files-cache' export class ActorFollowScheduler extends AbstractScheduler {