forks/peertube
1
0
Fork 0
Code Releases Activity

Lazy load avatars

Browse source
This commit is contained in:
Chocobozzz 2019-08-09 11:32:40 +02:00
parent c5407d7046
commit 557b13ae24
No known key found for this signature in database
GPG key ID: 583A612D890159BE
18 changed files with 324 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -130,6 +130,7 @@
"jsonld": "~1.1.0",
"jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017",
"lodash": "^4.17.10",
"lru-cache": "^5.1.1",
"magnet-uri": "^5.1.4",
"memoizee": "^0.4.14",
"morgan": "^1.5.3",
@ -179,6 +180,7 @@
"@types/fs-extra": "^8.0.0",
"@types/libxmljs": "^0.18.0",
"@types/lodash": "^4.14.64",
"@types/lru-cache": "^5.1.0",
"@types/magnet-uri": "^5.1.1",
"@types/maildev": "^0.0.1",
"@types/memoizee": "^0.4.2",

2
server.ts
View file

@ -97,6 +97,7 @@ import {
clientsRouter,
feedsRouter,
staticRouter,
lazyStaticRouter,
servicesRouter,
pluginsRouter,
webfingerRouter,
@ -192,6 +193,7 @@ app.use('/', botsRouter)
// Static files
app.use('/', staticRouter)
app.use('/', lazyStaticRouter)
// Client files, last valid routes!
if (cli.client) app.use('/', clientsRouter)

1
server/controllers/index.ts
View file

@ -4,6 +4,7 @@ export * from './client'
export * from './feeds'
export * from './services'
export * from './static'
export * from './lazy-static'
export * from './webfinger'
export * from './tracker'
export * from './bots'

80
server/controllers/lazy-static.ts Normal file
View file

@ -0,0 +1,80 @@
import * as cors from 'cors'
import * as express from 'express'
import { LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants'
import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
import { asyncMiddleware } from '../middlewares'
import { AvatarModel } from '../models/avatar/avatar'
import { logger } from '../helpers/logger'
import { avatarPathUnsafeCache, pushAvatarProcessInQueue } from '../lib/avatar'
const lazyStaticRouter = express.Router()
lazyStaticRouter.use(cors())
lazyStaticRouter.use(
LAZY_STATIC_PATHS.AVATARS + ':filename',
asyncMiddleware(getAvatar)
)
lazyStaticRouter.use(
LAZY_STATIC_PATHS.PREVIEWS + ':uuid.jpg',
asyncMiddleware(getPreview)
)
lazyStaticRouter.use(
LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt',
asyncMiddleware(getVideoCaption)
)
// ---------------------------------------------------------------------------
export {
lazyStaticRouter,
getPreview,
getVideoCaption
}
// ---------------------------------------------------------------------------
async function getAvatar (req: express.Request, res: express.Response) {
const filename = req.params.filename
if (avatarPathUnsafeCache.has(filename)) {
return res.sendFile(avatarPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
}
const avatar = await AvatarModel.loadByName(filename)
if (avatar.onDisk === false) {
if (!avatar.fileUrl) return res.sendStatus(404)
logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
avatar.onDisk = true
avatar.save()
.catch(err => logger.error('Cannot save new avatar disk state.', { err }))
}
const path = avatar.getPath()
avatarPathUnsafeCache.set(filename, path)
return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER })
}
async function getPreview (req: express.Request, res: express.Response) {
const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
if (!result) return res.sendStatus(404)
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })
}
async function getVideoCaption (req: express.Request, res: express.Response) {
const result = await VideosCaptionCache.Instance.getFilePath({
videoId: req.params.videoId,
language: req.params.captionLanguage
})
if (!result) return res.sendStatus(404)
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER })
}

24
server/controllers/static.ts
View file

@ -9,7 +9,6 @@ import {
STATIC_PATHS,
WEBSERVER
} from '../initializers/constants'
import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache'
import { cacheRoute } from '../middlewares/cache'
import { asyncMiddleware, videosGetValidator } from '../middlewares'
import { VideoModel } from '../models/video/video'
@ -19,6 +18,7 @@ import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/node
import { join } from 'path'
import { root } from '../helpers/core-utils'
import { CONFIG } from '../initializers/config'
import { getPreview, getVideoCaption } from './lazy-static'
const staticRouter = express.Router()
@ -72,19 +72,20 @@ staticRouter.use(
express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist
)
// DEPRECATED: use lazy-static route instead
const avatarsPhysicalPath = CONFIG.STORAGE.AVATARS_DIR
staticRouter.use(
STATIC_PATHS.AVATARS,
express.static(avatarsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }) // 404 if the file does not exist
)
// We don't have video previews, fetch them from the origin instance
// DEPRECATED: use lazy-static route instead
staticRouter.use(
STATIC_PATHS.PREVIEWS + ':uuid.jpg',
asyncMiddleware(getPreview)
)
// We don't have video captions, fetch them from the origin instance
// DEPRECATED: use lazy-static route instead
staticRouter.use(
STATIC_PATHS.VIDEO_CAPTIONS + ':videoId-:captionLanguage([a-z]+).vtt',
asyncMiddleware(getVideoCaption)
@ -177,23 +178,6 @@ export {
// ---------------------------------------------------------------------------
async function getPreview (req: express.Request, res: express.Response) {
const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid)
if (!result) return res.sendStatus(404)
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE })
}
async function getVideoCaption (req: express.Request, res: express.Response) {
const result = await VideosCaptionCache.Instance.getFilePath({
videoId: req.params.videoId,
language: req.params.captionLanguage
})
if (!result) return res.sendStatus(404)
return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE })
}
async function generateNodeinfo (req: express.Request, res: express.Response) {
const { totalVideos } = await VideoModel.getStats()
const { totalLocalVideoComments } = await VideoCommentModel.getStats()

22
server/initializers/constants.ts
View file

@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 415
const LAST_MIGRATION_VERSION = 420
// ---------------------------------------------------------------------------
@ -498,6 +498,11 @@ const STATIC_DOWNLOAD_PATHS = {
TORRENTS: '/download/torrents/',
VIDEOS: '/download/videos/'
}
const LAZY_STATIC_PATHS = {
AVATARS: '/lazy-static/avatars/',
PREVIEWS: '/static/previews/',
VIDEO_CAPTIONS: '/static/video-captions/'
}
// Cache control
let STATIC_MAX_AGE = {
@ -536,9 +541,12 @@ const FILES_CACHE = {
}
}
const CACHE = {
const LRU_CACHE = {
USER_TOKENS: {
MAX_SIZE: 10000
MAX_SIZE: 1000
},
AVATAR_STATIC: {
MAX_SIZE: 500
}
}
@ -549,6 +557,10 @@ const MEMOIZE_TTL = {
OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
}
const QUEUE_CONCURRENCY = {
AVATAR_PROCESS_IMAGE: 3
}
const REDUNDANCY = {
VIDEOS: {
RANDOMIZED_FACTOR: 5
@ -649,6 +661,7 @@ export {
WEBSERVER,
API_VERSION,
PEERTUBE_VERSION,
LAZY_STATIC_PATHS,
HLS_REDUNDANCY_DIRECTORY,
P2P_MEDIA_LOADER_PEER_VERSION,
AVATARS_SIZE,
@ -695,11 +708,12 @@ export {
VIDEO_PRIVACIES,
VIDEO_LICENCES,
VIDEO_STATES,
QUEUE_CONCURRENCY,
VIDEO_RATE_TYPES,
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
VIDEO_ABUSE_STATES,
CACHE,
LRU_CACHE,
JOB_REQUEST_TIMEOUT,
USER_PASSWORD_RESET_LIFETIME,
MEMOIZE_TTL,

60
server/initializers/migrations/0420-avatar-lazy.ts Normal file
View file

@ -0,0 +1,60 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction,
queryInterface: Sequelize.QueryInterface,
sequelize: Sequelize.Sequelize,
db: any
}): Promise<void> {
{
// We'll add a unique index on filename, so delete duplicates or PeerTube won't start
const query = 'DELETE FROM "avatar" s1 ' +
'USING (SELECT MIN(id) as id, filename FROM "avatar" GROUP BY "filename" HAVING COUNT(*) > 1) s2 ' +
'WHERE s1."filename" = s2."filename" AND s1.id <> s2.id'
await utils.sequelize.query(query)
}
{
const data = {
type: Sequelize.STRING,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('avatar', 'fileUrl', data)
}
{
const data = {
type: Sequelize.BOOLEAN,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('avatar', 'onDisk', data)
}
{
const query = 'UPDATE "avatar" SET "onDisk" = true;'
await utils.sequelize.query(query)
}
{
const data = {
type: Sequelize.BOOLEAN,
allowNull: false,
defaultValue: null
}
await utils.queryInterface.changeColumn('avatar', 'onDisk', data)
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

67
server/lib/activitypub/actor.ts
View file

@ -10,9 +10,9 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
import { doRequest, downloadImage } from '../../helpers/requests'
import { doRequest } from '../../helpers/requests'
import { getUrlFromWebfinger } from '../../helpers/webfinger'
import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants'
import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
import { AccountModel } from '../../models/account/account'
import { ActorModel } from '../../models/activitypub/actor'
import { AvatarModel } from '../../models/avatar/avatar'
@ -21,7 +21,6 @@ import { VideoChannelModel } from '../../models/video/video-channel'
import { JobQueue } from '../job-queue'
import { getServerActor } from '../../helpers/utils'
import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
import { CONFIG } from '../../initializers/config'
import { sequelizeTypescript } from '../../initializers/database'
// Set account keys, this could be long so process after the account creation and do not block the client
@ -141,25 +140,27 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ
actorInstance.followingUrl = attributes.following
}
async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
if (avatarName !== undefined) {
if (actorInstance.avatarId) {
async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) {
if (info.name !== undefined) {
if (actor.avatarId) {
try {
await actorInstance.Avatar.destroy({ transaction: t })
await actor.Avatar.destroy({ transaction: t })
} catch (err) {
logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
logger.error('Cannot remove old avatar of actor %s.', actor.url, { err })
}
}
const avatar = await AvatarModel.create({
filename: avatarName
filename: info.name,
onDisk: info.onDisk,
fileUrl: info.fileUrl
}, { transaction: t })
actorInstance.set('avatarId', avatar.id)
actorInstance.Avatar = avatar
actor.avatarId = avatar.id
actor.Avatar = avatar
}
return actorInstance
return actor
}
async function fetchActorTotalItems (url: string) {
@ -179,17 +180,17 @@ async function fetchActorTotalItems (url: string) {
}
}
async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) {
if (
actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
isActivityPubUrlValid(actorJSON.icon.url)
) {
const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
const avatarName = uuidv4() + extension
await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
return avatarName
return {
name: uuidv4() + extension,
fileUrl: actorJSON.icon.url
}
}
return undefined
@ -245,8 +246,14 @@ async function refreshActorIfNeeded (
return sequelizeTypescript.transaction(async t => {
updateInstanceWithAnother(actor, result.actor)
if (result.avatarName !== undefined) {
await updateActorAvatarInstance(actor, result.avatarName, t)
if (result.avatar !== undefined) {
const avatarInfo = {
name: result.avatar.name,
fileUrl: result.avatar.fileUrl,
onDisk: false
}
await updateActorAvatarInstance(actor, avatarInfo, t)
}
// Force update
@ -279,7 +286,7 @@ export {
buildActorInstance,
setAsyncActorKeys,
fetchActorTotalItems,
fetchAvatarIfExists,
getAvatarInfoIfExists,
updateActorInstance,
refreshActorIfNeeded,
updateActorAvatarInstance,
@ -314,14 +321,17 @@ function saveActorAndServerAndModelIfNotExist (
const [ server ] = await ServerModel.findOrCreate(serverOptions)
// Save our new account in database
actor.set('serverId', server.id)
actor.serverId = server.id
// Avatar?
if (result.avatarName) {
if (result.avatar) {
const avatar = await AvatarModel.create({
filename: result.avatarName
filename: result.avatar.name,
fileUrl: result.avatar.fileUrl,
onDisk: false
}, { transaction: t })
actor.set('avatarId', avatar.id)
actor.avatarId = avatar.id
}
// Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
@ -355,7 +365,10 @@ type FetchRemoteActorResult = {
summary: string
support?: string
playlists?: string
avatarName?: string
avatar?: {
name: string,
fileUrl: string
}
attributedTo: ActivityPubAttributedTo[]
}
async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
@ -399,7 +412,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
followingUrl: actorJSON.following
})
const avatarName = await fetchAvatarIfExists(actorJSON)
const avatarInfo = await getAvatarInfoIfExists(actorJSON)
const name = actorJSON.name || actorJSON.preferredUsername
return {
@ -407,7 +420,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
result: {
actor,
name,
avatarName,
avatar: avatarInfo,
summary: actorJSON.summary,
support: actorJSON.support,
playlists: actorJSON.playlists,

10
server/lib/activitypub/process/process-update.ts
View file

@ -6,7 +6,7 @@ import { sequelizeTypescript } from '../../../initializers'
import { AccountModel } from '../../../models/account/account'
import { ActorModel } from '../../../models/activitypub/actor'
import { VideoChannelModel } from '../../../models/video/video-channel'
import { fetchAvatarIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
import { getAvatarInfoIfExists, updateActorAvatarInstance, updateActorInstance } from '../actor'
import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
@ -105,7 +105,7 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
let accountOrChannelFieldsSave: object
// Fetch icon?
const avatarName = await fetchAvatarIfExists(actorAttributesToUpdate)
const avatarInfo = await getAvatarInfoIfExists(actorAttributesToUpdate)
try {
await sequelizeTypescript.transaction(async t => {
@ -118,8 +118,10 @@ async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate)
await updateActorInstance(actor, actorAttributesToUpdate)
if (avatarName !== undefined) {
await updateActorAvatarInstance(actor, avatarName, t)
if (avatarInfo !== undefined) {
const avatarOptions = Object.assign({}, avatarInfo, { onDisk: false })
await updateActorAvatarInstance(actor, avatarOptions, t)
}
await actor.save({ transaction: t })

40
server/lib/avatar.ts
View file

@ -1,6 +1,6 @@
import 'multer'
import { sendUpdateActor } from './activitypub/send'
import { AVATARS_SIZE } from '../initializers/constants'
import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants'
import { updateActorAvatarInstance } from './activitypub'
import { processImage } from '../helpers/image-utils'
import { AccountModel } from '../models/account/account'
@ -10,6 +10,9 @@ import { retryTransactionWrapper } from '../helpers/database-utils'
import * as uuidv4 from 'uuid/v4'
import { CONFIG } from '../initializers/config'
import { sequelizeTypescript } from '../initializers/database'
import * as LRUCache from 'lru-cache'
import { queue } from 'async'
import { downloadImage } from '../helpers/requests'
async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, accountOrChannel: AccountModel | VideoChannelModel) {
const extension = extname(avatarPhysicalFile.filename)
@ -19,7 +22,13 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
return retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async t => {
const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarName, t)
const avatarInfo = {
name: avatarName,
fileUrl: null,
onDisk: true
}
const updatedActor = await updateActorAvatarInstance(accountOrChannel.Actor, avatarInfo, t)
await updatedActor.save({ transaction: t })
await sendUpdateActor(accountOrChannel, t)
@ -29,6 +38,29 @@ async function updateActorAvatarFile (avatarPhysicalFile: Express.Multer.File, a
})
}
export {
updateActorAvatarFile
type DownloadImageQueueTask = { fileUrl: string, filename: string }
const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => {
downloadImage(task.fileUrl, CONFIG.STORAGE.AVATARS_DIR, task.filename, AVATARS_SIZE)
.then(() => cb())
.catch(err => cb(err))
}, QUEUE_CONCURRENCY.AVATAR_PROCESS_IMAGE)
function pushAvatarProcessInQueue (task: DownloadImageQueueTask) {
return new Promise((res, rej) => {
downloadImageQueue.push(task, err => {
if (err) return rej(err)
return res()
})
})
}
// Unsafe so could returns paths that does not exist anymore
const avatarPathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.AVATAR_STATIC.MAX_SIZE })
export {
avatarPathUnsafeCache,
updateActorAvatarFile,
pushAvatarProcessInQueue
}

34
server/lib/oauth-model.ts
View file

@ -4,13 +4,15 @@ import { logger } from '../helpers/logger'
import { UserModel } from '../models/account/user'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { OAuthTokenModel } from '../models/oauth/oauth-token'
import { CACHE } from '../initializers/constants'
import { LRU_CACHE } from '../initializers/constants'
import { Transaction } from 'sequelize'
import { CONFIG } from '../initializers/config'
import * as LRUCache from 'lru-cache'
type TokenInfo = { accessToken: string, refreshToken: string, accessTokenExpiresAt: Date, refreshTokenExpiresAt: Date }
let accessTokenCache: { [ accessToken: string ]: OAuthTokenModel } = {}
let userHavingToken: { [ userId: number ]: string } = {}
const accessTokenCache = new LRUCache<string, OAuthTokenModel>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
const userHavingToken = new LRUCache<number, string>({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE })
// ---------------------------------------------------------------------------
@ -21,18 +23,20 @@ function deleteUserToken (userId: number, t?: Transaction) {
}
function clearCacheByUserId (userId: number) {
const token = userHavingToken[userId]
const token = userHavingToken.get(userId)
if (token !== undefined) {
accessTokenCache[ token ] = undefined
userHavingToken[ userId ] = undefined
accessTokenCache.del(token)
userHavingToken.del(userId)
}
}
function clearCacheByToken (token: string) {
const tokenModel = accessTokenCache[ token ]
const tokenModel = accessTokenCache.get(token)
if (tokenModel !== undefined) {
userHavingToken[tokenModel.userId] = undefined
accessTokenCache[ token ] = undefined
userHavingToken.del(tokenModel.userId)
accessTokenCache.del(token)
}
}
@ -41,19 +45,13 @@ function getAccessToken (bearerToken: string) {
if (!bearerToken) return Bluebird.resolve(undefined)
if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken])
if (accessTokenCache.has(bearerToken)) return Bluebird.resolve(accessTokenCache.get(bearerToken))
return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken)
.then(tokenModel => {
if (tokenModel) {
// Reinit our cache
if (Object.keys(accessTokenCache).length > CACHE.USER_TOKENS.MAX_SIZE) {
accessTokenCache = {}
userHavingToken = {}
}
accessTokenCache[ bearerToken ] = tokenModel
userHavingToken[ tokenModel.userId ] = tokenModel.accessToken
accessTokenCache.set(bearerToken, tokenModel)
userHavingToken.set(tokenModel.userId, tokenModel.accessToken)
}
return tokenModel

4
server/models/account/user-notification.ts
View file

@ -410,7 +410,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
id: this.ActorFollow.ActorFollower.Account.id,
displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(),
name: this.ActorFollow.ActorFollower.preferredUsername,
avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getWebserverPath() } : undefined,
avatar: this.ActorFollow.ActorFollower.Avatar ? { path: this.ActorFollow.ActorFollower.Avatar.getStaticPath() } : undefined,
host: this.ActorFollow.ActorFollower.getHost()
},
following: {
@ -446,7 +446,7 @@ export class UserNotificationModel extends Model<UserNotificationModel> {
private formatActor (accountOrChannel: AccountModel | VideoChannelModel) {
const avatar = accountOrChannel.Actor.Avatar
? { path: accountOrChannel.Actor.Avatar.getWebserverPath() }
? { path: accountOrChannel.Actor.Avatar.getStaticPath() }
: undefined
return {

2
server/models/activitypub/actor.ts
View file

@ -513,7 +513,7 @@ export class ActorModel extends Model<ActorModel> {
getAvatarUrl () {
if (!this.avatarId) return undefined
return WEBSERVER.URL + this.Avatar.getWebserverPath()
return WEBSERVER.URL + this.Avatar.getStaticPath()
}
isOutdated () {

43
server/models/avatar/avatar.ts
View file

@ -1,13 +1,21 @@
import { join } from 'path'
import { AfterDestroy, AllowNull, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { AfterDestroy, AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { Avatar } from '../../../shared/models/avatars/avatar.model'
import { STATIC_PATHS } from '../../initializers/constants'
import { LAZY_STATIC_PATHS } from '../../initializers/constants'
import { logger } from '../../helpers/logger'
import { remove } from 'fs-extra'
import { CONFIG } from '../../initializers/config'
import { throwIfNotValid } from '../utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
@Table({
tableName: 'avatar'
tableName: 'avatar',
indexes: [
{
fields: [ 'filename' ],
unique: true
}
]
})
export class AvatarModel extends Model<AvatarModel> {
@ -15,6 +23,15 @@ export class AvatarModel extends Model<AvatarModel> {
@Column
filename: string
@AllowNull(true)
@Is('AvatarFileUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'fileUrl'))
@Column
fileUrl: string
@AllowNull(false)
@Column
onDisk: boolean
@CreatedAt
createdAt: Date
@ -30,16 +47,30 @@ export class AvatarModel extends Model<AvatarModel> {
.catch(err => logger.error('Cannot remove avatar file %s.', instance.filename, err))
}
static loadByName (filename: string) {
const query = {
where: {
filename
}
}
return AvatarModel.findOne(query)
}
toFormattedJSON (): Avatar {
return {
path: this.getWebserverPath(),
path: this.getStaticPath(),
createdAt: this.createdAt,
updatedAt: this.updatedAt
}
}
getWebserverPath () {
return join(STATIC_PATHS.AVATARS, this.filename)
getStaticPath () {
return join(LAZY_STATIC_PATHS.AVATARS, this.filename)
}
getPath () {
return join(CONFIG.STORAGE.AVATARS_DIR, this.filename)
}
removeAvatar () {

4
server/models/video/thumbnail.ts
View file

@ -1,6 +1,6 @@
import { join } from 'path'
import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
import { LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants'
import { logger } from '../../helpers/logger'
import { remove } from 'fs-extra'
import { CONFIG } from '../../initializers/config'
@ -87,7 +87,7 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
[ThumbnailType.PREVIEW]: {
label: 'preview',
directory: CONFIG.STORAGE.PREVIEWS_DIR,
staticPath: STATIC_PATHS.PREVIEWS
staticPath: LAZY_STATIC_PATHS.PREVIEWS
}
}

4
server/models/video/video-caption.ts
View file

@ -16,7 +16,7 @@ import { buildWhereIdOrUUID, throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model'
import { STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants'
import { LAZY_STATIC_PATHS, VIDEO_LANGUAGES } from '../../initializers/constants'
import { join } from 'path'
import { logger } from '../../helpers/logger'
import { remove } from 'fs-extra'
@ -163,7 +163,7 @@ export class VideoCaptionModel extends Model<VideoCaptionModel> {
}
getCaptionStaticPath () {
return join(STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName())
return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.getCaptionName())
}
getCaptionName () {

3
server/models/video/video.ts
View file

@ -63,6 +63,7 @@ import {
CONSTRAINTS_FIELDS,
HLS_REDUNDANCY_DIRECTORY,
HLS_STREAMING_PLAYLIST_DIRECTORY,
LAZY_STATIC_PATHS,
REMOTE_SCHEME,
STATIC_DOWNLOAD_PATHS,
STATIC_PATHS,
@ -1856,7 +1857,7 @@ export class VideoModel extends Model<VideoModel> {
if (!preview) return null
// We use a local cache, so specify our cache endpoint instead of potential remote URL
return join(STATIC_PATHS.PREVIEWS, preview.filename)
return join(LAZY_STATIC_PATHS.PREVIEWS, preview.filename)
}
toFormattedJSON (options?: VideoFormattingJSONOptions): Video {

14
yarn.lock
View file

@ -197,6 +197,11 @@
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.136.tgz#413e85089046b865d960c9ff1d400e04c31ab60f"
integrity sha512-0GJhzBdvsW2RUccNHOBkabI8HZVdOXmXbXhuKlDEd5Vv12P7oAVGfomGp3Ne21o5D/qu1WmthlNKFaoZJJeErA==
"@types/lru-cache@^5.1.0":
version "5.1.0"
resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03"
integrity sha512-RaE0B+14ToE4l6UqdarKPnXwVDuigfFv+5j9Dze/Nqr23yyuqdNvzcZi3xB+3Agvi5R4EOgAksfv3lXX4vBt9w==
"@types/magnet-uri@*", "@types/magnet-uri@^5.1.1":
version "5.1.2"
resolved "https://registry.yarnpkg.com/@types/magnet-uri/-/magnet-uri-5.1.2.tgz#7860417399d52ddc0be1021d570b4ac93ffc133e"
@ -4394,6 +4399,13 @@ lru-cache@4.1.x, lru-cache@^4.0.1:
pseudomap "^1.0.2"
yallist "^2.1.2"
lru-cache@^5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920"
integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==
dependencies:
yallist "^3.0.2"
lru-queue@0.1:
version "0.1.0"
resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
@ -8082,7 +8094,7 @@ yallist@^2.1.2:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
yallist@^3.0.0, yallist@^3.0.3:
yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3:
version "3.0.3"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.3.tgz#b4b049e314be545e3ce802236d6cd22cd91c3de9"
integrity sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==