1
0
Fork 0

Add tmp and redundancy directories

This commit is contained in:
Chocobozzz 2018-12-04 16:02:49 +01:00
parent 745778256c
commit 6040f87d14
No known key found for this signature in database
GPG key ID: 583A612D890159BE
26 changed files with 70 additions and 42 deletions

View file

@ -45,8 +45,10 @@ smtp:
# From the project root directory
storage:
tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
avatars: 'storage/avatars/'
videos: 'storage/videos/'
redundancy: 'storage/redundancy/'
logs: 'storage/logs/'
previews: 'storage/previews/'
thumbnails: 'storage/thumbnails/'
@ -75,7 +77,7 @@ trending:
redundancy:
videos:
check_interval: '1 hour' # How often you want to check new videos to cache
strategies:
strategies: # Just uncomment strategies you want
# -
# size: '10GB'
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)

View file

@ -46,8 +46,10 @@ smtp:
# From the project root directory
storage:
tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
avatars: '/var/www/peertube/storage/avatars/'
videos: '/var/www/peertube/storage/videos/'
redundancy: '/var/www/peertube/storage/videos/'
logs: '/var/www/peertube/storage/logs/'
previews: '/var/www/peertube/storage/previews/'
thumbnails: '/var/www/peertube/storage/thumbnails/'
@ -76,7 +78,7 @@ trending:
redundancy:
videos:
check_interval: '1 hour' # How often you want to check new videos to cache
strategies:
strategies: # Just uncomment strategies you want
# -
# size: '10GB'
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)

View file

@ -10,8 +10,10 @@ database:
# From the project root directory
storage:
tmp: 'test1/tmp/'
avatars: 'test1/avatars/'
videos: 'test1/videos/'
redundancy: 'test1/redundancy/'
logs: 'test1/logs/'
previews: 'test1/previews/'
thumbnails: 'test1/thumbnails/'

View file

@ -10,8 +10,10 @@ database:
# From the project root directory
storage:
tmp: 'test2/tmp/'
avatars: 'test2/avatars/'
videos: 'test2/videos/'
redundancy: 'test2/redundancy/'
logs: 'test2/logs/'
previews: 'test2/previews/'
thumbnails: 'test2/thumbnails/'

View file

@ -10,8 +10,10 @@ database:
# From the project root directory
storage:
tmp: 'test3/tmp/'
avatars: 'test3/avatars/'
videos: 'test3/videos/'
redundancy: 'test3/redundancy/'
logs: 'test3/logs/'
previews: 'test3/previews/'
thumbnails: 'test3/thumbnails/'

View file

@ -10,8 +10,10 @@ database:
# From the project root directory
storage:
tmp: 'test4/tmp/'
avatars: 'test4/avatars/'
videos: 'test4/videos/'
redundancy: 'test4/redundancy/'
logs: 'test4/logs/'
previews: 'test4/previews/'
thumbnails: 'test4/thumbnails/'

View file

@ -10,8 +10,10 @@ database:
# From the project root directory
storage:
tmp: 'test5/tmp/'
avatars: 'test5/avatars/'
videos: 'test5/videos/'
redundancy: 'test5/redundancy/'
logs: 'test5/logs/'
previews: 'test5/previews/'
thumbnails: 'test5/thumbnails/'

View file

@ -10,8 +10,10 @@ database:
# From the project root directory
storage:
tmp: 'test6/tmp/'
avatars: 'test6/avatars/'
videos: 'test6/videos/'
redundancy: 'test6/redundancy/'
logs: 'test6/logs/'
previews: 'test6/previews/'
thumbnails: 'test6/thumbnails/'

View file

@ -67,4 +67,4 @@ import:
enabled: true
instance:
default_nsfw_policy: 'display'
default_nsfw_policy: 'display'

View file

@ -42,7 +42,7 @@ import { AccountModel } from '../../../models/account/account'
const auditLogger = auditLoggerFactory('users-me')
const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
const meRouter = express.Router()
@ -348,7 +348,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
return res.sendStatus(204)
}
async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) {
async function updateMyAvatar (req: express.Request, res: express.Response) {
const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
const user: UserModel = res.locals.oauth.token.user
const oldUserAuditView = new UserAuditView(user.toFormattedJSON())

View file

@ -32,7 +32,7 @@ import { resetSequelizeInstance } from '../../helpers/database-utils'
import { UserModel } from '../../models/account/user'
const auditLogger = auditLoggerFactory('channels')
const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
const videoChannelRouter = express.Router()

View file

@ -37,9 +37,9 @@ const reqVideoFileImport = createReqFiles(
[ 'thumbnailfile', 'previewfile', 'torrentfile' ],
Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
{
thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
previewfile: CONFIG.STORAGE.PREVIEWS_DIR,
torrentfile: CONFIG.STORAGE.TORRENTS_DIR
thumbnailfile: CONFIG.STORAGE.TMP_DIR,
previewfile: CONFIG.STORAGE.TMP_DIR,
torrentfile: CONFIG.STORAGE.TMP_DIR
}
)

View file

@ -67,17 +67,17 @@ const reqVideoFileAdd = createReqFiles(
[ 'videofile', 'thumbnailfile', 'previewfile' ],
Object.assign({}, VIDEO_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
{
videofile: CONFIG.STORAGE.VIDEOS_DIR,
thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
previewfile: CONFIG.STORAGE.PREVIEWS_DIR
videofile: CONFIG.STORAGE.TMP_DIR,
thumbnailfile: CONFIG.STORAGE.TMP_DIR,
previewfile: CONFIG.STORAGE.TMP_DIR
}
)
const reqVideoFileUpdate = createReqFiles(
[ 'thumbnailfile', 'previewfile' ],
IMAGE_MIMETYPE_EXT,
{
thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
previewfile: CONFIG.STORAGE.PREVIEWS_DIR
thumbnailfile: CONFIG.STORAGE.TMP_DIR,
previewfile: CONFIG.STORAGE.TMP_DIR
}
)

View file

@ -34,12 +34,17 @@ staticRouter.use(
)
// Videos path for webseeding
const videosPhysicalPath = CONFIG.STORAGE.VIDEOS_DIR
staticRouter.use(
STATIC_PATHS.WEBSEED,
cors(),
express.static(videosPhysicalPath)
express.static(CONFIG.STORAGE.VIDEOS_DIR)
)
staticRouter.use(
STATIC_PATHS.WEBSEED,
cors(),
express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404, because we don't have this video
)
staticRouter.use(
STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
asyncMiddleware(videosGetValidator),

View file

@ -1,9 +1,9 @@
import * as Bluebird from 'bluebird'
import { createWriteStream } from 'fs-extra'
import * as request from 'request'
import { ACTIVITY_PUB } from '../initializers'
import { ACTIVITY_PUB, CONFIG } from '../initializers'
import { processImage } from './image-utils'
import { extname } from 'path'
import { join } from 'path'
function doRequest <T> (
requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }
@ -29,10 +29,11 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U
})
}
async function downloadImage (url: string, destPath: string, size: { width: number, height: number }) {
const tmpPath = destPath + '.tmp' + extname(destPath)
async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
const destPath = join(destDir, destName)
await processImage({ path: tmpPath }, destPath, size)
}

View file

@ -46,11 +46,11 @@ const getServerActor = memoizee(async function () {
return actor
})
function generateVideoTmpPath (target: string | ParseTorrent) {
function generateVideoImportTmpPath (target: string | ParseTorrent) {
const id = typeof target === 'string' ? target : target.infoHash
const hash = sha256(id)
return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4')
}
function getSecureTorrentName (originalName: string) {
@ -103,6 +103,6 @@ export {
getSecureTorrentName,
getServerActor,
getServerCommit,
generateVideoTmpPath,
generateVideoImportTmpPath,
getUUIDFromFilename
}

View file

@ -1,5 +1,5 @@
import { logger } from './logger'
import { generateVideoTmpPath } from './utils'
import { generateVideoImportTmpPath } from './utils'
import * as WebTorrent from 'webtorrent'
import { createWriteStream, ensureDir, remove } from 'fs-extra'
import { CONFIG } from '../initializers'
@ -9,10 +9,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
const id = target.magnetUri || target.torrentName
let timer
const path = generateVideoTmpPath(id)
const path = generateVideoImportTmpPath(id)
logger.info('Importing torrent video %s', id)
const directoryPath = join(CONFIG.STORAGE.VIDEOS_DIR, 'import')
const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
await ensureDir(directoryPath)
return new Promise<string>((res, rej) => {

View file

@ -1,7 +1,7 @@
import { truncate } from 'lodash'
import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
import { logger } from './logger'
import { generateVideoTmpPath } from './utils'
import { generateVideoImportTmpPath } from './utils'
import { join } from 'path'
import { root } from './core-utils'
import { ensureDir, writeFile, remove } from 'fs-extra'
@ -40,7 +40,7 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
}
function downloadYoutubeDLVideo (url: string, timeout: number) {
const path = generateVideoTmpPath(url)
const path = generateVideoImportTmpPath(url)
let timer
logger.info('Importing youtubeDL video %s', url)

View file

@ -12,6 +12,7 @@ function checkMissedConfig () {
'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
'storage.redundancy', 'storage.tmp',
'log.level',
'user.video_quota', 'user.video_quota_daily',
'cache.previews.size', 'admin.email',

View file

@ -185,9 +185,11 @@ const CONFIG = {
FROM_ADDRESS: config.get<string>('smtp.from_address')
},
STORAGE: {
TMP_DIR: buildPath(config.get<string>('storage.tmp')),
AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
LOG_DIR: buildPath(config.get<string>('storage.logs')),
VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),

View file

@ -178,9 +178,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
const avatarName = uuidv4() + extension
const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE)
await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
return avatarName
}

View file

@ -95,9 +95,8 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu
function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
const thumbnailName = video.getThumbnailName()
const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE)
return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
}
function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {

View file

@ -7,7 +7,7 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
import { extname, join } from 'path'
import { VideoFileModel } from '../../../models/video/video-file'
import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests'
import { downloadImage } from '../../../helpers/requests'
import { VideoState } from '../../../../shared'
import { JobQueue } from '../index'
import { federateVideoIfNeeded } from '../../activitypub'
@ -109,6 +109,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
let tempVideoPath: string
let videoDestFile: string
let videoFile: VideoFileModel
try {
// Download video from youtubeDL
tempVideoPath = await downloader()
@ -144,8 +145,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
// Process thumbnail
if (options.downloadThumbnail) {
if (options.thumbnailUrl) {
const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE)
await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE)
} else {
await videoImport.Video.createThumbnail(videoFile)
}
@ -156,8 +156,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
// Process preview
if (options.downloadPreview) {
if (options.thumbnailUrl) {
const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE)
await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE)
} else {
await videoImport.Video.createPreview(videoFile)
}

View file

@ -23,9 +23,7 @@ async function processVideosViews () {
for (const videoId of videoIds) {
try {
const views = await Redis.Instance.getVideoViews(videoId, hour)
if (isNaN(views)) {
logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views)
} else {
if (views) {
logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
try {

View file

@ -121,7 +121,14 @@ class Redis {
const key = this.generateVideoViewKey(videoId, hour)
const valueString = await this.getValue(key)
return parseInt(valueString, 10)
const valueInt = parseInt(valueString, 10)
if (isNaN(valueInt)) {
logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
return undefined
}
return valueInt
}
async getVideosIdViewed (hour: number) {

View file

@ -32,8 +32,10 @@ redis:
# From the project root directory
storage:
tmp: '../data/tmp/'
avatars: '../data/avatars/'
videos: '../data/videos/'
redundancy: '../data/redundancy/'
logs: '../data/logs/'
previews: '../data/previews/'
thumbnails: '../data/thumbnails/'