From 6040f87d143a5fa01db79867ece8197c3ce7be47 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 16:02:49 +0100 Subject: [PATCH] Add tmp and redundancy directories --- config/default.yaml | 4 +++- config/production.yaml.example | 4 +++- config/test-1.yaml | 2 ++ config/test-2.yaml | 2 ++ config/test-3.yaml | 2 ++ config/test-4.yaml | 2 ++ config/test-5.yaml | 2 ++ config/test-6.yaml | 2 ++ config/test.yaml | 2 +- server/controllers/api/users/me.ts | 4 ++-- server/controllers/api/video-channel.ts | 2 +- server/controllers/api/videos/import.ts | 6 +++--- server/controllers/api/videos/index.ts | 10 +++++----- server/controllers/static.ts | 9 +++++++-- server/helpers/requests.ts | 9 +++++---- server/helpers/utils.ts | 6 +++--- server/helpers/webtorrent.ts | 6 +++--- server/helpers/youtube-dl.ts | 4 ++-- server/initializers/checker-before-init.ts | 1 + server/initializers/constants.ts | 2 ++ server/lib/activitypub/actor.ts | 4 +--- server/lib/activitypub/videos.ts | 3 +-- server/lib/job-queue/handlers/video-import.ts | 9 ++++----- server/lib/job-queue/handlers/video-views.ts | 4 +--- server/lib/redis.ts | 9 ++++++++- support/docker/production/config/production.yaml | 2 ++ 26 files changed, 70 insertions(+), 42 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 257ec7ed1..d95fdc57b 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -45,8 +45,10 @@ smtp: # From the project root directory storage: + tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing... avatars: 'storage/avatars/' videos: 'storage/videos/' + redundancy: 'storage/redundancy/' logs: 'storage/logs/' previews: 'storage/previews/' thumbnails: 'storage/thumbnails/' @@ -75,7 +77,7 @@ trending: redundancy: videos: check_interval: '1 hour' # How often you want to check new videos to cache - strategies: + strategies: # Just uncomment strategies you want # - # size: '10GB' # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) diff --git a/config/production.yaml.example b/config/production.yaml.example index ac15fc736..4c50a550b 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -46,8 +46,10 @@ smtp: # From the project root directory storage: + tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing... avatars: '/var/www/peertube/storage/avatars/' videos: '/var/www/peertube/storage/videos/' + redundancy: '/var/www/peertube/storage/videos/' logs: '/var/www/peertube/storage/logs/' previews: '/var/www/peertube/storage/previews/' thumbnails: '/var/www/peertube/storage/thumbnails/' @@ -76,7 +78,7 @@ trending: redundancy: videos: check_interval: '1 hour' # How often you want to check new videos to cache - strategies: + strategies: # Just uncomment strategies you want # - # size: '10GB' # # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances) diff --git a/config/test-1.yaml b/config/test-1.yaml index 503bbc661..8f4f66d2a 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -10,8 +10,10 @@ database: # From the project root directory storage: + tmp: 'test1/tmp/' avatars: 'test1/avatars/' videos: 'test1/videos/' + redundancy: 'test1/redundancy/' logs: 'test1/logs/' previews: 'test1/previews/' thumbnails: 'test1/thumbnails/' diff --git a/config/test-2.yaml b/config/test-2.yaml index 8c77bf581..a80ec6e54 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -10,8 +10,10 @@ database: # From the project root directory storage: + tmp: 'test2/tmp/' avatars: 'test2/avatars/' videos: 'test2/videos/' + redundancy: 'test2/redundancy/' logs: 'test2/logs/' previews: 'test2/previews/' thumbnails: 'test2/thumbnails/' diff --git a/config/test-3.yaml b/config/test-3.yaml index 82d89567a..934401eb0 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -10,8 +10,10 @@ database: # From the project root directory storage: + tmp: 'test3/tmp/' avatars: 'test3/avatars/' videos: 'test3/videos/' + redundancy: 'test3/redundancy/' logs: 'test3/logs/' previews: 'test3/previews/' thumbnails: 'test3/thumbnails/' diff --git a/config/test-4.yaml b/config/test-4.yaml index 1aa56d041..ee99b250b 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -10,8 +10,10 @@ database: # From the project root directory storage: + tmp: 'test4/tmp/' avatars: 'test4/avatars/' videos: 'test4/videos/' + redundancy: 'test4/redundancy/' logs: 'test4/logs/' previews: 'test4/previews/' thumbnails: 'test4/thumbnails/' diff --git a/config/test-5.yaml b/config/test-5.yaml index 5f1c2f583..e2662bdd9 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -10,8 +10,10 @@ database: # From the project root directory storage: + tmp: 'test5/tmp/' avatars: 'test5/avatars/' videos: 'test5/videos/' + redundancy: 'test5/redundancy/' logs: 'test5/logs/' previews: 'test5/previews/' thumbnails: 'test5/thumbnails/' diff --git a/config/test-6.yaml b/config/test-6.yaml index 719629844..ad39c6a9f 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -10,8 +10,10 @@ database: # From the project root directory storage: + tmp: 'test6/tmp/' avatars: 'test6/avatars/' videos: 'test6/videos/' + redundancy: 'test6/redundancy/' logs: 'test6/logs/' previews: 'test6/previews/' thumbnails: 'test6/thumbnails/' diff --git a/config/test.yaml b/config/test.yaml index 9c051fabc..51a77e2fd 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -67,4 +67,4 @@ import: enabled: true instance: - default_nsfw_policy: 'display' \ No newline at end of file + default_nsfw_policy: 'display' diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 82299747d..47f2c9ec7 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -42,7 +42,7 @@ import { AccountModel } from '../../../models/account/account' const auditLogger = auditLoggerFactory('users-me') -const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) +const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) const meRouter = express.Router() @@ -348,7 +348,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr return res.sendStatus(204) } -async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { +async function updateMyAvatar (req: express.Request, res: express.Response) { const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] const user: UserModel = res.locals.oauth.token.user const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 9bf3c5fd8..63240dfa1 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -32,7 +32,7 @@ import { resetSequelizeInstance } from '../../helpers/database-utils' import { UserModel } from '../../models/account/user' const auditLogger = auditLoggerFactory('channels') -const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) +const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) const videoChannelRouter = express.Router() diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 398fd5a7f..f27d648c7 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -37,9 +37,9 @@ const reqVideoFileImport = createReqFiles( [ 'thumbnailfile', 'previewfile', 'torrentfile' ], Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT), { - thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, - previewfile: CONFIG.STORAGE.PREVIEWS_DIR, - torrentfile: CONFIG.STORAGE.TORRENTS_DIR + thumbnailfile: CONFIG.STORAGE.TMP_DIR, + previewfile: CONFIG.STORAGE.TMP_DIR, + torrentfile: CONFIG.STORAGE.TMP_DIR } ) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 3d1b2e1a2..4e4697ef4 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -67,17 +67,17 @@ const reqVideoFileAdd = createReqFiles( [ 'videofile', 'thumbnailfile', 'previewfile' ], Object.assign({}, VIDEO_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT), { - videofile: CONFIG.STORAGE.VIDEOS_DIR, - thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, - previewfile: CONFIG.STORAGE.PREVIEWS_DIR + videofile: CONFIG.STORAGE.TMP_DIR, + thumbnailfile: CONFIG.STORAGE.TMP_DIR, + previewfile: CONFIG.STORAGE.TMP_DIR } ) const reqVideoFileUpdate = createReqFiles( [ 'thumbnailfile', 'previewfile' ], IMAGE_MIMETYPE_EXT, { - thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, - previewfile: CONFIG.STORAGE.PREVIEWS_DIR + thumbnailfile: CONFIG.STORAGE.TMP_DIR, + previewfile: CONFIG.STORAGE.TMP_DIR } ) diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 75e30353c..f16a7d72b 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -34,12 +34,17 @@ staticRouter.use( ) // Videos path for webseeding -const videosPhysicalPath = CONFIG.STORAGE.VIDEOS_DIR staticRouter.use( STATIC_PATHS.WEBSEED, cors(), - express.static(videosPhysicalPath) + express.static(CONFIG.STORAGE.VIDEOS_DIR) ) +staticRouter.use( + STATIC_PATHS.WEBSEED, + cors(), + express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404, because we don't have this video +) + staticRouter.use( STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', asyncMiddleware(videosGetValidator), diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 5760ad1c1..3fc776f1a 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,9 +1,9 @@ import * as Bluebird from 'bluebird' import { createWriteStream } from 'fs-extra' import * as request from 'request' -import { ACTIVITY_PUB } from '../initializers' +import { ACTIVITY_PUB, CONFIG } from '../initializers' import { processImage } from './image-utils' -import { extname } from 'path' +import { join } from 'path' function doRequest ( requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } @@ -29,10 +29,11 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U }) } -async function downloadImage (url: string, destPath: string, size: { width: number, height: number }) { - const tmpPath = destPath + '.tmp' + extname(destPath) +async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { + const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) + const destPath = join(destDir, destName) await processImage({ path: tmpPath }, destPath, size) } diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 5c9d6fe2f..9b89e3e61 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -46,11 +46,11 @@ const getServerActor = memoizee(async function () { return actor }) -function generateVideoTmpPath (target: string | ParseTorrent) { +function generateVideoImportTmpPath (target: string | ParseTorrent) { const id = typeof target === 'string' ? target : target.infoHash const hash = sha256(id) - return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') + return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4') } function getSecureTorrentName (originalName: string) { @@ -103,6 +103,6 @@ export { getSecureTorrentName, getServerActor, getServerCommit, - generateVideoTmpPath, + generateVideoImportTmpPath, getUUIDFromFilename } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index ce35b87da..3c9a0b96a 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,5 +1,5 @@ import { logger } from './logger' -import { generateVideoTmpPath } from './utils' +import { generateVideoImportTmpPath } from './utils' import * as WebTorrent from 'webtorrent' import { createWriteStream, ensureDir, remove } from 'fs-extra' import { CONFIG } from '../initializers' @@ -9,10 +9,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName const id = target.magnetUri || target.torrentName let timer - const path = generateVideoTmpPath(id) + const path = generateVideoImportTmpPath(id) logger.info('Importing torrent video %s', id) - const directoryPath = join(CONFIG.STORAGE.VIDEOS_DIR, 'import') + const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent') await ensureDir(directoryPath) return new Promise((res, rej) => { diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 2a5663042..b74351b42 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -1,7 +1,7 @@ import { truncate } from 'lodash' import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' import { logger } from './logger' -import { generateVideoTmpPath } from './utils' +import { generateVideoImportTmpPath } from './utils' import { join } from 'path' import { root } from './core-utils' import { ensureDir, writeFile, remove } from 'fs-extra' @@ -40,7 +40,7 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise } function downloadYoutubeDLVideo (url: string, timeout: number) { - const path = generateVideoTmpPath(url) + const path = generateVideoImportTmpPath(url) let timer logger.info('Importing youtubeDL video %s', url) diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 9dfb5d68c..b51c7cfba 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -12,6 +12,7 @@ function checkMissedConfig () { 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', + 'storage.redundancy', 'storage.tmp', 'log.level', 'user.video_quota', 'user.video_quota_daily', 'cache.previews.size', 'admin.email', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 8a8bcd126..876aa1cf5 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -185,9 +185,11 @@ const CONFIG = { FROM_ADDRESS: config.get('smtp.from_address') }, STORAGE: { + TMP_DIR: buildPath(config.get('storage.tmp')), AVATARS_DIR: buildPath(config.get('storage.avatars')), LOG_DIR: buildPath(config.get('storage.logs')), VIDEOS_DIR: buildPath(config.get('storage.videos')), + REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')), THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')), PREVIEWS_DIR: buildPath(config.get('storage.previews')), CAPTIONS_DIR: buildPath(config.get('storage.captions')), diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 504263c99..bbe48833d 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -178,9 +178,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] const avatarName = uuidv4() + extension - const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName) - - await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE) + await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) return avatarName } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index a5d649391..3d17e6846 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -95,9 +95,8 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { const thumbnailName = video.getThumbnailName() - const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) - return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE) + return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) } function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 4de901c0c..51a0b5faf 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -7,7 +7,7 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro import { extname, join } from 'path' import { VideoFileModel } from '../../../models/video/video-file' import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers' -import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests' +import { downloadImage } from '../../../helpers/requests' import { VideoState } from '../../../../shared' import { JobQueue } from '../index' import { federateVideoIfNeeded } from '../../activitypub' @@ -109,6 +109,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide let tempVideoPath: string let videoDestFile: string let videoFile: VideoFileModel + try { // Download video from youtubeDL tempVideoPath = await downloader() @@ -144,8 +145,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Process thumbnail if (options.downloadThumbnail) { if (options.thumbnailUrl) { - const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) - await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE) + await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE) } else { await videoImport.Video.createThumbnail(videoFile) } @@ -156,8 +156,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Process preview if (options.downloadPreview) { if (options.thumbnailUrl) { - const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) - await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE) + await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE) } else { await videoImport.Video.createPreview(videoFile) } diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts index 038ef43e2..fa1fd13b3 100644 --- a/server/lib/job-queue/handlers/video-views.ts +++ b/server/lib/job-queue/handlers/video-views.ts @@ -23,9 +23,7 @@ async function processVideosViews () { for (const videoId of videoIds) { try { const views = await Redis.Instance.getVideoViews(videoId, hour) - if (isNaN(views)) { - logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views) - } else { + if (views) { logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) try { diff --git a/server/lib/redis.ts b/server/lib/redis.ts index abd75d512..3e25e6a2c 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts @@ -121,7 +121,14 @@ class Redis { const key = this.generateVideoViewKey(videoId, hour) const valueString = await this.getValue(key) - return parseInt(valueString, 10) + const valueInt = parseInt(valueString, 10) + + if (isNaN(valueInt)) { + logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) + return undefined + } + + return valueInt } async getVideosIdViewed (hour: number) { diff --git a/support/docker/production/config/production.yaml b/support/docker/production/config/production.yaml index 4970bbcca..846c838e8 100644 --- a/support/docker/production/config/production.yaml +++ b/support/docker/production/config/production.yaml @@ -32,8 +32,10 @@ redis: # From the project root directory storage: + tmp: '../data/tmp/' avatars: '../data/avatars/' videos: '../data/videos/' + redundancy: '../data/redundancy/' logs: '../data/logs/' previews: '../data/previews/' thumbnails: '../data/thumbnails/'