1
0
Fork 0
peertube/scripts/prune-storage.ts

173 lines
5.3 KiB
TypeScript
Raw Normal View History

2021-08-27 08:32:44 -04:00
import { map } from 'bluebird'
import { readdir, remove, stat } from 'fs-extra'
import { uniq, values } from 'lodash'
import { basename, join } from 'path'
import { get, start } from 'prompt'
2021-12-10 04:28:46 -05:00
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
import { VideoFileModel } from '@server/models/video/video-file'
2021-12-10 04:28:46 -05:00
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { ThumbnailType } from '@shared/models'
import { getUUIDFromFilename } from '../server/helpers/utils'
import { CONFIG } from '../server/initializers/config'
import { initDatabaseModels } from '../server/initializers/database'
import { ActorImageModel } from '../server/models/actor/actor-image'
import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
import { ThumbnailModel } from '../server/models/video/thumbnail'
import { VideoModel } from '../server/models/video/video'
2018-05-29 05:11:52 -04:00
run()
.then(() => process.exit(0))
.catch(err => {
console.error(err)
process.exit(-1)
})
async function run () {
const dirs = values(CONFIG.STORAGE)
if (uniq(dirs).length !== dirs.length) {
console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
process.exit(0)
}
2018-05-29 05:11:52 -04:00
await initDatabaseModels(true)
2019-08-09 09:04:36 -04:00
let toDelete: string[] = []
2018-05-29 05:11:52 -04:00
2021-04-28 02:56:08 -04:00
console.log('Detecting files to remove, it could take a while...')
2019-08-09 09:04:36 -04:00
toDelete = toDelete.concat(
2021-07-23 05:20:00 -04:00
await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
2021-12-10 04:28:46 -05:00
await pruneDirectory(HLS_STREAMING_PLAYLIST_DIRECTORY, doesHLSPlaylistExist()),
2021-07-23 05:20:00 -04:00
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
2018-09-28 04:56:13 -04:00
2019-08-09 09:04:36 -04:00
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
2018-09-28 04:56:13 -04:00
await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
2019-08-09 09:04:36 -04:00
2021-04-06 05:35:56 -04:00
await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES, doesActorImageExist)
2019-08-09 09:04:36 -04:00
)
2018-05-29 05:11:52 -04:00
2018-12-06 03:46:22 -05:00
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
2018-05-29 05:11:52 -04:00
if (toDelete.length === 0) {
console.log('No files to delete.')
return
}
console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
const res = await askConfirmation()
if (res === true) {
console.log('Processing delete...\n')
for (const path of toDelete) {
2018-08-27 10:23:34 -04:00
await remove(path)
2018-05-29 05:11:52 -04:00
}
console.log('Done!')
} else {
console.log('Exiting without deleting files.')
}
}
2019-08-09 09:04:36 -04:00
type ExistFun = (file: string) => Promise<boolean>
async function pruneDirectory (directory: string, existFun: ExistFun) {
2018-08-27 10:23:34 -04:00
const files = await readdir(directory)
2018-05-29 05:11:52 -04:00
const toDelete: string[] = []
2021-08-27 08:32:44 -04:00
await map(files, async file => {
2021-07-23 05:20:00 -04:00
const filePath = join(directory, file)
if (await existFun(filePath) !== true) {
toDelete.push(filePath)
2019-08-09 09:04:36 -04:00
}
}, { concurrency: 20 })
return toDelete
}
2021-07-23 05:20:00 -04:00
function doesWebTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
}
2018-05-29 05:11:52 -04:00
2021-12-10 04:28:46 -05:00
function doesHLSPlaylistExist () {
return (hlsPath: string) => VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
}
2021-07-23 05:20:00 -04:00
function doesTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
2019-08-09 09:04:36 -04:00
}
2018-05-29 05:11:52 -04:00
function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
2021-07-23 05:20:00 -04:00
return async (filePath: string) => {
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
2019-08-09 09:04:36 -04:00
if (!thumbnail) return false
if (keepOnlyOwned) {
const video = await VideoModel.load(thumbnail.videoId)
if (video.isOwned() === false) return false
2018-09-28 04:56:13 -04:00
}
2019-08-09 09:04:36 -04:00
return true
2018-05-29 05:11:52 -04:00
}
2019-08-09 09:04:36 -04:00
}
2018-05-29 05:11:52 -04:00
2021-07-23 05:20:00 -04:00
async function doesActorImageExist (filePath: string) {
const image = await ActorImageModel.loadByName(basename(filePath))
2019-08-09 09:04:36 -04:00
2021-04-06 05:35:56 -04:00
return !!image
2019-08-09 09:04:36 -04:00
}
2021-07-23 05:20:00 -04:00
async function doesRedundancyExist (filePath: string) {
const isPlaylist = (await stat(filePath)).isDirectory()
2019-08-09 09:04:36 -04:00
if (isPlaylist) {
2021-09-02 09:10:30 -04:00
// Don't delete HLS directory
if (filePath === HLS_REDUNDANCY_DIRECTORY) return true
2021-07-23 05:20:00 -04:00
const uuid = getUUIDFromFilename(filePath)
const video = await VideoModel.loadWithFiles(uuid)
if (!video) return false
2019-08-09 09:04:36 -04:00
const p = video.getHLSPlaylist()
if (!p) return false
const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
return !!redundancy
}
2021-07-23 05:20:00 -04:00
const file = await VideoFileModel.loadByFilename(basename(filePath))
if (!file) return false
2019-08-09 09:04:36 -04:00
2021-07-23 05:20:00 -04:00
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
2019-08-09 09:04:36 -04:00
return !!redundancy
2018-05-29 05:11:52 -04:00
}
async function askConfirmation () {
return new Promise((res, rej) => {
2021-08-27 08:32:44 -04:00
start()
2018-05-29 05:11:52 -04:00
const schema = {
properties: {
confirm: {
type: 'string',
2018-09-28 04:56:13 -04:00
description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
' Notice PeerTube must have been stopped when your ran this script.' +
2018-09-28 04:56:13 -04:00
' Can we delete these files?',
2018-05-29 05:11:52 -04:00
default: 'n',
required: true
}
}
}
2021-08-27 08:32:44 -04:00
get(schema, function (err, result) {
2018-05-29 05:11:52 -04:00
if (err) return rej(err)
2020-07-30 08:54:31 -04:00
return res(result.confirm?.match(/y/) !== null)
2018-05-29 05:11:52 -04:00
})
})
}