1
0
Fork 0

Support object storage in prune script

Also prune original files and user exports
This commit is contained in:
Chocobozzz 2024-06-03 16:37:44 +02:00
parent 568a1b1e85
commit 54c140c800
No known key found for this signature in database
GPG key ID: 583A612D890159BE
12 changed files with 618 additions and 295 deletions

View file

@ -53,7 +53,7 @@
"create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js", "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js",
"create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js", "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js",
"parse-log": "node ./dist/scripts/parse-log.js", "parse-log": "node ./dist/scripts/parse-log.js",
"prune-storage": "node ./dist/scripts/prune-storage.js", "prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js",
"test": "bash ./scripts/test.sh", "test": "bash ./scripts/test.sh",
"generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh",
"generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts", "generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts",

View file

@ -1,77 +1,49 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai' import { getAllFiles, wait } from '@peertube/peertube-core-utils'
import { createFile } from 'fs-extra/esm' import { FileStorage, HttpStatusCode, HttpStatusCodeType, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
import { readdir } from 'fs/promises' import { areMockObjectStorageTestsDisabled, buildUUID } from '@peertube/peertube-node-utils'
import { join } from 'path'
import { wait } from '@peertube/peertube-core-utils'
import { buildUUID } from '@peertube/peertube-node-utils'
import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
import { import {
cleanupTests,
CLICommand, CLICommand,
ObjectStorageCommand,
PeerTubeServer,
cleanupTests,
createMultipleServers, createMultipleServers,
doubleFollow, doubleFollow,
killallServers, killallServers,
makeGetRequest, makeGetRequest,
PeerTubeServer, makeRawRequest,
setAccessTokensToServers, setAccessTokensToServers,
setDefaultVideoChannel, setDefaultVideoChannel,
waitJobs waitJobs
} from '@peertube/peertube-server-commands' } from '@peertube/peertube-server-commands'
import { SQLCommand } from '@tests/shared/sql-command.js'
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) { import { expect } from 'chai'
const files = await readdir(server.servers.buildDirectory(directory)) import { createFile } from 'fs-extra/esm'
import { readdir } from 'fs/promises'
for (const f of files) { import { join } from 'path'
expect(f).to.not.contain(substring)
}
}
async function assertCountAreOkay (servers: PeerTubeServer[]) {
for (const server of servers) {
const videosCount = await server.servers.countFiles('web-videos')
expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
const privateVideosCount = await server.servers.countFiles('web-videos/private')
expect(privateVideosCount).to.equal(4)
const torrentsCount = await server.servers.countFiles('torrents')
expect(torrentsCount).to.equal(24)
const previewsCount = await server.servers.countFiles('previews')
expect(previewsCount).to.equal(3)
const thumbnailsCount = await server.servers.countFiles('thumbnails')
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
const avatarsCount = await server.servers.countFiles('avatars')
expect(avatarsCount).to.equal(8)
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
expect(hlsPrivateRootCount).to.equal(1)
}
}
describe('Test prune storage scripts', function () { describe('Test prune storage scripts', function () {
let servers: PeerTubeServer[] let servers: PeerTubeServer[]
const badNames: { [directory: string]: string[] } = {}
before(async function () { before(async function () {
this.timeout(120000) this.timeout(120000)
servers = await createMultipleServers(2, { transcoding: { enabled: true } }) servers = await createMultipleServers(2)
await setAccessTokensToServers(servers) await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers) await setDefaultVideoChannel(servers)
for (const server of servers) { for (const server of servers) {
await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } }) await server.config.enableMinimumTranscoding({ keepOriginal: true })
await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } }) await server.config.enableUserExport()
}
await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) for (const server of servers) {
await server.videos.quickUpload({ name: 'video 1', privacy: VideoPrivacy.PUBLIC })
await server.videos.quickUpload({ name: 'video 2', privacy: VideoPrivacy.PUBLIC })
await server.videos.quickUpload({ name: 'video 3', privacy: VideoPrivacy.PRIVATE })
await server.users.updateMyAvatar({ fixture: 'avatar.png' }) await server.users.updateMyAvatar({ fixture: 'avatar.png' })
@ -85,6 +57,12 @@ describe('Test prune storage scripts', function () {
}) })
} }
for (const server of servers) {
const user = await server.users.getMyInfo()
await server.userExports.request({ userId: user.id, withVideoFiles: false })
}
await doubleFollow(servers[0], servers[1]) await doubleFollow(servers[0], servers[1])
// Lazy load the remote avatars // Lazy load the remote avatars
@ -119,103 +97,259 @@ describe('Test prune storage scripts', function () {
await wait(1000) await wait(1000)
}) })
it('Should have the files on the disk', async function () { describe('On filesystem', function () {
await assertCountAreOkay(servers) const badNames: { [directory: string]: string[] } = {}
})
it('Should create some dirty files', async function () { async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
for (let i = 0; i < 2; i++) { const files = await readdir(server.servers.buildDirectory(directory))
{
const basePublic = servers[0].servers.buildDirectory('web-videos')
const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private'))
const n1 = buildUUID() + '.mp4' for (const f of files) {
const n2 = buildUUID() + '.webm' expect(f).to.not.contain(substring)
await createFile(join(basePublic, n1))
await createFile(join(basePublic, n2))
await createFile(join(basePrivate, n1))
await createFile(join(basePrivate, n2))
badNames['web-videos'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('torrents')
const n1 = buildUUID() + '-240.torrent'
const n2 = buildUUID() + '-480.torrent'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['torrents'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('thumbnails')
const n1 = buildUUID() + '.jpg'
const n2 = buildUUID() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['thumbnails'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('previews')
const n1 = buildUUID() + '.jpg'
const n2 = buildUUID() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['previews'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('avatars')
const n1 = buildUUID() + '.png'
const n2 = buildUUID() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['avatars'] = [ n1, n2 ]
}
{
const directory = join('streaming-playlists', 'hls')
const basePublic = servers[0].servers.buildDirectory(directory)
const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
const n1 = buildUUID()
await createFile(join(basePublic, n1))
await createFile(join(basePrivate, n1))
badNames[directory] = [ n1 ]
} }
} }
})
it('Should run prune storage', async function () { async function assertCountAreOkay () {
this.timeout(30000) for (const server of servers) {
const videosCount = await server.servers.countFiles('web-videos')
expect(videosCount).to.equal(5) // 2 videos with 2 resolutions + private directory
const env = servers[0].cli.getEnv() const privateVideosCount = await server.servers.countFiles('web-videos/private')
await CLICommand.exec(`echo y | ${env} npm run prune-storage`) expect(privateVideosCount).to.equal(2)
})
it('Should have removed files', async function () { const torrentsCount = await server.servers.countFiles('torrents')
await assertCountAreOkay(servers) expect(torrentsCount).to.equal(12)
for (const directory of Object.keys(badNames)) { const previewsCount = await server.servers.countFiles('previews')
for (const name of badNames[directory]) { expect(previewsCount).to.equal(3)
await assertNotExists(servers[0], directory, name)
const thumbnailsCount = await server.servers.countFiles('thumbnails')
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
const avatarsCount = await server.servers.countFiles('avatars')
expect(avatarsCount).to.equal(8)
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
expect(hlsPrivateRootCount).to.equal(1)
const originalVideoFilesCount = await server.servers.countFiles(join('original-video-files'))
expect(originalVideoFilesCount).to.equal(3)
const userExportFilesCount = await server.servers.countFiles(join('tmp-persistent'))
expect(userExportFilesCount).to.equal(1)
} }
} }
it('Should have the files on the disk', async function () {
await assertCountAreOkay()
})
it('Should create some dirty files', async function () {
for (let i = 0; i < 2; i++) {
{
const basePublic = servers[0].servers.buildDirectory('web-videos')
const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private'))
const n1 = buildUUID() + '.mp4'
const n2 = buildUUID() + '.webm'
await createFile(join(basePublic, n1))
await createFile(join(basePublic, n2))
await createFile(join(basePrivate, n1))
await createFile(join(basePrivate, n2))
badNames['web-videos'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('torrents')
const n1 = buildUUID() + '-240.torrent'
const n2 = buildUUID() + '-480.torrent'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['torrents'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('thumbnails')
const n1 = buildUUID() + '.jpg'
const n2 = buildUUID() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['thumbnails'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('previews')
const n1 = buildUUID() + '.jpg'
const n2 = buildUUID() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['previews'] = [ n1, n2 ]
}
{
const base = servers[0].servers.buildDirectory('avatars')
const n1 = buildUUID() + '.png'
const n2 = buildUUID() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['avatars'] = [ n1, n2 ]
}
{
const directory = join('streaming-playlists', 'hls')
const basePublic = servers[0].servers.buildDirectory(directory)
const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
const n1 = buildUUID()
await createFile(join(basePublic, n1))
await createFile(join(basePrivate, n1))
badNames[directory] = [ n1 ]
}
{
const base = servers[0].servers.buildDirectory('original-video-files')
const n1 = buildUUID() + '.mp4'
await createFile(join(base, n1))
badNames['original-video-files'] = [ n1 ]
}
{
const base = servers[0].servers.buildDirectory('tmp-persistent')
const n1 = 'user-export-1.zip'
const n2 = 'user-export-2.zip'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['tmp-persistent'] = [ n1, n2 ]
}
}
})
it('Should run prune storage', async function () {
this.timeout(30000)
const env = servers[0].cli.getEnv()
await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
})
it('Should have removed files', async function () {
await assertCountAreOkay()
for (const directory of Object.keys(badNames)) {
for (const name of badNames[directory]) {
await assertNotExists(servers[0], directory, name)
}
}
})
})
describe('On object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
const videos: string[] = []
const objectStorage = new ObjectStorageCommand()
let sqlCommand: SQLCommand
let rootId: number
async function checkVideosFiles (uuids: string[], expectedStatus: HttpStatusCodeType) {
for (const uuid of uuids) {
const video = await servers[0].videos.getWithToken({ id: uuid })
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.fileUrl, token: servers[0].accessToken, expectedStatus })
}
const source = await servers[0].videos.getSource({ id: uuid })
await makeRawRequest({ url: source.fileDownloadUrl, redirects: 1, token: servers[0].accessToken, expectedStatus })
}
}
async function checkUserExport (expectedStatus: HttpStatusCodeType) {
const { data } = await servers[0].userExports.list({ userId: rootId })
await makeRawRequest({ url: data[0].privateDownloadUrl, redirects: 1, expectedStatus })
}
before(async function () {
this.timeout(120000)
sqlCommand = new SQLCommand(servers[0])
await objectStorage.prepareDefaultMockBuckets()
await servers[0].run(objectStorage.getDefaultMockConfig())
{
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 1', privacy: VideoPrivacy.PUBLIC })
videos.push(uuid)
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 2', privacy: VideoPrivacy.PUBLIC })
videos.push(uuid)
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE })
videos.push(uuid)
}
const user = await servers[0].users.getMyInfo()
rootId = user.id
await servers[0].userExports.deleteAllArchives({ userId: rootId })
await servers[0].userExports.request({ userId: rootId, withVideoFiles: false })
await waitJobs([ servers[0] ])
})
it('Should have the files on object storage', async function () {
await checkVideosFiles(videos, HttpStatusCode.OK_200)
await checkUserExport(HttpStatusCode.OK_200)
})
it('Should run prune-storage script on videos', async function () {
await sqlCommand.setVideoFileStorageOf(videos[1], FileStorage.FILE_SYSTEM)
await sqlCommand.setVideoFileStorageOf(videos[2], FileStorage.FILE_SYSTEM)
await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
await checkUserExport(HttpStatusCode.OK_200)
})
it('Should run prune-storage script on exports', async function () {
await sqlCommand.setUserExportStorageOf(rootId, FileStorage.FILE_SYSTEM)
await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
await checkUserExport(HttpStatusCode.NOT_FOUND_404)
})
after(async function () {
await sqlCommand.cleanup()
})
}) })
after(async function () { after(async function () {

View file

@ -1,6 +1,7 @@
import { QueryTypes, Sequelize } from 'sequelize' import { QueryTypes, Sequelize } from 'sequelize'
import { forceNumber } from '@peertube/peertube-core-utils' import { forceNumber } from '@peertube/peertube-core-utils'
import { PeerTubeServer } from '@peertube/peertube-server-commands' import { PeerTubeServer } from '@peertube/peertube-server-commands'
import { FileStorageType } from '@peertube/peertube-models'
export class SQLCommand { export class SQLCommand {
private sequelize: Sequelize private sequelize: Sequelize
@ -58,6 +59,30 @@ export class SQLCommand {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
async setVideoFileStorageOf (uuid: string, storage: FileStorageType) {
await this.updateQuery(
`UPDATE "videoFile" SET storage = :storage ` +
`WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid) OR ` +
// eslint-disable-next-line max-len
`"videoStreamingPlaylistId" IN (` +
`SELECT "videoStreamingPlaylist".id FROM "videoStreamingPlaylist" ` +
`INNER JOIN video ON video.id = "videoStreamingPlaylist"."videoId" AND "video".uuid = :uuid` +
`)`,
{ storage, uuid }
)
await this.updateQuery(
`UPDATE "videoSource" SET storage = :storage WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`,
{ storage, uuid }
)
}
async setUserExportStorageOf (userId: number, storage: FileStorageType) {
await this.updateQuery(`UPDATE "userExport" SET storage = :storage WHERE "userId" = :userId`, { storage, userId })
}
// ---------------------------------------------------------------------------
setPluginVersion (pluginName: string, newVersion: string) { setPluginVersion (pluginName: string, newVersion: string) {
return this.setPluginField(pluginName, 'version', newVersion) return this.setPluginField(pluginName, 'version', newVersion)
} }

View file

@ -1,11 +1,11 @@
import { context } from '@opentelemetry/api'
import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
import { omit } from '@peertube/peertube-core-utils'
import { stat } from 'fs/promises' import { stat } from 'fs/promises'
import { join } from 'path' import { join } from 'path'
import { format as sqlFormat } from 'sql-formatter' import { format as sqlFormat } from 'sql-formatter'
import { createLogger, format, transports } from 'winston' import { createLogger, format, transports } from 'winston'
import { FileTransportOptions } from 'winston/lib/winston/transports' import { FileTransportOptions } from 'winston/lib/winston/transports'
import { context } from '@opentelemetry/api'
import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
import { omit } from '@peertube/peertube-core-utils'
import { CONFIG } from '../initializers/config.js' import { CONFIG } from '../initializers/config.js'
import { LOG_FILENAME } from '../initializers/constants.js' import { LOG_FILENAME } from '../initializers/constants.js'
@ -60,7 +60,7 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
function buildLogger (labelSuffix?: string) { function buildLogger (labelSuffix?: string) {
return createLogger({ return createLogger({
level: CONFIG.LOG.LEVEL, level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
defaultMeta: { defaultMeta: {
get traceId () { return getSpanContext(context.active())?.traceId }, get traceId () { return getSpanContext(context.active())?.traceId },
get spanId () { return getSpanContext(context.active())?.spanId }, get spanId () { return getSpanContext(context.active())?.spanId },
@ -154,18 +154,10 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export { export {
type LoggerTagsFn,
type LoggerTags,
buildLogger, buildLogger, bunyanLogger, consoleLoggerFormat,
timestampFormatter, jsonLoggerFormat, labelFormatter, logger,
labelFormatter, loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn
consoleLoggerFormat,
jsonLoggerFormat,
mtimeSortFilesDesc,
logger,
loggerTagsFactory,
bunyanLogger
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View file

@ -845,6 +845,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const USER_EXPORT_MAX_ITEMS = 1000 const USER_EXPORT_MAX_ITEMS = 1000
const USER_EXPORT_FILE_PREFIX = 'user-export-'
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -1268,6 +1269,7 @@ export {
CONSTRAINTS_FIELDS, CONSTRAINTS_FIELDS,
EMBED_SIZE, EMBED_SIZE,
REDUNDANCY, REDUNDANCY,
USER_EXPORT_FILE_PREFIX,
JOB_CONCURRENCY, JOB_CONCURRENCY,
JOB_ATTEMPTS, JOB_ATTEMPTS,
AP_CLEANER, AP_CLEANER,

View file

@ -7,9 +7,9 @@ import { createReadStream, createWriteStream } from 'fs'
import { ensureDir } from 'fs-extra/esm' import { ensureDir } from 'fs-extra/esm'
import { dirname } from 'path' import { dirname } from 'path'
import { Readable } from 'stream' import { Readable } from 'stream'
import { getInternalUrl } from '../urls.js' import { getInternalUrl } from './urls.js'
import { getClient } from './client.js' import { getClient } from './shared/client.js'
import { lTags } from './logger.js' import { lTags } from './shared/logger.js'
import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3' import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
@ -18,7 +18,7 @@ type BucketInfo = {
PREFIX?: string PREFIX?: string
} }
async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) {
const s3Client = await getClient() const s3Client = await getClient()
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
@ -26,14 +26,21 @@ async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
const commandPrefix = bucketInfo.PREFIX + prefix const commandPrefix = bucketInfo.PREFIX + prefix
const listCommand = new ListObjectsV2Command({ const listCommand = new ListObjectsV2Command({
Bucket: bucketInfo.BUCKET_NAME, Bucket: bucketInfo.BUCKET_NAME,
Prefix: commandPrefix Prefix: commandPrefix,
ContinuationToken: continuationToken
}) })
const listedObjects = await s3Client.send(listCommand) const listedObjects = await s3Client.send(listCommand)
if (isArray(listedObjects.Contents) !== true) return [] if (isArray(listedObjects.Contents) !== true) return []
return listedObjects.Contents.map(c => c.Key) let keys = listedObjects.Contents.map(c => c.Key)
if (listedObjects.IsTruncated) {
keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken))
}
return keys
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -145,7 +152,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
return removeObjectByFullKey(key, bucketInfo) return removeObjectByFullKey(key, bucketInfo)
} }
async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick<BucketInfo, 'BUCKET_NAME'>) {
logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')

View file

@ -1,3 +1,3 @@
export * from './client.js' export * from './client.js'
export * from './logger.js' export * from './logger.js'
export * from './object-storage-helpers.js' export * from '../object-storage-helpers.js'

View file

@ -1,23 +1,25 @@
import { FindOptions, Op } from 'sequelize' import { FileStorage, UserExportState, type FileStorageType, type UserExport, type UserExportStateType } from '@peertube/peertube-models'
import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
import { UserModel } from './user.js'
import { getSort } from '../shared/sort.js'
import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js' import { logger } from '@server/helpers/logger.js'
import { remove } from 'fs-extra/esm' import { CONFIG } from '@server/initializers/config.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
import { import {
JWT_TOKEN_USER_EXPORT_FILE_LIFETIME, JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
STATIC_DOWNLOAD_PATHS, STATIC_DOWNLOAD_PATHS,
USER_EXPORT_FILE_PREFIX,
USER_EXPORT_STATES, USER_EXPORT_STATES,
WEBSERVER WEBSERVER
} from '@server/initializers/constants.js' } from '@server/initializers/constants.js'
import { join } from 'path'
import jwt from 'jsonwebtoken'
import { CONFIG } from '@server/initializers/config.js'
import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js' import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
import { remove } from 'fs-extra/esm'
import jwt from 'jsonwebtoken'
import { join } from 'path'
import { FindOptions, Op } from 'sequelize'
import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
import { doesExist } from '../shared/query.js'
import { SequelizeModel } from '../shared/sequelize-type.js' import { SequelizeModel } from '../shared/sequelize-type.js'
import { getSort } from '../shared/sort.js'
import { UserModel } from './user.js'
@Table({ @Table({
tableName: 'userExport', tableName: 'userExport',
@ -147,11 +149,20 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
const query = 'SELECT 1 FROM "userExport" ' +
`WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
}
// ---------------------------------------------------------------------------
generateAndSetFilename () { generateAndSetFilename () {
if (!this.userId) throw new Error('Cannot generate filename without userId') if (!this.userId) throw new Error('Cannot generate filename without userId')
if (!this.createdAt) throw new Error('Cannot generate filename without createdAt') if (!this.createdAt) throw new Error('Cannot generate filename without createdAt')
this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip` this.filename = `${USER_EXPORT_FILE_PREFIX}${this.userId}-${this.createdAt.toISOString()}.zip`
} }
canBeSafelyRemoved () { canBeSafelyRemoved () {

View file

@ -289,11 +289,11 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
return doesExist({ sequelize: this.sequelize, query, bind: { filename } }) return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
} }
static async doesOwnedWebVideoFileExist (filename: string) { static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
`WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1` `WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
return doesExist({ sequelize: this.sequelize, query, bind: { filename } }) return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
} }
static loadByFilename (filename: string) { static loadByFilename (filename: string) {

View file

@ -1,12 +1,12 @@
import type { FileStorageType, VideoSource } from '@peertube/peertube-models' import { type FileStorageType, type VideoSource } from '@peertube/peertube-models'
import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js' import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { join } from 'path' import { join } from 'path'
import { Transaction } from 'sequelize' import { Transaction } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
import { SequelizeModel, getSort } from '../shared/index.js' import { SequelizeModel, doesExist, getSort } from '../shared/index.js'
import { getResolutionLabel } from './formatter/video-api-format.js' import { getResolutionLabel } from './formatter/video-api-format.js'
import { VideoModel } from './video.js' import { VideoModel } from './video.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
@Table({ @Table({
tableName: 'videoSource', tableName: 'videoSource',
@ -103,6 +103,18 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
}) })
} }
// ---------------------------------------------------------------------------
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
const query = 'SELECT 1 FROM "videoSource" ' +
'INNER JOIN "video" ON "video"."id" = "videoSource"."videoId" AND "video"."remote" IS FALSE ' +
`WHERE "keptOriginalFilename" = $filename AND "storage" = $storage LIMIT 1`
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
}
// ---------------------------------------------------------------------------
getFileDownloadUrl () { getFileDownloadUrl () {
if (!this.keptOriginalFilename) return null if (!this.keptOriginalFilename) return null

View file

@ -229,13 +229,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
return Object.assign(playlist, { Video: video }) return Object.assign(playlist, { Video: video })
} }
static doesOwnedHLSPlaylistExist (videoUUID: string) { static doesOwnedVideoUUIDExist (videoUUID: string, storage: FileStorageType) {
const query = `SELECT 1 FROM "videoStreamingPlaylist" ` + const query = `SELECT 1 FROM "videoStreamingPlaylist" ` +
`INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` + `INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` +
`AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` + `AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` +
`AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1` `AND "storage" = $storage LIMIT 1`
return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID } }) return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })
} }
assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) { assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {

View file

@ -1,13 +1,16 @@
import { uniqify } from '@peertube/peertube-core-utils'
import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
import { UserExportModel } from '@server/models/user/user-export.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import Bluebird from 'bluebird' import Bluebird from 'bluebird'
import { remove } from 'fs-extra/esm' import { remove } from 'fs-extra/esm'
import { readdir, stat } from 'fs/promises' import { readdir, stat } from 'fs/promises'
import { basename, join } from 'path' import { basename, dirname, join } from 'path'
import prompt from 'prompt' import prompt from 'prompt'
import { uniqify } from '@peertube/peertube-core-utils'
import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
import { DIRECTORIES } from '@server/initializers/constants.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import { getUUIDFromFilename } from '../core/helpers/utils.js' import { getUUIDFromFilename } from '../core/helpers/utils.js'
import { CONFIG } from '../core/initializers/config.js' import { CONFIG } from '../core/initializers/config.js'
import { initDatabaseModels } from '../core/initializers/database.js' import { initDatabaseModels } from '../core/initializers/database.js'
@ -24,141 +27,276 @@ run()
}) })
async function run () { async function run () {
const dirs = Object.values(CONFIG.STORAGE)
if (uniqify(dirs).length !== dirs.length) {
console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
process.exit(0)
}
await initDatabaseModels(true) await initDatabaseModels(true)
let toDelete: string[] = [] await new FSPruner().prune()
console.log('Detecting files to remove, it could take a while...') console.log('\n')
toDelete = toDelete.concat( await new ObjectStoragePruner().prune()
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()), }
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()), // ---------------------------------------------------------------------------
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()), // Object storage
// ---------------------------------------------------------------------------
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()), class ObjectStoragePruner {
private readonly keysToDelete: { bucket: string, key: string }[] = []
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), async prune () {
if (!CONFIG.OBJECT_STORAGE.ENABLED) return
await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)), console.log('Pruning object storage.')
await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist) await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
) await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) if (this.keysToDelete.length === 0) {
toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t))) console.log('No unknown object storage files to delete.')
return
}
if (toDelete.length === 0) { const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
console.log('No files to delete.') console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
return
const res = await askConfirmation()
if (res !== true) {
console.log('Exiting without deleting object storage files.')
return
}
console.log('Deleting object storage files...\n')
for (const { bucket, key } of this.keysToDelete) {
await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
}
console.log(`${this.keysToDelete.length} object storage files deleted.`)
} }
console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n')) private async findFilesToDelete (
config: { BUCKET_NAME: string, PREFIX?: string },
existFun: (file: string) => Promise<boolean> | boolean
) {
try {
const keys = await listKeysOfPrefix('', config)
const res = await askConfirmation() await Bluebird.map(keys, async key => {
if (res === true) { if (await existFun(key) !== true) {
console.log('Processing delete...\n') this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
}
}, { concurrency: 20 })
} catch (err) {
const prefixMessage = config.PREFIX
? ` and prefix ${config.PREFIX}`
: ''
for (const path of toDelete) { console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage)
}
}
private doesWebVideoFileExistFactory () {
return (key: string) => {
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
}
}
private doesStreamingPlaylistFileExistFactory () {
return (key: string) => {
const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)))
return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
}
}
private doesOriginalFileExistFactory () {
return (key: string) => {
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
}
}
private doesUserExportFileExistFactory () {
return (key: string) => {
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
}
}
private sanitizeKey (key: string, config: { PREFIX: string }) {
return key.replace(new RegExp(`^${config.PREFIX}`), '')
}
}
// ---------------------------------------------------------------------------
// FS
// ---------------------------------------------------------------------------
class FSPruner {
private pathsToDelete: string[] = []
async prune () {
const dirs = Object.values(CONFIG.STORAGE)
if (uniqify(dirs).length !== dirs.length) {
console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
process.exit(0)
}
console.log('Pruning filesystem storage.')
console.log('Detecting files to remove, it can take a while...')
await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
if (this.pathsToDelete.length === 0) {
console.log('No unknown filesystem files to delete.')
return
}
const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
const res = await askConfirmation()
if (res !== true) {
console.log('Exiting without deleting filesystem files.')
return
}
console.log('Deleting filesystem files...\n')
for (const path of this.pathsToDelete) {
await remove(path) await remove(path)
} }
console.log('Done!') console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
} else {
console.log('Exiting without deleting files.')
} }
}
type ExistFun = (file: string) => Promise<boolean> | boolean private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
async function pruneDirectory (directory: string, existFun: ExistFun) { const files = await readdir(directory)
const files = await readdir(directory)
const toDelete: string[] = [] await Bluebird.map(files, async file => {
await Bluebird.map(files, async file => { const filePath = join(directory, file)
const filePath = join(directory, file)
if (await existFun(filePath) !== true) { if (await existFun(filePath) !== true) {
toDelete.push(filePath) this.pathsToDelete.push(filePath)
}
}, { concurrency: 20 })
}
private doesWebVideoFileExistFactory () {
return (filePath: string) => {
// Don't delete private directory
if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
} }
}, { concurrency: 20 })
return toDelete
}
function doesWebVideoFileExist () {
return (filePath: string) => {
// Don't delete private directory
if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath))
} }
}
function doesHLSPlaylistExist () { private doesHLSPlaylistExistFactory () {
return (hlsPath: string) => { return (hlsPath: string) => {
// Don't delete private directory // Don't delete private directory
if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath)) return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
}
}
function doesTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
}
function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
return async (filePath: string) => {
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
if (!thumbnail) return false
if (keepOnlyOwned) {
const video = await VideoModel.load(thumbnail.videoId)
if (video.isOwned() === false) return false
} }
return true
}
}
async function doesActorImageExist (filePath: string) {
const image = await ActorImageModel.loadByName(basename(filePath))
return !!image
}
async function doesRedundancyExist (filePath: string) {
const isPlaylist = (await stat(filePath)).isDirectory()
if (isPlaylist) {
// Don't delete HLS redundancy directory
if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
const uuid = getUUIDFromFilename(filePath)
const video = await VideoModel.loadWithFiles(uuid)
if (!video) return false
const p = video.getHLSPlaylist()
if (!p) return false
const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
return !!redundancy
} }
const file = await VideoFileModel.loadByFilename(basename(filePath)) private doesOriginalVideoExistFactory () {
if (!file) return false return (filePath: string) => {
return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
}
}
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) private doesTorrentFileExistFactory () {
return !!redundancy return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
}
private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
return async (filePath: string) => {
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
if (!thumbnail) return false
if (keepOnlyOwned) {
const video = await VideoModel.load(thumbnail.videoId)
if (video.isOwned() === false) return false
}
return true
}
}
private doesActorImageExistFactory () {
return async (filePath: string) => {
const image = await ActorImageModel.loadByName(basename(filePath))
return !!image
}
}
private doesRedundancyExistFactory () {
return async (filePath: string) => {
const isPlaylist = (await stat(filePath)).isDirectory()
if (isPlaylist) {
// Don't delete HLS redundancy directory
if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
const uuid = getUUIDFromFilename(filePath)
const video = await VideoModel.loadWithFiles(uuid)
if (!video) return false
const p = video.getHLSPlaylist()
if (!p) return false
const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
return !!redundancy
}
const file = await VideoFileModel.loadByFilename(basename(filePath))
if (!file) return false
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
return !!redundancy
}
}
private doesUserExportExistFactory () {
return (filePath: string) => {
const filename = basename(filePath)
// Only detect non-existing user export
if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
}
}
} }
async function askConfirmation () { async function askConfirmation () {
@ -169,10 +307,12 @@ async function askConfirmation () {
properties: { properties: {
confirm: { confirm: {
type: 'string', type: 'string',
description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' + description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' +
' Notice PeerTube must have been stopped when your ran this script.' + ' Notice PeerTube must have been stopped when your ran this script.' +
' Can we delete these files?', ' Can we delete these files? (y/n)',
default: 'n', default: 'n',
validator: /y[es]*|n[o]?/,
warning: 'Must respond yes or no',
required: true required: true
} }
} }