Support object storage in prune script
Also prune original files and user exports
This commit is contained in:
parent
568a1b1e85
commit
54c140c800
12 changed files with 618 additions and 295 deletions
|
@ -53,7 +53,7 @@
|
||||||
"create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js",
|
"create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js",
|
||||||
"create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js",
|
"create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js",
|
||||||
"parse-log": "node ./dist/scripts/parse-log.js",
|
"parse-log": "node ./dist/scripts/parse-log.js",
|
||||||
"prune-storage": "node ./dist/scripts/prune-storage.js",
|
"prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js",
|
||||||
"test": "bash ./scripts/test.sh",
|
"test": "bash ./scripts/test.sh",
|
||||||
"generate-cli-doc": "bash ./scripts/generate-cli-doc.sh",
|
"generate-cli-doc": "bash ./scripts/generate-cli-doc.sh",
|
||||||
"generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts",
|
"generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts",
|
||||||
|
|
|
@ -1,77 +1,49 @@
|
||||||
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
||||||
|
|
||||||
import { expect } from 'chai'
|
import { getAllFiles, wait } from '@peertube/peertube-core-utils'
|
||||||
import { createFile } from 'fs-extra/esm'
|
import { FileStorage, HttpStatusCode, HttpStatusCodeType, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
|
||||||
import { readdir } from 'fs/promises'
|
import { areMockObjectStorageTestsDisabled, buildUUID } from '@peertube/peertube-node-utils'
|
||||||
import { join } from 'path'
|
|
||||||
import { wait } from '@peertube/peertube-core-utils'
|
|
||||||
import { buildUUID } from '@peertube/peertube-node-utils'
|
|
||||||
import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
|
|
||||||
import {
|
import {
|
||||||
cleanupTests,
|
|
||||||
CLICommand,
|
CLICommand,
|
||||||
|
ObjectStorageCommand,
|
||||||
|
PeerTubeServer,
|
||||||
|
cleanupTests,
|
||||||
createMultipleServers,
|
createMultipleServers,
|
||||||
doubleFollow,
|
doubleFollow,
|
||||||
killallServers,
|
killallServers,
|
||||||
makeGetRequest,
|
makeGetRequest,
|
||||||
PeerTubeServer,
|
makeRawRequest,
|
||||||
setAccessTokensToServers,
|
setAccessTokensToServers,
|
||||||
setDefaultVideoChannel,
|
setDefaultVideoChannel,
|
||||||
waitJobs
|
waitJobs
|
||||||
} from '@peertube/peertube-server-commands'
|
} from '@peertube/peertube-server-commands'
|
||||||
|
import { SQLCommand } from '@tests/shared/sql-command.js'
|
||||||
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
|
import { expect } from 'chai'
|
||||||
const files = await readdir(server.servers.buildDirectory(directory))
|
import { createFile } from 'fs-extra/esm'
|
||||||
|
import { readdir } from 'fs/promises'
|
||||||
for (const f of files) {
|
import { join } from 'path'
|
||||||
expect(f).to.not.contain(substring)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function assertCountAreOkay (servers: PeerTubeServer[]) {
|
|
||||||
for (const server of servers) {
|
|
||||||
const videosCount = await server.servers.countFiles('web-videos')
|
|
||||||
expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
|
|
||||||
|
|
||||||
const privateVideosCount = await server.servers.countFiles('web-videos/private')
|
|
||||||
expect(privateVideosCount).to.equal(4)
|
|
||||||
|
|
||||||
const torrentsCount = await server.servers.countFiles('torrents')
|
|
||||||
expect(torrentsCount).to.equal(24)
|
|
||||||
|
|
||||||
const previewsCount = await server.servers.countFiles('previews')
|
|
||||||
expect(previewsCount).to.equal(3)
|
|
||||||
|
|
||||||
const thumbnailsCount = await server.servers.countFiles('thumbnails')
|
|
||||||
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
|
|
||||||
|
|
||||||
const avatarsCount = await server.servers.countFiles('avatars')
|
|
||||||
expect(avatarsCount).to.equal(8)
|
|
||||||
|
|
||||||
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
|
|
||||||
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
|
|
||||||
|
|
||||||
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
|
|
||||||
expect(hlsPrivateRootCount).to.equal(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('Test prune storage scripts', function () {
|
describe('Test prune storage scripts', function () {
|
||||||
let servers: PeerTubeServer[]
|
let servers: PeerTubeServer[]
|
||||||
const badNames: { [directory: string]: string[] } = {}
|
|
||||||
|
|
||||||
before(async function () {
|
before(async function () {
|
||||||
this.timeout(120000)
|
this.timeout(120000)
|
||||||
|
|
||||||
servers = await createMultipleServers(2, { transcoding: { enabled: true } })
|
servers = await createMultipleServers(2)
|
||||||
|
|
||||||
await setAccessTokensToServers(servers)
|
await setAccessTokensToServers(servers)
|
||||||
await setDefaultVideoChannel(servers)
|
await setDefaultVideoChannel(servers)
|
||||||
|
|
||||||
for (const server of servers) {
|
for (const server of servers) {
|
||||||
await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
|
await server.config.enableMinimumTranscoding({ keepOriginal: true })
|
||||||
await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
|
await server.config.enableUserExport()
|
||||||
|
}
|
||||||
|
|
||||||
await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
|
for (const server of servers) {
|
||||||
|
await server.videos.quickUpload({ name: 'video 1', privacy: VideoPrivacy.PUBLIC })
|
||||||
|
await server.videos.quickUpload({ name: 'video 2', privacy: VideoPrivacy.PUBLIC })
|
||||||
|
|
||||||
|
await server.videos.quickUpload({ name: 'video 3', privacy: VideoPrivacy.PRIVATE })
|
||||||
|
|
||||||
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
|
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
|
||||||
|
|
||||||
|
@ -85,6 +57,12 @@ describe('Test prune storage scripts', function () {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const server of servers) {
|
||||||
|
const user = await server.users.getMyInfo()
|
||||||
|
|
||||||
|
await server.userExports.request({ userId: user.id, withVideoFiles: false })
|
||||||
|
}
|
||||||
|
|
||||||
await doubleFollow(servers[0], servers[1])
|
await doubleFollow(servers[0], servers[1])
|
||||||
|
|
||||||
// Lazy load the remote avatars
|
// Lazy load the remote avatars
|
||||||
|
@ -119,103 +97,259 @@ describe('Test prune storage scripts', function () {
|
||||||
await wait(1000)
|
await wait(1000)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should have the files on the disk', async function () {
|
describe('On filesystem', function () {
|
||||||
await assertCountAreOkay(servers)
|
const badNames: { [directory: string]: string[] } = {}
|
||||||
})
|
|
||||||
|
|
||||||
it('Should create some dirty files', async function () {
|
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
|
||||||
for (let i = 0; i < 2; i++) {
|
const files = await readdir(server.servers.buildDirectory(directory))
|
||||||
{
|
|
||||||
const basePublic = servers[0].servers.buildDirectory('web-videos')
|
|
||||||
const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private'))
|
|
||||||
|
|
||||||
const n1 = buildUUID() + '.mp4'
|
for (const f of files) {
|
||||||
const n2 = buildUUID() + '.webm'
|
expect(f).to.not.contain(substring)
|
||||||
|
|
||||||
await createFile(join(basePublic, n1))
|
|
||||||
await createFile(join(basePublic, n2))
|
|
||||||
await createFile(join(basePrivate, n1))
|
|
||||||
await createFile(join(basePrivate, n2))
|
|
||||||
|
|
||||||
badNames['web-videos'] = [ n1, n2 ]
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
const base = servers[0].servers.buildDirectory('torrents')
|
|
||||||
|
|
||||||
const n1 = buildUUID() + '-240.torrent'
|
|
||||||
const n2 = buildUUID() + '-480.torrent'
|
|
||||||
|
|
||||||
await createFile(join(base, n1))
|
|
||||||
await createFile(join(base, n2))
|
|
||||||
|
|
||||||
badNames['torrents'] = [ n1, n2 ]
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
const base = servers[0].servers.buildDirectory('thumbnails')
|
|
||||||
|
|
||||||
const n1 = buildUUID() + '.jpg'
|
|
||||||
const n2 = buildUUID() + '.jpg'
|
|
||||||
|
|
||||||
await createFile(join(base, n1))
|
|
||||||
await createFile(join(base, n2))
|
|
||||||
|
|
||||||
badNames['thumbnails'] = [ n1, n2 ]
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
const base = servers[0].servers.buildDirectory('previews')
|
|
||||||
|
|
||||||
const n1 = buildUUID() + '.jpg'
|
|
||||||
const n2 = buildUUID() + '.jpg'
|
|
||||||
|
|
||||||
await createFile(join(base, n1))
|
|
||||||
await createFile(join(base, n2))
|
|
||||||
|
|
||||||
badNames['previews'] = [ n1, n2 ]
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
const base = servers[0].servers.buildDirectory('avatars')
|
|
||||||
|
|
||||||
const n1 = buildUUID() + '.png'
|
|
||||||
const n2 = buildUUID() + '.jpg'
|
|
||||||
|
|
||||||
await createFile(join(base, n1))
|
|
||||||
await createFile(join(base, n2))
|
|
||||||
|
|
||||||
badNames['avatars'] = [ n1, n2 ]
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
const directory = join('streaming-playlists', 'hls')
|
|
||||||
const basePublic = servers[0].servers.buildDirectory(directory)
|
|
||||||
const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
|
|
||||||
|
|
||||||
const n1 = buildUUID()
|
|
||||||
await createFile(join(basePublic, n1))
|
|
||||||
await createFile(join(basePrivate, n1))
|
|
||||||
badNames[directory] = [ n1 ]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
|
|
||||||
it('Should run prune storage', async function () {
|
async function assertCountAreOkay () {
|
||||||
this.timeout(30000)
|
for (const server of servers) {
|
||||||
|
const videosCount = await server.servers.countFiles('web-videos')
|
||||||
|
expect(videosCount).to.equal(5) // 2 videos with 2 resolutions + private directory
|
||||||
|
|
||||||
const env = servers[0].cli.getEnv()
|
const privateVideosCount = await server.servers.countFiles('web-videos/private')
|
||||||
await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
|
expect(privateVideosCount).to.equal(2)
|
||||||
})
|
|
||||||
|
|
||||||
it('Should have removed files', async function () {
|
const torrentsCount = await server.servers.countFiles('torrents')
|
||||||
await assertCountAreOkay(servers)
|
expect(torrentsCount).to.equal(12)
|
||||||
|
|
||||||
for (const directory of Object.keys(badNames)) {
|
const previewsCount = await server.servers.countFiles('previews')
|
||||||
for (const name of badNames[directory]) {
|
expect(previewsCount).to.equal(3)
|
||||||
await assertNotExists(servers[0], directory, name)
|
|
||||||
|
const thumbnailsCount = await server.servers.countFiles('thumbnails')
|
||||||
|
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
|
||||||
|
|
||||||
|
const avatarsCount = await server.servers.countFiles('avatars')
|
||||||
|
expect(avatarsCount).to.equal(8)
|
||||||
|
|
||||||
|
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
|
||||||
|
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
|
||||||
|
|
||||||
|
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
|
||||||
|
expect(hlsPrivateRootCount).to.equal(1)
|
||||||
|
|
||||||
|
const originalVideoFilesCount = await server.servers.countFiles(join('original-video-files'))
|
||||||
|
expect(originalVideoFilesCount).to.equal(3)
|
||||||
|
|
||||||
|
const userExportFilesCount = await server.servers.countFiles(join('tmp-persistent'))
|
||||||
|
expect(userExportFilesCount).to.equal(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
it('Should have the files on the disk', async function () {
|
||||||
|
await assertCountAreOkay()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should create some dirty files', async function () {
|
||||||
|
for (let i = 0; i < 2; i++) {
|
||||||
|
{
|
||||||
|
const basePublic = servers[0].servers.buildDirectory('web-videos')
|
||||||
|
const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private'))
|
||||||
|
|
||||||
|
const n1 = buildUUID() + '.mp4'
|
||||||
|
const n2 = buildUUID() + '.webm'
|
||||||
|
|
||||||
|
await createFile(join(basePublic, n1))
|
||||||
|
await createFile(join(basePublic, n2))
|
||||||
|
await createFile(join(basePrivate, n1))
|
||||||
|
await createFile(join(basePrivate, n2))
|
||||||
|
|
||||||
|
badNames['web-videos'] = [ n1, n2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const base = servers[0].servers.buildDirectory('torrents')
|
||||||
|
|
||||||
|
const n1 = buildUUID() + '-240.torrent'
|
||||||
|
const n2 = buildUUID() + '-480.torrent'
|
||||||
|
|
||||||
|
await createFile(join(base, n1))
|
||||||
|
await createFile(join(base, n2))
|
||||||
|
|
||||||
|
badNames['torrents'] = [ n1, n2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const base = servers[0].servers.buildDirectory('thumbnails')
|
||||||
|
|
||||||
|
const n1 = buildUUID() + '.jpg'
|
||||||
|
const n2 = buildUUID() + '.jpg'
|
||||||
|
|
||||||
|
await createFile(join(base, n1))
|
||||||
|
await createFile(join(base, n2))
|
||||||
|
|
||||||
|
badNames['thumbnails'] = [ n1, n2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const base = servers[0].servers.buildDirectory('previews')
|
||||||
|
|
||||||
|
const n1 = buildUUID() + '.jpg'
|
||||||
|
const n2 = buildUUID() + '.jpg'
|
||||||
|
|
||||||
|
await createFile(join(base, n1))
|
||||||
|
await createFile(join(base, n2))
|
||||||
|
|
||||||
|
badNames['previews'] = [ n1, n2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const base = servers[0].servers.buildDirectory('avatars')
|
||||||
|
|
||||||
|
const n1 = buildUUID() + '.png'
|
||||||
|
const n2 = buildUUID() + '.jpg'
|
||||||
|
|
||||||
|
await createFile(join(base, n1))
|
||||||
|
await createFile(join(base, n2))
|
||||||
|
|
||||||
|
badNames['avatars'] = [ n1, n2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const directory = join('streaming-playlists', 'hls')
|
||||||
|
const basePublic = servers[0].servers.buildDirectory(directory)
|
||||||
|
const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
|
||||||
|
|
||||||
|
const n1 = buildUUID()
|
||||||
|
await createFile(join(basePublic, n1))
|
||||||
|
await createFile(join(basePrivate, n1))
|
||||||
|
badNames[directory] = [ n1 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const base = servers[0].servers.buildDirectory('original-video-files')
|
||||||
|
|
||||||
|
const n1 = buildUUID() + '.mp4'
|
||||||
|
await createFile(join(base, n1))
|
||||||
|
|
||||||
|
badNames['original-video-files'] = [ n1 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const base = servers[0].servers.buildDirectory('tmp-persistent')
|
||||||
|
|
||||||
|
const n1 = 'user-export-1.zip'
|
||||||
|
const n2 = 'user-export-2.zip'
|
||||||
|
|
||||||
|
await createFile(join(base, n1))
|
||||||
|
await createFile(join(base, n2))
|
||||||
|
|
||||||
|
badNames['tmp-persistent'] = [ n1, n2 ]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should run prune storage', async function () {
|
||||||
|
this.timeout(30000)
|
||||||
|
|
||||||
|
const env = servers[0].cli.getEnv()
|
||||||
|
await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should have removed files', async function () {
|
||||||
|
await assertCountAreOkay()
|
||||||
|
|
||||||
|
for (const directory of Object.keys(badNames)) {
|
||||||
|
for (const name of badNames[directory]) {
|
||||||
|
await assertNotExists(servers[0], directory, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('On object storage', function () {
|
||||||
|
if (areMockObjectStorageTestsDisabled()) return
|
||||||
|
|
||||||
|
const videos: string[] = []
|
||||||
|
const objectStorage = new ObjectStorageCommand()
|
||||||
|
|
||||||
|
let sqlCommand: SQLCommand
|
||||||
|
let rootId: number
|
||||||
|
|
||||||
|
async function checkVideosFiles (uuids: string[], expectedStatus: HttpStatusCodeType) {
|
||||||
|
for (const uuid of uuids) {
|
||||||
|
const video = await servers[0].videos.getWithToken({ id: uuid })
|
||||||
|
|
||||||
|
for (const file of getAllFiles(video)) {
|
||||||
|
await makeRawRequest({ url: file.fileUrl, token: servers[0].accessToken, expectedStatus })
|
||||||
|
}
|
||||||
|
|
||||||
|
const source = await servers[0].videos.getSource({ id: uuid })
|
||||||
|
await makeRawRequest({ url: source.fileDownloadUrl, redirects: 1, token: servers[0].accessToken, expectedStatus })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function checkUserExport (expectedStatus: HttpStatusCodeType) {
|
||||||
|
const { data } = await servers[0].userExports.list({ userId: rootId })
|
||||||
|
await makeRawRequest({ url: data[0].privateDownloadUrl, redirects: 1, expectedStatus })
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
this.timeout(120000)
|
||||||
|
|
||||||
|
sqlCommand = new SQLCommand(servers[0])
|
||||||
|
|
||||||
|
await objectStorage.prepareDefaultMockBuckets()
|
||||||
|
|
||||||
|
await servers[0].run(objectStorage.getDefaultMockConfig())
|
||||||
|
|
||||||
|
{
|
||||||
|
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 1', privacy: VideoPrivacy.PUBLIC })
|
||||||
|
videos.push(uuid)
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 2', privacy: VideoPrivacy.PUBLIC })
|
||||||
|
videos.push(uuid)
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE })
|
||||||
|
videos.push(uuid)
|
||||||
|
}
|
||||||
|
|
||||||
|
const user = await servers[0].users.getMyInfo()
|
||||||
|
rootId = user.id
|
||||||
|
|
||||||
|
await servers[0].userExports.deleteAllArchives({ userId: rootId })
|
||||||
|
await servers[0].userExports.request({ userId: rootId, withVideoFiles: false })
|
||||||
|
|
||||||
|
await waitJobs([ servers[0] ])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should have the files on object storage', async function () {
|
||||||
|
await checkVideosFiles(videos, HttpStatusCode.OK_200)
|
||||||
|
await checkUserExport(HttpStatusCode.OK_200)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should run prune-storage script on videos', async function () {
|
||||||
|
await sqlCommand.setVideoFileStorageOf(videos[1], FileStorage.FILE_SYSTEM)
|
||||||
|
await sqlCommand.setVideoFileStorageOf(videos[2], FileStorage.FILE_SYSTEM)
|
||||||
|
|
||||||
|
await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
|
||||||
|
await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
|
||||||
|
|
||||||
|
await checkUserExport(HttpStatusCode.OK_200)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should run prune-storage script on exports', async function () {
|
||||||
|
await sqlCommand.setUserExportStorageOf(rootId, FileStorage.FILE_SYSTEM)
|
||||||
|
|
||||||
|
await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
|
||||||
|
await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
|
||||||
|
|
||||||
|
await checkUserExport(HttpStatusCode.NOT_FOUND_404)
|
||||||
|
})
|
||||||
|
|
||||||
|
after(async function () {
|
||||||
|
await sqlCommand.cleanup()
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
after(async function () {
|
after(async function () {
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import { QueryTypes, Sequelize } from 'sequelize'
|
import { QueryTypes, Sequelize } from 'sequelize'
|
||||||
import { forceNumber } from '@peertube/peertube-core-utils'
|
import { forceNumber } from '@peertube/peertube-core-utils'
|
||||||
import { PeerTubeServer } from '@peertube/peertube-server-commands'
|
import { PeerTubeServer } from '@peertube/peertube-server-commands'
|
||||||
|
import { FileStorageType } from '@peertube/peertube-models'
|
||||||
|
|
||||||
export class SQLCommand {
|
export class SQLCommand {
|
||||||
private sequelize: Sequelize
|
private sequelize: Sequelize
|
||||||
|
@ -58,6 +59,30 @@ export class SQLCommand {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
async setVideoFileStorageOf (uuid: string, storage: FileStorageType) {
|
||||||
|
await this.updateQuery(
|
||||||
|
`UPDATE "videoFile" SET storage = :storage ` +
|
||||||
|
`WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid) OR ` +
|
||||||
|
// eslint-disable-next-line max-len
|
||||||
|
`"videoStreamingPlaylistId" IN (` +
|
||||||
|
`SELECT "videoStreamingPlaylist".id FROM "videoStreamingPlaylist" ` +
|
||||||
|
`INNER JOIN video ON video.id = "videoStreamingPlaylist"."videoId" AND "video".uuid = :uuid` +
|
||||||
|
`)`,
|
||||||
|
{ storage, uuid }
|
||||||
|
)
|
||||||
|
|
||||||
|
await this.updateQuery(
|
||||||
|
`UPDATE "videoSource" SET storage = :storage WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`,
|
||||||
|
{ storage, uuid }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async setUserExportStorageOf (userId: number, storage: FileStorageType) {
|
||||||
|
await this.updateQuery(`UPDATE "userExport" SET storage = :storage WHERE "userId" = :userId`, { storage, userId })
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
setPluginVersion (pluginName: string, newVersion: string) {
|
setPluginVersion (pluginName: string, newVersion: string) {
|
||||||
return this.setPluginField(pluginName, 'version', newVersion)
|
return this.setPluginField(pluginName, 'version', newVersion)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
|
import { context } from '@opentelemetry/api'
|
||||||
|
import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
|
||||||
|
import { omit } from '@peertube/peertube-core-utils'
|
||||||
import { stat } from 'fs/promises'
|
import { stat } from 'fs/promises'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { format as sqlFormat } from 'sql-formatter'
|
import { format as sqlFormat } from 'sql-formatter'
|
||||||
import { createLogger, format, transports } from 'winston'
|
import { createLogger, format, transports } from 'winston'
|
||||||
import { FileTransportOptions } from 'winston/lib/winston/transports'
|
import { FileTransportOptions } from 'winston/lib/winston/transports'
|
||||||
import { context } from '@opentelemetry/api'
|
|
||||||
import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
|
|
||||||
import { omit } from '@peertube/peertube-core-utils'
|
|
||||||
import { CONFIG } from '../initializers/config.js'
|
import { CONFIG } from '../initializers/config.js'
|
||||||
import { LOG_FILENAME } from '../initializers/constants.js'
|
import { LOG_FILENAME } from '../initializers/constants.js'
|
||||||
|
|
||||||
|
@ -60,7 +60,7 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
|
||||||
|
|
||||||
function buildLogger (labelSuffix?: string) {
|
function buildLogger (labelSuffix?: string) {
|
||||||
return createLogger({
|
return createLogger({
|
||||||
level: CONFIG.LOG.LEVEL,
|
level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
|
||||||
defaultMeta: {
|
defaultMeta: {
|
||||||
get traceId () { return getSpanContext(context.active())?.traceId },
|
get traceId () { return getSpanContext(context.active())?.traceId },
|
||||||
get spanId () { return getSpanContext(context.active())?.spanId },
|
get spanId () { return getSpanContext(context.active())?.spanId },
|
||||||
|
@ -154,18 +154,10 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
export {
|
export {
|
||||||
type LoggerTagsFn,
|
|
||||||
type LoggerTags,
|
|
||||||
|
|
||||||
buildLogger,
|
buildLogger, bunyanLogger, consoleLoggerFormat,
|
||||||
timestampFormatter,
|
jsonLoggerFormat, labelFormatter, logger,
|
||||||
labelFormatter,
|
loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn
|
||||||
consoleLoggerFormat,
|
|
||||||
jsonLoggerFormat,
|
|
||||||
mtimeSortFilesDesc,
|
|
||||||
logger,
|
|
||||||
loggerTagsFactory,
|
|
||||||
bunyanLogger
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
|
@ -845,6 +845,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const USER_EXPORT_MAX_ITEMS = 1000
|
const USER_EXPORT_MAX_ITEMS = 1000
|
||||||
|
const USER_EXPORT_FILE_PREFIX = 'user-export-'
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -1268,6 +1269,7 @@ export {
|
||||||
CONSTRAINTS_FIELDS,
|
CONSTRAINTS_FIELDS,
|
||||||
EMBED_SIZE,
|
EMBED_SIZE,
|
||||||
REDUNDANCY,
|
REDUNDANCY,
|
||||||
|
USER_EXPORT_FILE_PREFIX,
|
||||||
JOB_CONCURRENCY,
|
JOB_CONCURRENCY,
|
||||||
JOB_ATTEMPTS,
|
JOB_ATTEMPTS,
|
||||||
AP_CLEANER,
|
AP_CLEANER,
|
||||||
|
|
|
@ -7,9 +7,9 @@ import { createReadStream, createWriteStream } from 'fs'
|
||||||
import { ensureDir } from 'fs-extra/esm'
|
import { ensureDir } from 'fs-extra/esm'
|
||||||
import { dirname } from 'path'
|
import { dirname } from 'path'
|
||||||
import { Readable } from 'stream'
|
import { Readable } from 'stream'
|
||||||
import { getInternalUrl } from '../urls.js'
|
import { getInternalUrl } from './urls.js'
|
||||||
import { getClient } from './client.js'
|
import { getClient } from './shared/client.js'
|
||||||
import { lTags } from './logger.js'
|
import { lTags } from './shared/logger.js'
|
||||||
|
|
||||||
import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
|
import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
|
||||||
|
|
||||||
|
@ -18,7 +18,7 @@ type BucketInfo = {
|
||||||
PREFIX?: string
|
PREFIX?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
|
async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) {
|
||||||
const s3Client = await getClient()
|
const s3Client = await getClient()
|
||||||
|
|
||||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||||
|
@ -26,14 +26,21 @@ async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
|
||||||
const commandPrefix = bucketInfo.PREFIX + prefix
|
const commandPrefix = bucketInfo.PREFIX + prefix
|
||||||
const listCommand = new ListObjectsV2Command({
|
const listCommand = new ListObjectsV2Command({
|
||||||
Bucket: bucketInfo.BUCKET_NAME,
|
Bucket: bucketInfo.BUCKET_NAME,
|
||||||
Prefix: commandPrefix
|
Prefix: commandPrefix,
|
||||||
|
ContinuationToken: continuationToken
|
||||||
})
|
})
|
||||||
|
|
||||||
const listedObjects = await s3Client.send(listCommand)
|
const listedObjects = await s3Client.send(listCommand)
|
||||||
|
|
||||||
if (isArray(listedObjects.Contents) !== true) return []
|
if (isArray(listedObjects.Contents) !== true) return []
|
||||||
|
|
||||||
return listedObjects.Contents.map(c => c.Key)
|
let keys = listedObjects.Contents.map(c => c.Key)
|
||||||
|
|
||||||
|
if (listedObjects.IsTruncated) {
|
||||||
|
keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken))
|
||||||
|
}
|
||||||
|
|
||||||
|
return keys
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
@ -145,7 +152,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
|
||||||
return removeObjectByFullKey(key, bucketInfo)
|
return removeObjectByFullKey(key, bucketInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
|
async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick<BucketInfo, 'BUCKET_NAME'>) {
|
||||||
logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
|
logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
|
||||||
|
|
||||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
|
@ -1,3 +1,3 @@
|
||||||
export * from './client.js'
|
export * from './client.js'
|
||||||
export * from './logger.js'
|
export * from './logger.js'
|
||||||
export * from './object-storage-helpers.js'
|
export * from '../object-storage-helpers.js'
|
||||||
|
|
|
@ -1,23 +1,25 @@
|
||||||
import { FindOptions, Op } from 'sequelize'
|
import { FileStorage, UserExportState, type FileStorageType, type UserExport, type UserExportStateType } from '@peertube/peertube-models'
|
||||||
import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
|
||||||
import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
|
|
||||||
import { UserModel } from './user.js'
|
|
||||||
import { getSort } from '../shared/sort.js'
|
|
||||||
import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models'
|
|
||||||
import { logger } from '@server/helpers/logger.js'
|
import { logger } from '@server/helpers/logger.js'
|
||||||
import { remove } from 'fs-extra/esm'
|
import { CONFIG } from '@server/initializers/config.js'
|
||||||
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
|
||||||
import {
|
import {
|
||||||
JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
|
JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
|
||||||
STATIC_DOWNLOAD_PATHS,
|
STATIC_DOWNLOAD_PATHS,
|
||||||
|
USER_EXPORT_FILE_PREFIX,
|
||||||
USER_EXPORT_STATES,
|
USER_EXPORT_STATES,
|
||||||
WEBSERVER
|
WEBSERVER
|
||||||
} from '@server/initializers/constants.js'
|
} from '@server/initializers/constants.js'
|
||||||
import { join } from 'path'
|
|
||||||
import jwt from 'jsonwebtoken'
|
|
||||||
import { CONFIG } from '@server/initializers/config.js'
|
|
||||||
import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js'
|
import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js'
|
||||||
|
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
||||||
|
import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
|
||||||
|
import { remove } from 'fs-extra/esm'
|
||||||
|
import jwt from 'jsonwebtoken'
|
||||||
|
import { join } from 'path'
|
||||||
|
import { FindOptions, Op } from 'sequelize'
|
||||||
|
import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
||||||
|
import { doesExist } from '../shared/query.js'
|
||||||
import { SequelizeModel } from '../shared/sequelize-type.js'
|
import { SequelizeModel } from '../shared/sequelize-type.js'
|
||||||
|
import { getSort } from '../shared/sort.js'
|
||||||
|
import { UserModel } from './user.js'
|
||||||
|
|
||||||
@Table({
|
@Table({
|
||||||
tableName: 'userExport',
|
tableName: 'userExport',
|
||||||
|
@ -147,11 +149,20 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
|
||||||
|
const query = 'SELECT 1 FROM "userExport" ' +
|
||||||
|
`WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
|
||||||
|
|
||||||
|
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
generateAndSetFilename () {
|
generateAndSetFilename () {
|
||||||
if (!this.userId) throw new Error('Cannot generate filename without userId')
|
if (!this.userId) throw new Error('Cannot generate filename without userId')
|
||||||
if (!this.createdAt) throw new Error('Cannot generate filename without createdAt')
|
if (!this.createdAt) throw new Error('Cannot generate filename without createdAt')
|
||||||
|
|
||||||
this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip`
|
this.filename = `${USER_EXPORT_FILE_PREFIX}${this.userId}-${this.createdAt.toISOString()}.zip`
|
||||||
}
|
}
|
||||||
|
|
||||||
canBeSafelyRemoved () {
|
canBeSafelyRemoved () {
|
||||||
|
|
|
@ -289,11 +289,11 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
|
||||||
return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
|
return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
|
||||||
}
|
}
|
||||||
|
|
||||||
static async doesOwnedWebVideoFileExist (filename: string) {
|
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
|
||||||
const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
|
const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
|
||||||
`WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
|
`WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
|
||||||
|
|
||||||
return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
|
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
|
||||||
}
|
}
|
||||||
|
|
||||||
static loadByFilename (filename: string) {
|
static loadByFilename (filename: string) {
|
||||||
|
|
|
@ -1,12 +1,12 @@
|
||||||
import type { FileStorageType, VideoSource } from '@peertube/peertube-models'
|
import { type FileStorageType, type VideoSource } from '@peertube/peertube-models'
|
||||||
import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
|
import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
|
||||||
|
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||||
import { join } from 'path'
|
import { join } from 'path'
|
||||||
import { Transaction } from 'sequelize'
|
import { Transaction } from 'sequelize'
|
||||||
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
||||||
import { SequelizeModel, getSort } from '../shared/index.js'
|
import { SequelizeModel, doesExist, getSort } from '../shared/index.js'
|
||||||
import { getResolutionLabel } from './formatter/video-api-format.js'
|
import { getResolutionLabel } from './formatter/video-api-format.js'
|
||||||
import { VideoModel } from './video.js'
|
import { VideoModel } from './video.js'
|
||||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
|
||||||
|
|
||||||
@Table({
|
@Table({
|
||||||
tableName: 'videoSource',
|
tableName: 'videoSource',
|
||||||
|
@ -103,6 +103,18 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
|
||||||
|
const query = 'SELECT 1 FROM "videoSource" ' +
|
||||||
|
'INNER JOIN "video" ON "video"."id" = "videoSource"."videoId" AND "video"."remote" IS FALSE ' +
|
||||||
|
`WHERE "keptOriginalFilename" = $filename AND "storage" = $storage LIMIT 1`
|
||||||
|
|
||||||
|
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
getFileDownloadUrl () {
|
getFileDownloadUrl () {
|
||||||
if (!this.keptOriginalFilename) return null
|
if (!this.keptOriginalFilename) return null
|
||||||
|
|
||||||
|
|
|
@ -229,13 +229,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
|
||||||
return Object.assign(playlist, { Video: video })
|
return Object.assign(playlist, { Video: video })
|
||||||
}
|
}
|
||||||
|
|
||||||
static doesOwnedHLSPlaylistExist (videoUUID: string) {
|
static doesOwnedVideoUUIDExist (videoUUID: string, storage: FileStorageType) {
|
||||||
const query = `SELECT 1 FROM "videoStreamingPlaylist" ` +
|
const query = `SELECT 1 FROM "videoStreamingPlaylist" ` +
|
||||||
`INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` +
|
`INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` +
|
||||||
`AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` +
|
`AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` +
|
||||||
`AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
|
`AND "storage" = $storage LIMIT 1`
|
||||||
|
|
||||||
return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID } })
|
return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })
|
||||||
}
|
}
|
||||||
|
|
||||||
assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
|
assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
|
||||||
|
|
|
@ -1,13 +1,16 @@
|
||||||
|
import { uniqify } from '@peertube/peertube-core-utils'
|
||||||
|
import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
|
||||||
|
import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
|
||||||
|
import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
|
||||||
|
import { UserExportModel } from '@server/models/user/user-export.js'
|
||||||
|
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||||
|
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||||
|
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||||
import Bluebird from 'bluebird'
|
import Bluebird from 'bluebird'
|
||||||
import { remove } from 'fs-extra/esm'
|
import { remove } from 'fs-extra/esm'
|
||||||
import { readdir, stat } from 'fs/promises'
|
import { readdir, stat } from 'fs/promises'
|
||||||
import { basename, join } from 'path'
|
import { basename, dirname, join } from 'path'
|
||||||
import prompt from 'prompt'
|
import prompt from 'prompt'
|
||||||
import { uniqify } from '@peertube/peertube-core-utils'
|
|
||||||
import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
|
|
||||||
import { DIRECTORIES } from '@server/initializers/constants.js'
|
|
||||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
|
||||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
|
||||||
import { getUUIDFromFilename } from '../core/helpers/utils.js'
|
import { getUUIDFromFilename } from '../core/helpers/utils.js'
|
||||||
import { CONFIG } from '../core/initializers/config.js'
|
import { CONFIG } from '../core/initializers/config.js'
|
||||||
import { initDatabaseModels } from '../core/initializers/database.js'
|
import { initDatabaseModels } from '../core/initializers/database.js'
|
||||||
|
@ -24,141 +27,276 @@ run()
|
||||||
})
|
})
|
||||||
|
|
||||||
async function run () {
|
async function run () {
|
||||||
const dirs = Object.values(CONFIG.STORAGE)
|
|
||||||
|
|
||||||
if (uniqify(dirs).length !== dirs.length) {
|
|
||||||
console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
|
|
||||||
process.exit(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
await initDatabaseModels(true)
|
await initDatabaseModels(true)
|
||||||
|
|
||||||
let toDelete: string[] = []
|
await new FSPruner().prune()
|
||||||
|
|
||||||
console.log('Detecting files to remove, it could take a while...')
|
console.log('\n')
|
||||||
|
|
||||||
toDelete = toDelete.concat(
|
await new ObjectStoragePruner().prune()
|
||||||
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()),
|
}
|
||||||
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()),
|
|
||||||
|
|
||||||
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
|
// ---------------------------------------------------------------------------
|
||||||
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
|
// Object storage
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
|
class ObjectStoragePruner {
|
||||||
|
private readonly keysToDelete: { bucket: string, key: string }[] = []
|
||||||
|
|
||||||
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
|
async prune () {
|
||||||
|
if (!CONFIG.OBJECT_STORAGE.ENABLED) return
|
||||||
|
|
||||||
await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
|
console.log('Pruning object storage.')
|
||||||
await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
|
|
||||||
|
|
||||||
await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist)
|
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
|
||||||
)
|
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
|
||||||
|
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
|
||||||
|
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
|
||||||
|
|
||||||
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
|
if (this.keysToDelete.length === 0) {
|
||||||
toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
|
console.log('No unknown object storage files to delete.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (toDelete.length === 0) {
|
const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
|
||||||
console.log('No files to delete.')
|
console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
|
||||||
return
|
|
||||||
|
const res = await askConfirmation()
|
||||||
|
if (res !== true) {
|
||||||
|
console.log('Exiting without deleting object storage files.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Deleting object storage files...\n')
|
||||||
|
|
||||||
|
for (const { bucket, key } of this.keysToDelete) {
|
||||||
|
await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`${this.keysToDelete.length} object storage files deleted.`)
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
|
private async findFilesToDelete (
|
||||||
|
config: { BUCKET_NAME: string, PREFIX?: string },
|
||||||
|
existFun: (file: string) => Promise<boolean> | boolean
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const keys = await listKeysOfPrefix('', config)
|
||||||
|
|
||||||
const res = await askConfirmation()
|
await Bluebird.map(keys, async key => {
|
||||||
if (res === true) {
|
if (await existFun(key) !== true) {
|
||||||
console.log('Processing delete...\n')
|
this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
|
||||||
|
}
|
||||||
|
}, { concurrency: 20 })
|
||||||
|
} catch (err) {
|
||||||
|
const prefixMessage = config.PREFIX
|
||||||
|
? ` and prefix ${config.PREFIX}`
|
||||||
|
: ''
|
||||||
|
|
||||||
for (const path of toDelete) {
|
console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesWebVideoFileExistFactory () {
|
||||||
|
return (key: string) => {
|
||||||
|
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
|
||||||
|
|
||||||
|
return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesStreamingPlaylistFileExistFactory () {
|
||||||
|
return (key: string) => {
|
||||||
|
const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)))
|
||||||
|
|
||||||
|
return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesOriginalFileExistFactory () {
|
||||||
|
return (key: string) => {
|
||||||
|
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
|
||||||
|
|
||||||
|
return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesUserExportFileExistFactory () {
|
||||||
|
return (key: string) => {
|
||||||
|
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
|
||||||
|
|
||||||
|
return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sanitizeKey (key: string, config: { PREFIX: string }) {
|
||||||
|
return key.replace(new RegExp(`^${config.PREFIX}`), '')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// FS
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class FSPruner {
|
||||||
|
private pathsToDelete: string[] = []
|
||||||
|
|
||||||
|
async prune () {
|
||||||
|
const dirs = Object.values(CONFIG.STORAGE)
|
||||||
|
|
||||||
|
if (uniqify(dirs).length !== dirs.length) {
|
||||||
|
console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Pruning filesystem storage.')
|
||||||
|
|
||||||
|
console.log('Detecting files to remove, it can take a while...')
|
||||||
|
|
||||||
|
await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
|
||||||
|
await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
|
||||||
|
|
||||||
|
await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
|
||||||
|
await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
|
||||||
|
|
||||||
|
await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
|
||||||
|
|
||||||
|
await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
|
||||||
|
|
||||||
|
await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
|
||||||
|
|
||||||
|
await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
|
||||||
|
await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
|
||||||
|
|
||||||
|
await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
|
||||||
|
|
||||||
|
await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
|
||||||
|
|
||||||
|
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
|
||||||
|
this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
|
||||||
|
|
||||||
|
if (this.pathsToDelete.length === 0) {
|
||||||
|
console.log('No unknown filesystem files to delete.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
|
||||||
|
console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
|
||||||
|
|
||||||
|
const res = await askConfirmation()
|
||||||
|
if (res !== true) {
|
||||||
|
console.log('Exiting without deleting filesystem files.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Deleting filesystem files...\n')
|
||||||
|
|
||||||
|
for (const path of this.pathsToDelete) {
|
||||||
await remove(path)
|
await remove(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('Done!')
|
console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
|
||||||
} else {
|
|
||||||
console.log('Exiting without deleting files.')
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
type ExistFun = (file: string) => Promise<boolean> | boolean
|
private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
|
||||||
async function pruneDirectory (directory: string, existFun: ExistFun) {
|
const files = await readdir(directory)
|
||||||
const files = await readdir(directory)
|
|
||||||
|
|
||||||
const toDelete: string[] = []
|
await Bluebird.map(files, async file => {
|
||||||
await Bluebird.map(files, async file => {
|
const filePath = join(directory, file)
|
||||||
const filePath = join(directory, file)
|
|
||||||
|
|
||||||
if (await existFun(filePath) !== true) {
|
if (await existFun(filePath) !== true) {
|
||||||
toDelete.push(filePath)
|
this.pathsToDelete.push(filePath)
|
||||||
|
}
|
||||||
|
}, { concurrency: 20 })
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesWebVideoFileExistFactory () {
|
||||||
|
return (filePath: string) => {
|
||||||
|
// Don't delete private directory
|
||||||
|
if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
|
||||||
|
|
||||||
|
return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
|
||||||
}
|
}
|
||||||
}, { concurrency: 20 })
|
|
||||||
|
|
||||||
return toDelete
|
|
||||||
}
|
|
||||||
|
|
||||||
function doesWebVideoFileExist () {
|
|
||||||
return (filePath: string) => {
|
|
||||||
// Don't delete private directory
|
|
||||||
if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
|
|
||||||
|
|
||||||
return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath))
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
function doesHLSPlaylistExist () {
|
private doesHLSPlaylistExistFactory () {
|
||||||
return (hlsPath: string) => {
|
return (hlsPath: string) => {
|
||||||
// Don't delete private directory
|
// Don't delete private directory
|
||||||
if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
|
if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
|
||||||
|
|
||||||
return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
|
return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function doesTorrentFileExist () {
|
|
||||||
return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
|
|
||||||
}
|
|
||||||
|
|
||||||
function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
|
|
||||||
return async (filePath: string) => {
|
|
||||||
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
|
|
||||||
if (!thumbnail) return false
|
|
||||||
|
|
||||||
if (keepOnlyOwned) {
|
|
||||||
const video = await VideoModel.load(thumbnail.videoId)
|
|
||||||
if (video.isOwned() === false) return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function doesActorImageExist (filePath: string) {
|
|
||||||
const image = await ActorImageModel.loadByName(basename(filePath))
|
|
||||||
|
|
||||||
return !!image
|
|
||||||
}
|
|
||||||
|
|
||||||
async function doesRedundancyExist (filePath: string) {
|
|
||||||
const isPlaylist = (await stat(filePath)).isDirectory()
|
|
||||||
|
|
||||||
if (isPlaylist) {
|
|
||||||
// Don't delete HLS redundancy directory
|
|
||||||
if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
|
|
||||||
|
|
||||||
const uuid = getUUIDFromFilename(filePath)
|
|
||||||
const video = await VideoModel.loadWithFiles(uuid)
|
|
||||||
if (!video) return false
|
|
||||||
|
|
||||||
const p = video.getHLSPlaylist()
|
|
||||||
if (!p) return false
|
|
||||||
|
|
||||||
const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
|
|
||||||
return !!redundancy
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const file = await VideoFileModel.loadByFilename(basename(filePath))
|
private doesOriginalVideoExistFactory () {
|
||||||
if (!file) return false
|
return (filePath: string) => {
|
||||||
|
return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
|
private doesTorrentFileExistFactory () {
|
||||||
return !!redundancy
|
return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
|
||||||
|
return async (filePath: string) => {
|
||||||
|
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
|
||||||
|
if (!thumbnail) return false
|
||||||
|
|
||||||
|
if (keepOnlyOwned) {
|
||||||
|
const video = await VideoModel.load(thumbnail.videoId)
|
||||||
|
if (video.isOwned() === false) return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesActorImageExistFactory () {
|
||||||
|
return async (filePath: string) => {
|
||||||
|
const image = await ActorImageModel.loadByName(basename(filePath))
|
||||||
|
|
||||||
|
return !!image
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesRedundancyExistFactory () {
|
||||||
|
return async (filePath: string) => {
|
||||||
|
const isPlaylist = (await stat(filePath)).isDirectory()
|
||||||
|
|
||||||
|
if (isPlaylist) {
|
||||||
|
// Don't delete HLS redundancy directory
|
||||||
|
if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
|
||||||
|
|
||||||
|
const uuid = getUUIDFromFilename(filePath)
|
||||||
|
const video = await VideoModel.loadWithFiles(uuid)
|
||||||
|
if (!video) return false
|
||||||
|
|
||||||
|
const p = video.getHLSPlaylist()
|
||||||
|
if (!p) return false
|
||||||
|
|
||||||
|
const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
|
||||||
|
return !!redundancy
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = await VideoFileModel.loadByFilename(basename(filePath))
|
||||||
|
if (!file) return false
|
||||||
|
|
||||||
|
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
|
||||||
|
return !!redundancy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private doesUserExportExistFactory () {
|
||||||
|
return (filePath: string) => {
|
||||||
|
const filename = basename(filePath)
|
||||||
|
|
||||||
|
// Only detect non-existing user export
|
||||||
|
if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
|
||||||
|
|
||||||
|
return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function askConfirmation () {
|
async function askConfirmation () {
|
||||||
|
@ -169,10 +307,12 @@ async function askConfirmation () {
|
||||||
properties: {
|
properties: {
|
||||||
confirm: {
|
confirm: {
|
||||||
type: 'string',
|
type: 'string',
|
||||||
description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
|
description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' +
|
||||||
' Notice PeerTube must have been stopped when your ran this script.' +
|
' Notice PeerTube must have been stopped when your ran this script.' +
|
||||||
' Can we delete these files?',
|
' Can we delete these files? (y/n)',
|
||||||
default: 'n',
|
default: 'n',
|
||||||
|
validator: /y[es]*|n[o]?/,
|
||||||
|
warning: 'Must respond yes or no',
|
||||||
required: true
|
required: true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue