Support object storage in prune script
Also prune original files and user exports
This commit is contained in:
Chocobozzz
parent
568a1b1e85
commit
54c140c800
12 changed files with 618 additions and 295 deletions
|
|
@ -53,7 +53,7 @@
|
|||
"create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js",
|
||||
"create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js",
|
||||
"parse-log": "node ./dist/scripts/parse-log.js",
|
||||
"prune-storage": "node ./dist/scripts/prune-storage.js",
|
||||
"prune-storage": "LOGGER_LEVEL=warn node ./dist/scripts/prune-storage.js",
|
||||
"test": "bash ./scripts/test.sh",
|
||||
"generate-cli-doc": "bash ./scripts/generate-cli-doc.sh",
|
||||
"generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts",
|
||||
|
|
|
|||
|
|
@ -1,77 +1,49 @@
|
|||
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
|
||||
|
||||
import { expect } from 'chai'
|
||||
import { createFile } from 'fs-extra/esm'
|
||||
import { readdir } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { wait } from '@peertube/peertube-core-utils'
|
||||
import { buildUUID } from '@peertube/peertube-node-utils'
|
||||
import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
|
||||
import { getAllFiles, wait } from '@peertube/peertube-core-utils'
|
||||
import { FileStorage, HttpStatusCode, HttpStatusCodeType, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models'
|
||||
import { areMockObjectStorageTestsDisabled, buildUUID } from '@peertube/peertube-node-utils'
|
||||
import {
|
||||
cleanupTests,
|
||||
CLICommand,
|
||||
ObjectStorageCommand,
|
||||
PeerTubeServer,
|
||||
cleanupTests,
|
||||
createMultipleServers,
|
||||
doubleFollow,
|
||||
killallServers,
|
||||
makeGetRequest,
|
||||
PeerTubeServer,
|
||||
makeRawRequest,
|
||||
setAccessTokensToServers,
|
||||
setDefaultVideoChannel,
|
||||
waitJobs
|
||||
} from '@peertube/peertube-server-commands'
|
||||
|
||||
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
|
||||
const files = await readdir(server.servers.buildDirectory(directory))
|
||||
|
||||
for (const f of files) {
|
||||
expect(f).to.not.contain(substring)
|
||||
}
|
||||
}
|
||||
|
||||
async function assertCountAreOkay (servers: PeerTubeServer[]) {
|
||||
for (const server of servers) {
|
||||
const videosCount = await server.servers.countFiles('web-videos')
|
||||
expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
|
||||
|
||||
const privateVideosCount = await server.servers.countFiles('web-videos/private')
|
||||
expect(privateVideosCount).to.equal(4)
|
||||
|
||||
const torrentsCount = await server.servers.countFiles('torrents')
|
||||
expect(torrentsCount).to.equal(24)
|
||||
|
||||
const previewsCount = await server.servers.countFiles('previews')
|
||||
expect(previewsCount).to.equal(3)
|
||||
|
||||
const thumbnailsCount = await server.servers.countFiles('thumbnails')
|
||||
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
|
||||
|
||||
const avatarsCount = await server.servers.countFiles('avatars')
|
||||
expect(avatarsCount).to.equal(8)
|
||||
|
||||
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
|
||||
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
|
||||
|
||||
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
|
||||
expect(hlsPrivateRootCount).to.equal(1)
|
||||
}
|
||||
}
|
||||
import { SQLCommand } from '@tests/shared/sql-command.js'
|
||||
import { expect } from 'chai'
|
||||
import { createFile } from 'fs-extra/esm'
|
||||
import { readdir } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
|
||||
describe('Test prune storage scripts', function () {
|
||||
let servers: PeerTubeServer[]
|
||||
const badNames: { [directory: string]: string[] } = {}
|
||||
|
||||
before(async function () {
|
||||
this.timeout(120000)
|
||||
|
||||
servers = await createMultipleServers(2, { transcoding: { enabled: true } })
|
||||
servers = await createMultipleServers(2)
|
||||
|
||||
await setAccessTokensToServers(servers)
|
||||
await setDefaultVideoChannel(servers)
|
||||
|
||||
for (const server of servers) {
|
||||
await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
|
||||
await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
|
||||
await server.config.enableMinimumTranscoding({ keepOriginal: true })
|
||||
await server.config.enableUserExport()
|
||||
}
|
||||
|
||||
await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
|
||||
for (const server of servers) {
|
||||
await server.videos.quickUpload({ name: 'video 1', privacy: VideoPrivacy.PUBLIC })
|
||||
await server.videos.quickUpload({ name: 'video 2', privacy: VideoPrivacy.PUBLIC })
|
||||
|
||||
await server.videos.quickUpload({ name: 'video 3', privacy: VideoPrivacy.PRIVATE })
|
||||
|
||||
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
|
||||
|
||||
|
|
@ -85,6 +57,12 @@ describe('Test prune storage scripts', function () {
|
|||
})
|
||||
}
|
||||
|
||||
for (const server of servers) {
|
||||
const user = await server.users.getMyInfo()
|
||||
|
||||
await server.userExports.request({ userId: user.id, withVideoFiles: false })
|
||||
}
|
||||
|
||||
await doubleFollow(servers[0], servers[1])
|
||||
|
||||
// Lazy load the remote avatars
|
||||
|
|
@ -119,8 +97,53 @@ describe('Test prune storage scripts', function () {
|
|||
await wait(1000)
|
||||
})
|
||||
|
||||
describe('On filesystem', function () {
|
||||
const badNames: { [directory: string]: string[] } = {}
|
||||
|
||||
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
|
||||
const files = await readdir(server.servers.buildDirectory(directory))
|
||||
|
||||
for (const f of files) {
|
||||
expect(f).to.not.contain(substring)
|
||||
}
|
||||
}
|
||||
|
||||
async function assertCountAreOkay () {
|
||||
for (const server of servers) {
|
||||
const videosCount = await server.servers.countFiles('web-videos')
|
||||
expect(videosCount).to.equal(5) // 2 videos with 2 resolutions + private directory
|
||||
|
||||
const privateVideosCount = await server.servers.countFiles('web-videos/private')
|
||||
expect(privateVideosCount).to.equal(2)
|
||||
|
||||
const torrentsCount = await server.servers.countFiles('torrents')
|
||||
expect(torrentsCount).to.equal(12)
|
||||
|
||||
const previewsCount = await server.servers.countFiles('previews')
|
||||
expect(previewsCount).to.equal(3)
|
||||
|
||||
const thumbnailsCount = await server.servers.countFiles('thumbnails')
|
||||
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
|
||||
|
||||
const avatarsCount = await server.servers.countFiles('avatars')
|
||||
expect(avatarsCount).to.equal(8)
|
||||
|
||||
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
|
||||
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
|
||||
|
||||
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
|
||||
expect(hlsPrivateRootCount).to.equal(1)
|
||||
|
||||
const originalVideoFilesCount = await server.servers.countFiles(join('original-video-files'))
|
||||
expect(originalVideoFilesCount).to.equal(3)
|
||||
|
||||
const userExportFilesCount = await server.servers.countFiles(join('tmp-persistent'))
|
||||
expect(userExportFilesCount).to.equal(1)
|
||||
}
|
||||
}
|
||||
|
||||
it('Should have the files on the disk', async function () {
|
||||
await assertCountAreOkay(servers)
|
||||
await assertCountAreOkay()
|
||||
})
|
||||
|
||||
it('Should create some dirty files', async function () {
|
||||
|
|
@ -198,6 +221,27 @@ describe('Test prune storage scripts', function () {
|
|||
await createFile(join(basePrivate, n1))
|
||||
badNames[directory] = [ n1 ]
|
||||
}
|
||||
|
||||
{
|
||||
const base = servers[0].servers.buildDirectory('original-video-files')
|
||||
|
||||
const n1 = buildUUID() + '.mp4'
|
||||
await createFile(join(base, n1))
|
||||
|
||||
badNames['original-video-files'] = [ n1 ]
|
||||
}
|
||||
|
||||
{
|
||||
const base = servers[0].servers.buildDirectory('tmp-persistent')
|
||||
|
||||
const n1 = 'user-export-1.zip'
|
||||
const n2 = 'user-export-2.zip'
|
||||
|
||||
await createFile(join(base, n1))
|
||||
await createFile(join(base, n2))
|
||||
|
||||
badNames['tmp-persistent'] = [ n1, n2 ]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -209,7 +253,7 @@ describe('Test prune storage scripts', function () {
|
|||
})
|
||||
|
||||
it('Should have removed files', async function () {
|
||||
await assertCountAreOkay(servers)
|
||||
await assertCountAreOkay()
|
||||
|
||||
for (const directory of Object.keys(badNames)) {
|
||||
for (const name of badNames[directory]) {
|
||||
|
|
@ -217,6 +261,96 @@ describe('Test prune storage scripts', function () {
|
|||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('On object storage', function () {
|
||||
if (areMockObjectStorageTestsDisabled()) return
|
||||
|
||||
const videos: string[] = []
|
||||
const objectStorage = new ObjectStorageCommand()
|
||||
|
||||
let sqlCommand: SQLCommand
|
||||
let rootId: number
|
||||
|
||||
async function checkVideosFiles (uuids: string[], expectedStatus: HttpStatusCodeType) {
|
||||
for (const uuid of uuids) {
|
||||
const video = await servers[0].videos.getWithToken({ id: uuid })
|
||||
|
||||
for (const file of getAllFiles(video)) {
|
||||
await makeRawRequest({ url: file.fileUrl, token: servers[0].accessToken, expectedStatus })
|
||||
}
|
||||
|
||||
const source = await servers[0].videos.getSource({ id: uuid })
|
||||
await makeRawRequest({ url: source.fileDownloadUrl, redirects: 1, token: servers[0].accessToken, expectedStatus })
|
||||
}
|
||||
}
|
||||
|
||||
async function checkUserExport (expectedStatus: HttpStatusCodeType) {
|
||||
const { data } = await servers[0].userExports.list({ userId: rootId })
|
||||
await makeRawRequest({ url: data[0].privateDownloadUrl, redirects: 1, expectedStatus })
|
||||
}
|
||||
|
||||
before(async function () {
|
||||
this.timeout(120000)
|
||||
|
||||
sqlCommand = new SQLCommand(servers[0])
|
||||
|
||||
await objectStorage.prepareDefaultMockBuckets()
|
||||
|
||||
await servers[0].run(objectStorage.getDefaultMockConfig())
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 1', privacy: VideoPrivacy.PUBLIC })
|
||||
videos.push(uuid)
|
||||
}
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 2', privacy: VideoPrivacy.PUBLIC })
|
||||
videos.push(uuid)
|
||||
}
|
||||
|
||||
{
|
||||
const { uuid } = await servers[0].videos.quickUpload({ name: 's3 video 3', privacy: VideoPrivacy.PRIVATE })
|
||||
videos.push(uuid)
|
||||
}
|
||||
|
||||
const user = await servers[0].users.getMyInfo()
|
||||
rootId = user.id
|
||||
|
||||
await servers[0].userExports.deleteAllArchives({ userId: rootId })
|
||||
await servers[0].userExports.request({ userId: rootId, withVideoFiles: false })
|
||||
|
||||
await waitJobs([ servers[0] ])
|
||||
})
|
||||
|
||||
it('Should have the files on object storage', async function () {
|
||||
await checkVideosFiles(videos, HttpStatusCode.OK_200)
|
||||
await checkUserExport(HttpStatusCode.OK_200)
|
||||
})
|
||||
|
||||
it('Should run prune-storage script on videos', async function () {
|
||||
await sqlCommand.setVideoFileStorageOf(videos[1], FileStorage.FILE_SYSTEM)
|
||||
await sqlCommand.setVideoFileStorageOf(videos[2], FileStorage.FILE_SYSTEM)
|
||||
|
||||
await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
|
||||
await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
|
||||
|
||||
await checkUserExport(HttpStatusCode.OK_200)
|
||||
})
|
||||
|
||||
it('Should run prune-storage script on exports', async function () {
|
||||
await sqlCommand.setUserExportStorageOf(rootId, FileStorage.FILE_SYSTEM)
|
||||
|
||||
await checkVideosFiles([ videos[1], videos[2] ], HttpStatusCode.NOT_FOUND_404)
|
||||
await checkVideosFiles([ videos[0] ], HttpStatusCode.OK_200)
|
||||
|
||||
await checkUserExport(HttpStatusCode.NOT_FOUND_404)
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
await sqlCommand.cleanup()
|
||||
})
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
await cleanupTests(servers)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { QueryTypes, Sequelize } from 'sequelize'
|
||||
import { forceNumber } from '@peertube/peertube-core-utils'
|
||||
import { PeerTubeServer } from '@peertube/peertube-server-commands'
|
||||
import { FileStorageType } from '@peertube/peertube-models'
|
||||
|
||||
export class SQLCommand {
|
||||
private sequelize: Sequelize
|
||||
|
|
@ -58,6 +59,30 @@ export class SQLCommand {
|
|||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async setVideoFileStorageOf (uuid: string, storage: FileStorageType) {
|
||||
await this.updateQuery(
|
||||
`UPDATE "videoFile" SET storage = :storage ` +
|
||||
`WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid) OR ` +
|
||||
// eslint-disable-next-line max-len
|
||||
`"videoStreamingPlaylistId" IN (` +
|
||||
`SELECT "videoStreamingPlaylist".id FROM "videoStreamingPlaylist" ` +
|
||||
`INNER JOIN video ON video.id = "videoStreamingPlaylist"."videoId" AND "video".uuid = :uuid` +
|
||||
`)`,
|
||||
{ storage, uuid }
|
||||
)
|
||||
|
||||
await this.updateQuery(
|
||||
`UPDATE "videoSource" SET storage = :storage WHERE "videoId" IN (SELECT id FROM "video" WHERE uuid = :uuid)`,
|
||||
{ storage, uuid }
|
||||
)
|
||||
}
|
||||
|
||||
async setUserExportStorageOf (userId: number, storage: FileStorageType) {
|
||||
await this.updateQuery(`UPDATE "userExport" SET storage = :storage WHERE "userId" = :userId`, { storage, userId })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
setPluginVersion (pluginName: string, newVersion: string) {
|
||||
return this.setPluginField(pluginName, 'version', newVersion)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import { context } from '@opentelemetry/api'
|
||||
import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
|
||||
import { omit } from '@peertube/peertube-core-utils'
|
||||
import { stat } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { format as sqlFormat } from 'sql-formatter'
|
||||
import { createLogger, format, transports } from 'winston'
|
||||
import { FileTransportOptions } from 'winston/lib/winston/transports'
|
||||
import { context } from '@opentelemetry/api'
|
||||
import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js'
|
||||
import { omit } from '@peertube/peertube-core-utils'
|
||||
import { CONFIG } from '../initializers/config.js'
|
||||
import { LOG_FILENAME } from '../initializers/constants.js'
|
||||
|
||||
|
|
@ -60,7 +60,7 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
|
|||
|
||||
function buildLogger (labelSuffix?: string) {
|
||||
return createLogger({
|
||||
level: CONFIG.LOG.LEVEL,
|
||||
level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
|
||||
defaultMeta: {
|
||||
get traceId () { return getSpanContext(context.active())?.traceId },
|
||||
get spanId () { return getSpanContext(context.active())?.spanId },
|
||||
|
|
@ -154,18 +154,10 @@ async function mtimeSortFilesDesc (files: string[], basePath: string) {
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
type LoggerTagsFn,
|
||||
type LoggerTags,
|
||||
|
||||
buildLogger,
|
||||
timestampFormatter,
|
||||
labelFormatter,
|
||||
consoleLoggerFormat,
|
||||
jsonLoggerFormat,
|
||||
mtimeSortFilesDesc,
|
||||
logger,
|
||||
loggerTagsFactory,
|
||||
bunyanLogger
|
||||
buildLogger, bunyanLogger, consoleLoggerFormat,
|
||||
jsonLoggerFormat, labelFormatter, logger,
|
||||
loggerTagsFactory, mtimeSortFilesDesc, timestampFormatter, type LoggerTags, type LoggerTagsFn
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -845,6 +845,7 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
const USER_EXPORT_MAX_ITEMS = 1000
|
||||
const USER_EXPORT_FILE_PREFIX = 'user-export-'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
|
|
@ -1268,6 +1269,7 @@ export {
|
|||
CONSTRAINTS_FIELDS,
|
||||
EMBED_SIZE,
|
||||
REDUNDANCY,
|
||||
USER_EXPORT_FILE_PREFIX,
|
||||
JOB_CONCURRENCY,
|
||||
JOB_ATTEMPTS,
|
||||
AP_CLEANER,
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ import { createReadStream, createWriteStream } from 'fs'
|
|||
import { ensureDir } from 'fs-extra/esm'
|
||||
import { dirname } from 'path'
|
||||
import { Readable } from 'stream'
|
||||
import { getInternalUrl } from '../urls.js'
|
||||
import { getClient } from './client.js'
|
||||
import { lTags } from './logger.js'
|
||||
import { getInternalUrl } from './urls.js'
|
||||
import { getClient } from './shared/client.js'
|
||||
import { lTags } from './shared/logger.js'
|
||||
|
||||
import type { _Object, ObjectCannedACL, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ type BucketInfo = {
|
|||
PREFIX?: string
|
||||
}
|
||||
|
||||
async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
|
||||
async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo, continuationToken?: string) {
|
||||
const s3Client = await getClient()
|
||||
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
|
||||
|
|
@ -26,14 +26,21 @@ async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
|
|||
const commandPrefix = bucketInfo.PREFIX + prefix
|
||||
const listCommand = new ListObjectsV2Command({
|
||||
Bucket: bucketInfo.BUCKET_NAME,
|
||||
Prefix: commandPrefix
|
||||
Prefix: commandPrefix,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
|
||||
const listedObjects = await s3Client.send(listCommand)
|
||||
|
||||
if (isArray(listedObjects.Contents) !== true) return []
|
||||
|
||||
return listedObjects.Contents.map(c => c.Key)
|
||||
let keys = listedObjects.Contents.map(c => c.Key)
|
||||
|
||||
if (listedObjects.IsTruncated) {
|
||||
keys = keys.concat(await listKeysOfPrefix(prefix, bucketInfo, listedObjects.NextContinuationToken))
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -145,7 +152,7 @@ function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
|
|||
return removeObjectByFullKey(key, bucketInfo)
|
||||
}
|
||||
|
||||
async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
|
||||
async function removeObjectByFullKey (fullKey: string, bucketInfo: Pick<BucketInfo, 'BUCKET_NAME'>) {
|
||||
logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
|
||||
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
export * from './client.js'
|
||||
export * from './logger.js'
|
||||
export * from './object-storage-helpers.js'
|
||||
export * from '../object-storage-helpers.js'
|
||||
|
|
|
|||
|
|
@ -1,23 +1,25 @@
|
|||
import { FindOptions, Op } from 'sequelize'
|
||||
import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
||||
import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
|
||||
import { UserModel } from './user.js'
|
||||
import { getSort } from '../shared/sort.js'
|
||||
import { UserExportState, type UserExport, type UserExportStateType, type FileStorageType, FileStorage } from '@peertube/peertube-models'
|
||||
import { FileStorage, UserExportState, type FileStorageType, type UserExport, type UserExportStateType } from '@peertube/peertube-models'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import {
|
||||
JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
|
||||
STATIC_DOWNLOAD_PATHS,
|
||||
USER_EXPORT_FILE_PREFIX,
|
||||
USER_EXPORT_STATES,
|
||||
WEBSERVER
|
||||
} from '@server/initializers/constants.js'
|
||||
import { join } from 'path'
|
||||
import jwt from 'jsonwebtoken'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { removeUserExportObjectStorage } from '@server/lib/object-storage/user-export.js'
|
||||
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
||||
import { MUserAccountId, MUserExport } from '@server/types/models/index.js'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import jwt from 'jsonwebtoken'
|
||||
import { join } from 'path'
|
||||
import { FindOptions, Op } from 'sequelize'
|
||||
import { AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
||||
import { doesExist } from '../shared/query.js'
|
||||
import { SequelizeModel } from '../shared/sequelize-type.js'
|
||||
import { getSort } from '../shared/sort.js'
|
||||
import { UserModel } from './user.js'
|
||||
|
||||
@Table({
|
||||
tableName: 'userExport',
|
||||
|
|
@ -147,11 +149,20 @@ export class UserExportModel extends SequelizeModel<UserExportModel> {
|
|||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
|
||||
const query = 'SELECT 1 FROM "userExport" ' +
|
||||
`WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
|
||||
|
||||
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
generateAndSetFilename () {
|
||||
if (!this.userId) throw new Error('Cannot generate filename without userId')
|
||||
if (!this.createdAt) throw new Error('Cannot generate filename without createdAt')
|
||||
|
||||
this.filename = `user-export-${this.userId}-${this.createdAt.toISOString()}.zip`
|
||||
this.filename = `${USER_EXPORT_FILE_PREFIX}${this.userId}-${this.createdAt.toISOString()}.zip`
|
||||
}
|
||||
|
||||
canBeSafelyRemoved () {
|
||||
|
|
|
|||
|
|
@ -289,11 +289,11 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
|
|||
return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
|
||||
}
|
||||
|
||||
static async doesOwnedWebVideoFileExist (filename: string) {
|
||||
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
|
||||
const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
|
||||
`WHERE "filename" = $filename AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
|
||||
`WHERE "filename" = $filename AND "storage" = $storage LIMIT 1`
|
||||
|
||||
return doesExist({ sequelize: this.sequelize, query, bind: { filename } })
|
||||
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
|
||||
}
|
||||
|
||||
static loadByFilename (filename: string) {
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import type { FileStorageType, VideoSource } from '@peertube/peertube-models'
|
||||
import { type FileStorageType, type VideoSource } from '@peertube/peertube-models'
|
||||
import { STATIC_DOWNLOAD_PATHS, WEBSERVER } from '@server/initializers/constants.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { join } from 'path'
|
||||
import { Transaction } from 'sequelize'
|
||||
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript'
|
||||
import { SequelizeModel, getSort } from '../shared/index.js'
|
||||
import { SequelizeModel, doesExist, getSort } from '../shared/index.js'
|
||||
import { getResolutionLabel } from './formatter/video-api-format.js'
|
||||
import { VideoModel } from './video.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
|
||||
@Table({
|
||||
tableName: 'videoSource',
|
||||
|
|
@ -103,6 +103,18 @@ export class VideoSourceModel extends SequelizeModel<VideoSourceModel> {
|
|||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static async doesOwnedFileExist (filename: string, storage: FileStorageType) {
|
||||
const query = 'SELECT 1 FROM "videoSource" ' +
|
||||
'INNER JOIN "video" ON "video"."id" = "videoSource"."videoId" AND "video"."remote" IS FALSE ' +
|
||||
`WHERE "keptOriginalFilename" = $filename AND "storage" = $storage LIMIT 1`
|
||||
|
||||
return doesExist({ sequelize: this.sequelize, query, bind: { filename, storage } })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
getFileDownloadUrl () {
|
||||
if (!this.keptOriginalFilename) return null
|
||||
|
||||
|
|
|
|||
|
|
@ -229,13 +229,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
|
|||
return Object.assign(playlist, { Video: video })
|
||||
}
|
||||
|
||||
static doesOwnedHLSPlaylistExist (videoUUID: string) {
|
||||
static doesOwnedVideoUUIDExist (videoUUID: string, storage: FileStorageType) {
|
||||
const query = `SELECT 1 FROM "videoStreamingPlaylist" ` +
|
||||
`INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` +
|
||||
`AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` +
|
||||
`AND "storage" = ${FileStorage.FILE_SYSTEM} LIMIT 1`
|
||||
`AND "storage" = $storage LIMIT 1`
|
||||
|
||||
return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID } })
|
||||
return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })
|
||||
}
|
||||
|
||||
assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
import { uniqify } from '@peertube/peertube-core-utils'
|
||||
import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
|
||||
import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
|
||||
import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
|
||||
import { UserExportModel } from '@server/models/user/user-export.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import Bluebird from 'bluebird'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { readdir, stat } from 'fs/promises'
|
||||
import { basename, join } from 'path'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import prompt from 'prompt'
|
||||
import { uniqify } from '@peertube/peertube-core-utils'
|
||||
import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
|
||||
import { DIRECTORIES } from '@server/initializers/constants.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import { getUUIDFromFilename } from '../core/helpers/utils.js'
|
||||
import { CONFIG } from '../core/initializers/config.js'
|
||||
import { initDatabaseModels } from '../core/initializers/database.js'
|
||||
|
|
@ -24,6 +27,121 @@ run()
|
|||
})
|
||||
|
||||
async function run () {
|
||||
await initDatabaseModels(true)
|
||||
|
||||
await new FSPruner().prune()
|
||||
|
||||
console.log('\n')
|
||||
|
||||
await new ObjectStoragePruner().prune()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Object storage
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class ObjectStoragePruner {
|
||||
private readonly keysToDelete: { bucket: string, key: string }[] = []
|
||||
|
||||
async prune () {
|
||||
if (!CONFIG.OBJECT_STORAGE.ENABLED) return
|
||||
|
||||
console.log('Pruning object storage.')
|
||||
|
||||
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
|
||||
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
|
||||
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
|
||||
await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
|
||||
|
||||
if (this.keysToDelete.length === 0) {
|
||||
console.log('No unknown object storage files to delete.')
|
||||
return
|
||||
}
|
||||
|
||||
const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
|
||||
console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
|
||||
|
||||
const res = await askConfirmation()
|
||||
if (res !== true) {
|
||||
console.log('Exiting without deleting object storage files.')
|
||||
return
|
||||
}
|
||||
|
||||
console.log('Deleting object storage files...\n')
|
||||
|
||||
for (const { bucket, key } of this.keysToDelete) {
|
||||
await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
|
||||
}
|
||||
|
||||
console.log(`${this.keysToDelete.length} object storage files deleted.`)
|
||||
}
|
||||
|
||||
private async findFilesToDelete (
|
||||
config: { BUCKET_NAME: string, PREFIX?: string },
|
||||
existFun: (file: string) => Promise<boolean> | boolean
|
||||
) {
|
||||
try {
|
||||
const keys = await listKeysOfPrefix('', config)
|
||||
|
||||
await Bluebird.map(keys, async key => {
|
||||
if (await existFun(key) !== true) {
|
||||
this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
|
||||
}
|
||||
}, { concurrency: 20 })
|
||||
} catch (err) {
|
||||
const prefixMessage = config.PREFIX
|
||||
? ` and prefix ${config.PREFIX}`
|
||||
: ''
|
||||
|
||||
console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage)
|
||||
}
|
||||
}
|
||||
|
||||
private doesWebVideoFileExistFactory () {
|
||||
return (key: string) => {
|
||||
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
|
||||
|
||||
return VideoFileModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
|
||||
}
|
||||
}
|
||||
|
||||
private doesStreamingPlaylistFileExistFactory () {
|
||||
return (key: string) => {
|
||||
const uuid = basename(dirname(this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)))
|
||||
|
||||
return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
|
||||
}
|
||||
}
|
||||
|
||||
private doesOriginalFileExistFactory () {
|
||||
return (key: string) => {
|
||||
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
|
||||
|
||||
return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
|
||||
}
|
||||
}
|
||||
|
||||
private doesUserExportFileExistFactory () {
|
||||
return (key: string) => {
|
||||
const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
|
||||
|
||||
return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
|
||||
}
|
||||
}
|
||||
|
||||
private sanitizeKey (key: string, config: { PREFIX: string }) {
|
||||
return key.replace(new RegExp(`^${config.PREFIX}`), '')
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FS
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class FSPruner {
|
||||
private pathsToDelete: string[] = []
|
||||
|
||||
async prune () {
|
||||
const dirs = Object.values(CONFIG.STORAGE)
|
||||
|
||||
if (uniqify(dirs).length !== dirs.length) {
|
||||
|
|
@ -31,92 +149,96 @@ async function run () {
|
|||
process.exit(0)
|
||||
}
|
||||
|
||||
await initDatabaseModels(true)
|
||||
console.log('Pruning filesystem storage.')
|
||||
|
||||
let toDelete: string[] = []
|
||||
console.log('Detecting files to remove, it can take a while...')
|
||||
|
||||
console.log('Detecting files to remove, it could take a while...')
|
||||
await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
|
||||
await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
|
||||
|
||||
toDelete = toDelete.concat(
|
||||
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PUBLIC, doesWebVideoFileExist()),
|
||||
await pruneDirectory(DIRECTORIES.WEB_VIDEOS.PRIVATE, doesWebVideoFileExist()),
|
||||
await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
|
||||
await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
|
||||
|
||||
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
|
||||
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
|
||||
await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
|
||||
|
||||
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
|
||||
await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
|
||||
|
||||
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
|
||||
await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
|
||||
|
||||
await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)),
|
||||
await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)),
|
||||
await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
|
||||
await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
|
||||
|
||||
await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist)
|
||||
)
|
||||
await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
|
||||
|
||||
await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
|
||||
|
||||
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
|
||||
toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
|
||||
this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
|
||||
|
||||
if (toDelete.length === 0) {
|
||||
console.log('No files to delete.')
|
||||
if (this.pathsToDelete.length === 0) {
|
||||
console.log('No unknown filesystem files to delete.')
|
||||
return
|
||||
}
|
||||
|
||||
console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n'))
|
||||
const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
|
||||
console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
|
||||
|
||||
const res = await askConfirmation()
|
||||
if (res === true) {
|
||||
console.log('Processing delete...\n')
|
||||
if (res !== true) {
|
||||
console.log('Exiting without deleting filesystem files.')
|
||||
return
|
||||
}
|
||||
|
||||
for (const path of toDelete) {
|
||||
console.log('Deleting filesystem files...\n')
|
||||
|
||||
for (const path of this.pathsToDelete) {
|
||||
await remove(path)
|
||||
}
|
||||
|
||||
console.log('Done!')
|
||||
} else {
|
||||
console.log('Exiting without deleting files.')
|
||||
}
|
||||
console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
|
||||
}
|
||||
|
||||
type ExistFun = (file: string) => Promise<boolean> | boolean
|
||||
async function pruneDirectory (directory: string, existFun: ExistFun) {
|
||||
private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
|
||||
const files = await readdir(directory)
|
||||
|
||||
const toDelete: string[] = []
|
||||
await Bluebird.map(files, async file => {
|
||||
const filePath = join(directory, file)
|
||||
|
||||
if (await existFun(filePath) !== true) {
|
||||
toDelete.push(filePath)
|
||||
this.pathsToDelete.push(filePath)
|
||||
}
|
||||
}, { concurrency: 20 })
|
||||
|
||||
return toDelete
|
||||
}
|
||||
|
||||
function doesWebVideoFileExist () {
|
||||
private doesWebVideoFileExistFactory () {
|
||||
return (filePath: string) => {
|
||||
// Don't delete private directory
|
||||
if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
|
||||
|
||||
return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath))
|
||||
return VideoFileModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
|
||||
}
|
||||
}
|
||||
|
||||
function doesHLSPlaylistExist () {
|
||||
private doesHLSPlaylistExistFactory () {
|
||||
return (hlsPath: string) => {
|
||||
// Don't delete private directory
|
||||
if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
|
||||
|
||||
return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
|
||||
return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
|
||||
}
|
||||
}
|
||||
|
||||
function doesTorrentFileExist () {
|
||||
private doesOriginalVideoExistFactory () {
|
||||
return (filePath: string) => {
|
||||
return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
|
||||
}
|
||||
}
|
||||
|
||||
private doesTorrentFileExistFactory () {
|
||||
return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
|
||||
}
|
||||
|
||||
function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
|
||||
private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
|
||||
return async (filePath: string) => {
|
||||
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
|
||||
if (!thumbnail) return false
|
||||
|
|
@ -130,13 +252,16 @@ function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
|
|||
}
|
||||
}
|
||||
|
||||
async function doesActorImageExist (filePath: string) {
|
||||
private doesActorImageExistFactory () {
|
||||
return async (filePath: string) => {
|
||||
const image = await ActorImageModel.loadByName(basename(filePath))
|
||||
|
||||
return !!image
|
||||
}
|
||||
}
|
||||
|
||||
async function doesRedundancyExist (filePath: string) {
|
||||
private doesRedundancyExistFactory () {
|
||||
return async (filePath: string) => {
|
||||
const isPlaylist = (await stat(filePath)).isDirectory()
|
||||
|
||||
if (isPlaylist) {
|
||||
|
|
@ -160,6 +285,19 @@ async function doesRedundancyExist (filePath: string) {
|
|||
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
|
||||
return !!redundancy
|
||||
}
|
||||
}
|
||||
|
||||
private doesUserExportExistFactory () {
|
||||
return (filePath: string) => {
|
||||
const filename = basename(filePath)
|
||||
|
||||
// Only detect non-existing user export
|
||||
if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
|
||||
|
||||
return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function askConfirmation () {
|
||||
return new Promise((res, rej) => {
|
||||
|
|
@ -169,10 +307,12 @@ async function askConfirmation () {
|
|||
properties: {
|
||||
confirm: {
|
||||
type: 'string',
|
||||
description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
|
||||
description: 'These unknown files can be deleted, but please check your backups first (bugs happen).' +
|
||||
' Notice PeerTube must have been stopped when your ran this script.' +
|
||||
' Can we delete these files?',
|
||||
' Can we delete these files? (y/n)',
|
||||
default: 'n',
|
||||
validator: /y[es]*|n[o]?/,
|
||||
warning: 'Must respond yes or no',
|
||||
required: true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue