diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts index c9c8fb368..cd9741ea7 100644 --- a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts @@ -94,8 +94,7 @@ export class VideoRedundanciesListComponent extends RestTable implements OnInit } getTotalSize (redundancy: VideoRedundancy) { - return redundancy.redundancies.files.reduce((a, b) => a + b.size, 0) + - redundancy.redundancies.streamingPlaylists.reduce((a, b) => a + b.size, 0) + return redundancy.redundancies.streamingPlaylists.reduce((a, b) => a + b.size, 0) } onDisplayTypeChanged () { @@ -106,8 +105,9 @@ export class VideoRedundanciesListComponent extends RestTable implements OnInit } getRedundancyStrategy (redundancy: VideoRedundancy) { - if (redundancy.redundancies.files.length !== 0) return redundancy.redundancies.files[0].strategy - if (redundancy.redundancies.streamingPlaylists.length !== 0) return redundancy.redundancies.streamingPlaylists[0].strategy + if (redundancy.redundancies.streamingPlaylists.length !== 0) { + return redundancy.redundancies.streamingPlaylists[0].strategy + } return '' } diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts index a6e54ce80..00ce59df8 100644 --- a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts @@ -1,6 +1,6 @@ import { Component, Input } from '@angular/core' import { PTDatePipe } from '@app/shared/shared-main/common/date.pipe' -import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@peertube/peertube-models' +import { RedundancyInformation } from '@peertube/peertube-models' import { BytesPipe } from '../../../shared/shared-main/common/bytes.pipe' @Component({ @@ -11,5 +11,5 @@ import { BytesPipe } from '../../../shared/shared-main/common/bytes.pipe' imports: [ PTDatePipe, BytesPipe ] }) export class VideoRedundancyInformationComponent { - @Input() redundancyElement: FileRedundancyInformation | StreamingPlaylistRedundancyInformation + @Input() redundancyElement: RedundancyInformation } diff --git a/client/src/app/shared/shared-main/video/redundancy.service.ts b/client/src/app/shared/shared-main/video/redundancy.service.ts index 8904ee425..e017003ff 100644 --- a/client/src/app/shared/shared-main/video/redundancy.service.ts +++ b/client/src/app/shared/shared-main/video/redundancy.service.ts @@ -53,7 +53,6 @@ export class RedundancyService { removeVideoRedundancies (redundancy: VideoRedundancy) { const observables = redundancy.redundancies.streamingPlaylists.map(r => r.id) - .concat(redundancy.redundancies.files.map(r => r.id)) .map(id => this.removeRedundancy(id)) return concat(...observables) diff --git a/package.json b/package.json index 54d30897f..5f4239c65 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,9 @@ "@types/express": "4.17.9", "@types/express-serve-static-core": "4.19.5" }, + "optionalDependencies": { + "webtorrent": "2.5.17" + }, "dependencies": { "@aws-sdk/client-s3": "^3.190.0", "@aws-sdk/lib-storage": "^3.190.0", @@ -187,7 +190,6 @@ "useragent": "^2.3.0", "validator": "^13.0.0", "webfinger.js": "^2.6.6", - "webtorrent": "2.1.27", "winston": "3.14.2", "ws": "^8.0.0", "yauzl": "^3.1.0" diff --git a/packages/models/src/activitypub/objects/cache-file-object.ts b/packages/models/src/activitypub/objects/cache-file-object.ts index a40ef339c..d34516089 100644 --- a/packages/models/src/activitypub/objects/cache-file-object.ts +++ b/packages/models/src/activitypub/objects/cache-file-object.ts @@ -1,9 +1,9 @@ -import { ActivityVideoUrlObject, ActivityPlaylistUrlObject } from './common-objects.js' +import { ActivityPlaylistUrlObject } from './common-objects.js' export interface CacheFileObject { id: string type: 'CacheFile' object: string expires: string - url: ActivityVideoUrlObject | ActivityPlaylistUrlObject + url: ActivityPlaylistUrlObject } diff --git a/packages/models/src/redundancy/video-redundancy.model.ts b/packages/models/src/redundancy/video-redundancy.model.ts index fa6e05832..01b1272d4 100644 --- a/packages/models/src/redundancy/video-redundancy.model.ts +++ b/packages/models/src/redundancy/video-redundancy.model.ts @@ -5,13 +5,14 @@ export interface VideoRedundancy { uuid: string redundancies: { - files: FileRedundancyInformation[] + // FIXME: remove in v8 + files: [] - streamingPlaylists: StreamingPlaylistRedundancyInformation[] + streamingPlaylists: RedundancyInformation[] } } -interface RedundancyInformation { +export interface RedundancyInformation { id: number fileUrl: string strategy: string @@ -23,13 +24,3 @@ interface RedundancyInformation { size: number } - -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface FileRedundancyInformation extends RedundancyInformation { - -} - -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface StreamingPlaylistRedundancyInformation extends RedundancyInformation { - -} diff --git a/packages/tests/src/api/redundancy/manage-redundancy.ts b/packages/tests/src/api/redundancy/manage-redundancy.ts index 14556e26c..02769bf1b 100644 --- a/packages/tests/src/api/redundancy/manage-redundancy.ts +++ b/packages/tests/src/api/redundancy/manage-redundancy.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' +import { VideoPrivacy, VideoRedundanciesTarget } from '@peertube/peertube-models' import { cleanupTests, createMultipleServers, @@ -10,7 +10,7 @@ import { setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' -import { VideoPrivacy, VideoRedundanciesTarget } from '@peertube/peertube-models' +import { expect } from 'chai' describe('Test manage videos redundancy', function () { const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ] @@ -95,7 +95,7 @@ describe('Test manage videos redundancy', function () { this.timeout(120000) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 10) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 2) await waitJobs(servers) const body = await commands[1].listVideos({ target: 'remote-videos' }) @@ -119,12 +119,10 @@ describe('Test manage videos redundancy', function () { expect(videos1.name).to.equal('video 1 server 2') expect(videos2.name).to.equal('video 2 server 2') - expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.files).to.have.lengthOf(0) expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) - const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) - - for (const r of redundancies) { + for (const r of videos1.redundancies.streamingPlaylists) { expect(r.strategy).to.be.null expect(r.fileUrl).to.exist expect(r.createdAt).to.exist @@ -155,12 +153,10 @@ describe('Test manage videos redundancy', function () { expect(videos1.name).to.equal('video 1 server 2') expect(videos2.name).to.equal('video 2 server 2') - expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.files).to.have.lengthOf(0) expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) - const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) - - for (const r of redundancies) { + for (const r of videos1.redundancies.streamingPlaylists) { expect(r.strategy).to.equal('recently-added') expect(r.fileUrl).to.exist expect(r.createdAt).to.exist @@ -218,7 +214,7 @@ describe('Test manage videos redundancy', function () { await commands[0].addVideo({ videoId }) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 15) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 3) await waitJobs(servers) { @@ -232,12 +228,10 @@ describe('Test manage videos redundancy', function () { const video = body.data[0] expect(video.name).to.equal('video 3 server 2') - expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.files).to.have.lengthOf(0) expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) - const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - - for (const r of redundancies) { + for (const r of video.redundancies.streamingPlaylists) { redundanciesToRemove.push(r.id) expect(r.strategy).to.equal('manual') @@ -257,12 +251,10 @@ describe('Test manage videos redundancy', function () { const video = body.data[0] expect(video.name).to.equal('video 3 server 2') - expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.files).to.have.lengthOf(0) expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) - const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - - for (const r of redundancies) { + for (const r of video.redundancies.streamingPlaylists) { expect(r.strategy).to.be.null expect(r.fileUrl).to.exist expect(r.createdAt).to.exist @@ -292,12 +284,10 @@ describe('Test manage videos redundancy', function () { const video = videos[0] expect(video.name).to.equal('video 2 server 2') - expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.files).to.have.lengthOf(0) expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) - const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - - redundanciesToRemove = redundancies.map(r => r.id) + redundanciesToRemove = video.redundancies.streamingPlaylists.map(r => r.id) } }) diff --git a/packages/tests/src/api/redundancy/redundancy-constraints.ts b/packages/tests/src/api/redundancy/redundancy-constraints.ts index 24966b270..f61b5166b 100644 --- a/packages/tests/src/api/redundancy/redundancy-constraints.ts +++ b/packages/tests/src/api/redundancy/redundancy-constraints.ts @@ -92,7 +92,7 @@ describe('Test redundancy constraints', function () { this.timeout(120000) await waitJobs(servers) - await remoteServer.servers.waitUntilLog('Duplicated ', 5) + await remoteServer.servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) { @@ -121,7 +121,7 @@ describe('Test redundancy constraints', function () { await uploadWrapper('video 2 server 2') - await remoteServer.servers.waitUntilLog('Duplicated ', 10) + await remoteServer.servers.waitUntilLog('Duplicated playlist ', 2) await waitJobs(servers) { @@ -150,7 +150,7 @@ describe('Test redundancy constraints', function () { await uploadWrapper('video 3 server 2') - await remoteServer.servers.waitUntilLog('Duplicated ', 15) + await remoteServer.servers.waitUntilLog('Duplicated playlist ', 3) await waitJobs(servers) { @@ -171,7 +171,7 @@ describe('Test redundancy constraints', function () { await waitJobs(servers) await uploadWrapper('video 4 server 2') - await remoteServer.servers.waitUntilLog('Duplicated ', 20) + await remoteServer.servers.waitUntilLog('Duplicated playlist ', 4) await waitJobs(servers) { diff --git a/packages/tests/src/api/redundancy/redundancy.ts b/packages/tests/src/api/redundancy/redundancy.ts index 2540abb40..ad27e0fb5 100644 --- a/packages/tests/src/api/redundancy/redundancy.ts +++ b/packages/tests/src/api/redundancy/redundancy.ts @@ -1,13 +1,8 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' -import { readdir } from 'fs/promises' -import { basename, join } from 'path' import { wait } from '@peertube/peertube-core-utils' import { - HttpStatusCode, VideoDetails, - VideoFile, VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual @@ -17,33 +12,19 @@ import { createMultipleServers, doubleFollow, killallServers, - makeRawRequest, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' import { checkSegmentHash } from '@tests/shared/streaming-playlists.js' import { checkVideoFilesWereRemoved, saveVideoInServers } from '@tests/shared/videos.js' -import { magnetUriDecode } from '@tests/shared/webtorrent.js' +import { expect } from 'chai' +import { readdir } from 'fs/promises' +import { basename, join } from 'path' let servers: PeerTubeServer[] = [] let video1Server2: VideoDetails -async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) { - const parsed = await magnetUriDecode(file.magnetUri) - - for (const ws of baseWebseeds) { - const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`) - expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined - } - - expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) - - for (const url of parsed.urlList) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - } -} - async function createServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebVideo = true) { const strategies: any[] = [] @@ -52,7 +33,7 @@ async function createServers (strategy: VideoRedundancyStrategy | null, addition { min_lifetime: '1 hour', strategy, - size: '400KB', + size: '200KB', ...additionalParams } @@ -100,83 +81,26 @@ async function createServers (strategy: VideoRedundancyStrategy | null, addition await waitJobs(servers) } -async function ensureSameFilenames (videoUUID: string) { - let webVideoFilenames: string[] +async function ensureSameFilenames (videoUUID: string, serverArgs = servers) { let hlsFilenames: string[] - for (const server of servers) { + for (const server of serverArgs) { const video = await server.videos.getWithToken({ id: videoUUID }) - // Ensure we use the same filenames that the origin - - const localWebVideoFilenames = video.files.map(f => basename(f.fileUrl)).sort() + // Ensure we use the same filenames as the origin const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort() - if (webVideoFilenames) expect(webVideoFilenames).to.deep.equal(localWebVideoFilenames) - else webVideoFilenames = localWebVideoFilenames - if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames) else hlsFilenames = localHLSFilenames } - return { webVideoFilenames, hlsFilenames } + return { hlsFilenames } } -async function check1WebSeed (videoUUID?: string) { +async function check0PlaylistRedundancies (videoUUID?: string, serverArgs = servers) { if (!videoUUID) videoUUID = video1Server2.uuid - const webseeds = [ - `${servers[1].url}/static/web-videos/` - ] - - for (const server of servers) { - // With token to avoid issues with video follow constraints - const video = await server.videos.getWithToken({ id: videoUUID }) - - for (const f of video.files) { - await checkMagnetWebseeds(f, webseeds, server) - } - } - - await ensureSameFilenames(videoUUID) -} - -async function check2Webseeds (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2.uuid - - const webseeds = [ - `${servers[0].url}/static/redundancy/`, - `${servers[1].url}/static/web-videos/` - ] - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - for (const file of video.files) { - await checkMagnetWebseeds(file, webseeds, server) - } - } - - const { webVideoFilenames } = await ensureSameFilenames(videoUUID) - - const directories = [ - servers[0].getDirectoryPath('redundancy'), - servers[1].getDirectoryPath('web-videos') - ] - - for (const directory of directories) { - const files = await readdir(directory) - expect(files).to.have.length.at.least(4) - - // Ensure we files exist on disk - expect(files.find(f => webVideoFilenames.includes(f))).to.exist - } -} - -async function check0PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2.uuid - - for (const server of servers) { + for (const server of serverArgs) { // With token to avoid issues with video follow constraints const video = await server.videos.getWithToken({ id: videoUUID }) @@ -185,7 +109,7 @@ async function check0PlaylistRedundancies (videoUUID?: string) { expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) } - await ensureSameFilenames(videoUUID) + await ensureSameFilenames(videoUUID, serverArgs) } async function check1PlaylistRedundancies (videoUUID?: string) { @@ -233,7 +157,7 @@ async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { let statsLength = 1 if (strategy !== 'manual') { - totalSize = 409600 + totalSize = 204800 statsLength = 2 } @@ -247,11 +171,11 @@ async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { return stat } -async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) { +async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual) { const stat = await checkStatsGlobal(strategy) - expect(stat.totalUsed).to.be.at.least(1).and.below(409601) - expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8) + expect(stat.totalUsed).to.be.at.least(1).and.below(204801) + expect(stat.totalVideoFiles).to.equal(4) expect(stat.totalVideos).to.equal(1) } @@ -307,8 +231,7 @@ describe('Test videos redundancy', function () { return createServers(strategy) }) - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() + it('Should not have redundancy', async function () { await check0PlaylistRedundancies() await checkStatsWithoutRedundancy(strategy) }) @@ -317,14 +240,13 @@ describe('Test videos redundancy', function () { return enableRedundancyOnServer1() }) - it('Should have 2 webseeds on the first video', async function () { + it('Should have redundancy on the first video', async function () { this.timeout(80000) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) - await check2Webseeds() await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) }) @@ -337,7 +259,6 @@ describe('Test videos redundancy', function () { await waitJobs(servers) await wait(5000) - await check1WebSeed() await check0PlaylistRedundancies() await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) @@ -357,8 +278,7 @@ describe('Test videos redundancy', function () { return createServers(strategy) }) - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() + it('Should not have redundancy on the first video', async function () { await check0PlaylistRedundancies() await checkStatsWithoutRedundancy(strategy) }) @@ -367,14 +287,13 @@ describe('Test videos redundancy', function () { return enableRedundancyOnServer1() }) - it('Should have 2 webseeds on the first video', async function () { + it('Should have redundancy on the first video', async function () { this.timeout(80000) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) - await check2Webseeds() await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) }) @@ -387,7 +306,6 @@ describe('Test videos redundancy', function () { await waitJobs(servers) await wait(5000) - await check2Webseeds() await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) }) @@ -400,7 +318,6 @@ describe('Test videos redundancy', function () { await waitJobs(servers) await wait(5000) - await check1WebSeed() await check0PlaylistRedundancies() await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) @@ -420,8 +337,7 @@ describe('Test videos redundancy', function () { return createServers(strategy, { min_views: 3 }) }) - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() + it('Should not have redundancy on the first video', async function () { await check0PlaylistRedundancies() await checkStatsWithoutRedundancy(strategy) }) @@ -430,14 +346,13 @@ describe('Test videos redundancy', function () { return enableRedundancyOnServer1() }) - it('Should still have 1 webseed on the first video', async function () { + it('Should still not have redundancy on the first video', async function () { this.timeout(80000) await waitJobs(servers) await wait(15000) await waitJobs(servers) - await check1WebSeed() await check0PlaylistRedundancies() await checkStatsWithoutRedundancy(strategy) }) @@ -452,14 +367,13 @@ describe('Test videos redundancy', function () { await waitJobs(servers) }) - it('Should have 2 webseeds on the first video', async function () { + it('Should now have redundancy on the first video', async function () { this.timeout(80000) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) - await check2Webseeds() await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) }) @@ -492,7 +406,6 @@ describe('Test videos redundancy', function () { }) it('Should have 0 playlist redundancy on the first video', async function () { - await check1WebSeed() await check0PlaylistRedundancies() }) @@ -521,11 +434,11 @@ describe('Test videos redundancy', function () { await waitJobs(servers) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 1) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy, true) + await checkStatsWith1Redundancy(strategy) }) it('Should remove the video and the redundancy files', async function () { @@ -553,8 +466,7 @@ describe('Test videos redundancy', function () { return createServers(null) }) - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() + it('Should not have redundancy on the first video', async function () { await check0PlaylistRedundancies() await checkStatsWithoutRedundancy('manual') }) @@ -563,14 +475,13 @@ describe('Test videos redundancy', function () { await servers[0].redundancy.addVideo({ videoId: video1Server2.id }) }) - it('Should have 2 webseeds on the first video', async function () { + it('Should now have redundancy on the first video', async function () { this.timeout(80000) await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) - await check2Webseeds() await check1PlaylistRedundancies() await checkStatsWith1Redundancy('manual') }) @@ -585,14 +496,13 @@ describe('Test videos redundancy', function () { const video = videos[0] - for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { + for (const r of video.redundancies.streamingPlaylists) { await servers[0].redundancy.removeVideo({ redundancyId: r.id }) } await waitJobs(servers) await wait(5000) - await check1WebSeed() await check0PlaylistRedundancies() await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) @@ -606,26 +516,6 @@ describe('Test videos redundancy', function () { describe('Test expiration', function () { const strategy = 'recently-added' - async function checkContains (servers: PeerTubeServer[], str: string) { - for (const server of servers) { - const video = await server.videos.get({ id: video1Server2.uuid }) - - for (const f of video.files) { - expect(f.magnetUri).to.contain(str) - } - } - } - - async function checkNotContains (servers: PeerTubeServer[], str: string) { - for (const server of servers) { - const video = await server.videos.get({ id: video1Server2.uuid }) - - for (const f of video.files) { - expect(f.magnetUri).to.not.contain(str) - } - } - } - before(async function () { this.timeout(240000) @@ -634,18 +524,18 @@ describe('Test videos redundancy', function () { await enableRedundancyOnServer1() }) - it('Should still have 2 webseeds after 10 seconds', async function () { + it('Should still have redundancy after 10 seconds', async function () { this.timeout(80000) await wait(10000) try { - await checkContains(servers, 'http%3A%2F%2F' + servers[0].hostname + '%3A' + servers[0].port) + await check1PlaylistRedundancies() } catch { // Maybe a server deleted a redundancy in the scheduler await wait(2000) - await checkContains(servers, 'http%3A%2F%2F' + servers[0].hostname + '%3A' + servers[0].port) + await check1PlaylistRedundancies() } }) @@ -656,7 +546,7 @@ describe('Test videos redundancy', function () { await wait(15000) - await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2F' + servers[0].port + '%3A' + servers[0].port) + await check0PlaylistRedundancies(video1Server2.uuid, [ servers[1], servers[2] ]) }) after(async function () { @@ -676,10 +566,9 @@ describe('Test videos redundancy', function () { await enableRedundancyOnServer1() await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) + await servers[0].servers.waitUntilLog('Duplicated playlist ', 1) await waitJobs(servers) - await check2Webseeds() await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) @@ -692,7 +581,7 @@ describe('Test videos redundancy', function () { await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) }) - it('Should cache video 2 webseeds on the first video', async function () { + it('Should replace first video redundancy by video 2', async function () { this.timeout(240000) await waitJobs(servers) @@ -703,10 +592,8 @@ describe('Test videos redundancy', function () { await wait(1000) try { - await check1WebSeed() await check0PlaylistRedundancies() - await check2Webseeds(video2Server2UUID) await check1PlaylistRedundancies(video2Server2UUID) checked = true diff --git a/server/core/controllers/activitypub/client.ts b/server/core/controllers/activitypub/client.ts index 0161ac004..48bf51129 100644 --- a/server/core/controllers/activitypub/client.ts +++ b/server/core/controllers/activitypub/client.ts @@ -49,7 +49,7 @@ import { getVideoLocalViewerValidator, videoCommentGetValidator } from '../../middlewares/validators/index.js' -import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy.js' +import { videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy.js' import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists.js' import { AccountVideoRateModel } from '../../models/account/account-video-rate.js' import { AccountModel } from '../../models/account/account.js' @@ -224,12 +224,6 @@ activityPubClientRouter.get('/video-channels/:nameWithHost/playlists', asyncMiddleware(videoChannelPlaylistsController) ) -activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoFileRedundancyGetValidator), - asyncMiddleware(videoRedundancyController) -) activityPubClientRouter.get('/redundancy/streaming-playlists/:streamingPlaylistType/:videoId', executeIfActivityPub, activityPubRateLimiter, diff --git a/server/core/helpers/webtorrent.ts b/server/core/helpers/webtorrent.ts index 318632fb3..cbff7208a 100644 --- a/server/core/helpers/webtorrent.ts +++ b/server/core/helpers/webtorrent.ts @@ -1,3 +1,11 @@ +import { promisify2 } from '@peertube/peertube-core-utils' +import { sha1 } from '@peertube/peertube-node-utils' +import { WEBSERVER } from '@server/initializers/constants.js' +import { generateTorrentFileName } from '@server/lib/paths.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { MVideoFile } from '@server/types/models/video/video-file.js' +import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js' +import { MVideo } from '@server/types/models/video/video.js' import bencode from 'bencode' import createTorrent from 'create-torrent' import { createWriteStream } from 'fs' @@ -7,15 +15,6 @@ import { encode as magnetUriEncode } from 'magnet-uri' import parseTorrent from 'parse-torrent' import { dirname, join } from 'path' import { pipeline } from 'stream' -import { promisify2 } from '@peertube/peertube-core-utils' -import { isArray } from '@server/helpers/custom-validators/misc.js' -import { WEBSERVER } from '@server/initializers/constants.js' -import { generateTorrentFileName } from '@server/lib/paths.js' -import { VideoPathManager } from '@server/lib/video-path-manager.js' -import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file.js' -import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js' -import { MVideo } from '@server/types/models/video/video.js' -import { sha1 } from '@peertube/peertube-node-utils' import { CONFIG } from '../initializers/config.js' import { logger } from './logger.js' import { generateVideoImportTmpPath } from './utils.js' @@ -25,7 +24,7 @@ import type { Instance, TorrentFile } from 'webtorrent' const createTorrentPromise = promisify2(createTorrent) -async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { +export async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { const id = target.uri || target.torrentName let timer @@ -39,7 +38,10 @@ async function downloadWebTorrentVideo (target: { uri: string, torrentName?: str const webtorrent = new (await import('webtorrent')).default({ natUpnp: false, natPmp: false, - utp: false + utp: false, + lsd: false, + downloadLimit: 5_000_000, + uploadLimit: 5_000_000 } as any) return new Promise((res, rej) => { @@ -99,13 +101,13 @@ async function downloadWebTorrentVideo (target: { uri: string, torrentName?: str }) } -function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { +export function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => { return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath) }) } -async function createTorrentAndSetInfoHashFromPath ( +export async function createTorrentAndSetInfoHashFromPath ( videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, filePath: string @@ -139,7 +141,7 @@ async function createTorrentAndSetInfoHashFromPath ( videoFile.torrentFilename = torrentFilename } -async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { +export async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { const video = extractVideo(videoOrPlaylist) if (!videoFile.torrentFilename) { @@ -177,21 +179,18 @@ async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlayli videoFile.infoHash = sha1(bencode.encode(decoded.info)) } -function generateMagnetUri ( +export function generateMagnetUri ( video: MVideo, - videoFile: MVideoFileRedundanciesOpt, + videoFile: MVideoFile, trackerUrls: string[] ) { const xs = videoFile.getTorrentUrl() const announce = trackerUrls - let urlList = video.hasPrivateStaticPath() + const urlList = video.hasPrivateStaticPath() ? [] : [ videoFile.getFileUrl(video) ] - const redundancies = videoFile.RedundancyVideos - if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) - const magnetHash = { xs, announce, @@ -204,18 +203,7 @@ function generateMagnetUri ( } // --------------------------------------------------------------------------- - -export { - createTorrentPromise, - updateTorrentMetadata, - - createTorrentAndSetInfoHash, - createTorrentAndSetInfoHashFromPath, - - generateMagnetUri, - downloadWebTorrentVideo -} - +// Private // --------------------------------------------------------------------------- function safeWebtorrentDestroy ( diff --git a/server/core/initializers/migrations/0870-remove-web-video-redundancy.ts b/server/core/initializers/migrations/0870-remove-web-video-redundancy.ts new file mode 100644 index 000000000..8bf405d27 --- /dev/null +++ b/server/core/initializers/migrations/0870-remove-web-video-redundancy.ts @@ -0,0 +1,71 @@ +import { logger } from '@server/helpers/logger.js' +import { getServerActor } from '@server/models/application/application.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { remove } from 'fs-extra' +import { join } from 'path' +import * as Sequelize from 'sequelize' +import { QueryTypes } from 'sequelize' +import { CONFIG } from '../config.js' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + const actor = await getServerActor() + + { + const query = 'SELECT "videoFileId" FROM "videoRedundancy" WHERE "actor" = :actorId AND "videoFileId" IS NOT NULL' + + const rows = await utils.sequelize.query<{ videoFileId: number }>(query, { + bind: { actorId: actor.id }, + transaction, + type: QueryTypes.SELECT as QueryTypes.SELECT + }) + + for (const { videoFileId } of rows) { + try { + const videoFile = await VideoFileModel.loadWithVideo(videoFileId, transaction) + const filePath = join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) + + await remove(filePath) + } catch (err) { + logger.error(`Cannot delete redundancy file (videoFileId: ${videoFileId}).`, { err }) + } + } + } + + { + const query = 'DELETE FROM "videoRedundancy" WHERE "videoFileId" IS NOT NULL' + + await utils.sequelize.query(query, { transaction }) + } + + { + await utils.sequelize.query('DROP INDEX IF EXISTS video_redundancy_video_file_id') + } + + { + await utils.queryInterface.removeColumn('videoRedundancy', 'videoFileId', { transaction }) + } + + { + const data = { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: false + } + + await utils.queryInterface.changeColumn('videoRedundancy', 'videoStreamingPlaylistId', data, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + down, up +} diff --git a/server/core/lib/activitypub/cache-file.ts b/server/core/lib/activitypub/cache-file.ts index 475a55a2b..d46274dab 100644 --- a/server/core/lib/activitypub/cache-file.ts +++ b/server/core/lib/activitypub/cache-file.ts @@ -1,8 +1,8 @@ -import { Transaction } from 'sequelize' -import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models/index.js' import { CacheFileObject, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models/index.js' +import { Transaction } from 'sequelize' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js' -import { exists } from '@server/helpers/custom-validators/misc.js' async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) @@ -48,40 +48,22 @@ function updateCacheFile ( } function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { - - if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { - const url = cacheFileObject.url - - const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) - if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) - - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoStreamingPlaylistId: playlist.id, - actorId: byActor.id - } + if (cacheFileObject.url.mediaType !== 'application/x-mpegURL') { + logger.debug('Do not create remoet cache file of non application/x-mpegURL media type', { cacheFileObject }) + return } const url = cacheFileObject.url - const urlFPS = exists(url.fps) // TODO: compat with < 6.1, remove in 8.0 - ? url.fps - : url['_:fps'] - const videoFile = video.VideoFiles.find(f => { - return f.resolution === url.height && f.fps === urlFPS - }) - - if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${urlFPS} of video ${video.url}`) + const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) + if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) return { expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, url: cacheFileObject.id, fileUrl: url.href, strategy: null, - videoFileId: videoFile.id, + videoStreamingPlaylistId: playlist.id, actorId: byActor.id } } diff --git a/server/core/lib/activitypub/send/send-create.ts b/server/core/lib/activitypub/send/send-create.ts index f1e132634..b99d023de 100644 --- a/server/core/lib/activitypub/send/send-create.ts +++ b/server/core/lib/activitypub/send/send-create.ts @@ -19,7 +19,6 @@ import { MLocalVideoViewerWithWatchSections, MVideoAP, MVideoAccountLight, MVideoPlaylistFull, - MVideoRedundancyFileVideo, MVideoRedundancyStreamingPlaylistVideo } from '../../../types/models/index.js' import { audiencify, getAudience } from '../audience.js' @@ -60,7 +59,7 @@ export async function sendCreateVideo (video: MVideoAP, transaction: Transaction export async function sendCreateCacheFile ( byActor: MActorLight, video: MVideoAccountLight, - fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo + fileRedundancy: MVideoRedundancyStreamingPlaylistVideo ) { logger.info('Creating job to send file cache of %s.', fileRedundancy.url, lTags(video.uuid)) diff --git a/server/core/lib/activitypub/url.ts b/server/core/lib/activitypub/url.ts index 0f7aec44e..af04f6411 100644 --- a/server/core/lib/activitypub/url.ts +++ b/server/core/lib/activitypub/url.ts @@ -13,7 +13,6 @@ import { MVideoUrl, MVideoWithHost } from '../../types/models/index.js' -import { MVideoFileVideoUUID } from '../../types/models/video/video-file.js' import { MVideoPlaylist, MVideoPlaylistUUID } from '../../types/models/video/video-playlist.js' import { MStreamingPlaylist } from '../../types/models/video/video-streaming-playlist.js' @@ -29,12 +28,6 @@ export function getLocalVideoPlaylistElementActivityPubUrl (playlist: MVideoPlay return WEBSERVER.URL + '/video-playlists/' + playlist.uuid + '/videos/' + element.id } -export function getLocalVideoCacheFileActivityPubUrl (videoFile: MVideoFileVideoUUID) { - const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' - - return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` -} - export function getLocalVideoCacheStreamingPlaylistActivityPubUrl (video: MVideoUUID, playlist: MStreamingPlaylist) { return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}` } diff --git a/server/core/lib/schedulers/videos-redundancy-scheduler.ts b/server/core/lib/schedulers/videos-redundancy-scheduler.ts index 60d8686b2..077bbf7ab 100644 --- a/server/core/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/core/lib/schedulers/videos-redundancy-scheduler.ts @@ -1,29 +1,25 @@ -import { move } from 'fs-extra/esm' -import { join } from 'path' +import { VideosRedundancyStrategy } from '@peertube/peertube-models' import { getServerActor } from '@server/models/application/application.js' import { VideoModel } from '@server/models/video/video.js' import { MStreamingPlaylistFiles, MVideoAccountLight, MVideoFile, - MVideoFileVideo, - MVideoRedundancyFileVideo, MVideoRedundancyStreamingPlaylistVideo, MVideoRedundancyVideo, MVideoWithAllFiles } from '@server/types/models/index.js' -import { VideosRedundancyStrategy } from '@peertube/peertube-models' +import { join } from 'path' import { logger, loggerTagsFactory } from '../../helpers/logger.js' -import { downloadWebTorrentVideo } from '../../helpers/webtorrent.js' import { CONFIG } from '../../initializers/config.js' import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants.js' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send/index.js' -import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url.js' +import { getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url.js' import { getOrCreateAPVideo } from '../activitypub/videos/index.js' import { downloadPlaylistSegments } from '../hls.js' import { removeVideoRedundancy } from '../redundancy.js' -import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls.js' +import { generateHLSRedundancyUrl } from '../video-urls.js' import { AbstractScheduler } from './abstract-scheduler.js' const lTags = loggerTagsFactory('redundancy') @@ -31,16 +27,9 @@ const lTags = loggerTagsFactory('redundancy') type CandidateToDuplicate = { redundancy: VideosRedundancyStrategy video: MVideoWithAllFiles - files: MVideoFile[] streamingPlaylists: MStreamingPlaylistFiles[] } -function isMVideoRedundancyFileVideo ( - o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo -): o is MVideoRedundancyFileVideo { - return !!(o as MVideoRedundancyFileVideo).VideoFile -} - export class VideosRedundancyScheduler extends AbstractScheduler { private static instance: VideosRedundancyScheduler @@ -62,7 +51,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return this.createVideoRedundancies({ video: videoToDuplicate, redundancy: null, - files: videoToDuplicate.VideoFiles, streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists }) } @@ -197,17 +185,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return } - for (const file of data.files) { - const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) - if (existingRedundancy) { - await this.extendsRedundancy(existingRedundancy) - - continue - } - - await this.createVideoFileRedundancy(data.redundancy, video, file) - } - + // Only HLS player supports redundancy, so do not duplicate web videos for (const streamingPlaylist of data.streamingPlaylists) { const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) if (existingRedundancy) { @@ -220,43 +198,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } - private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { - let strategy = 'manual' - let expiresOn: Date = null - - if (redundancy) { - strategy = redundancy.strategy - expiresOn = this.buildNewExpiration(redundancy.minLifetime) - } - - const file = fileArg as MVideoFileVideo - file.Video = video - - const serverActor = await getServerActor() - - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) - - const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) - - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) - await move(tmpPath, destPath, { overwrite: true }) - - const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn, - url: getLocalVideoCacheFileActivityPubUrl(file), - fileUrl: generateWebVideoRedundancyUrl(file), - strategy, - videoFileId: file.id, - actorId: serverActor.id - }) - - createdModel.VideoFile = file - - await sendCreateCacheFile(serverActor, video, createdModel) - - logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid)) - } - private async createStreamingPlaylistRedundancy ( redundancy: VideosRedundancyStrategy, video: MVideoAccountLight, @@ -278,7 +219,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) - const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 + const maxSizeKB = this.getTotalFileSizes([ playlist ]) / 1000 const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) @@ -315,11 +256,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) if (!toDelete) return - const videoId = toDelete.VideoFile - ? toDelete.VideoFile.videoId - : toDelete.VideoStreamingPlaylist.videoId - - const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) + const redundancies = await VideoRedundancyModel.listLocalByStreamingPlaylistId(toDelete.VideoStreamingPlaylist.id) for (const redundancy of redundancies) { await removeVideoRedundancy(redundancy) @@ -332,7 +269,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) - const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) + const videoSize = this.getTotalFileSizes(candidateToDuplicate.streamingPlaylists) const willUse = alreadyUsed + videoSize logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) }) @@ -344,16 +281,14 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return new Date(Date.now() + expiresAfterMs) } - private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { - if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` - + private buildEntryLogId (object: MVideoRedundancyStreamingPlaylistVideo) { return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` } - private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { + private getTotalFileSizes (playlists: MStreamingPlaylistFiles[]): number { const fileReducer = (previous: number, current: MVideoFile) => previous + current.size - let allFiles = files + let allFiles: MVideoFile[] = [] for (const p of playlists) { allFiles = allFiles.concat(p.VideoFiles) } diff --git a/server/core/lib/video-path-manager.ts b/server/core/lib/video-path-manager.ts index 7bdacc92e..06aa2dbec 100644 --- a/server/core/lib/video-path-manager.ts +++ b/server/core/lib/video-path-manager.ts @@ -17,7 +17,7 @@ import { Mutex } from 'async-mutex' import { remove } from 'fs-extra/esm' import { extname, join } from 'path' import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage/index.js' -import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths.js' +import { getHLSDirectory, getHlsResolutionPlaylistFilename } from './paths.js' import { isVideoInPrivateDirectory } from './video-privacy.js' type MakeAvailableCB = (path: string) => Awaitable @@ -42,16 +42,6 @@ class VideoPathManager { return join(base, filename) } - getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - if (videoFile.isHLS()) { - const video = extractVideo(videoOrPlaylist) - - return join(getHLSRedundancyDirectory(video), videoFile.filename) - } - - return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) - } - getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { const video = extractVideo(videoOrPlaylist) diff --git a/server/core/lib/video-urls.ts b/server/core/lib/video-urls.ts index b07675f57..f2b17d402 100644 --- a/server/core/lib/video-urls.ts +++ b/server/core/lib/video-urls.ts @@ -3,28 +3,15 @@ import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/type // ################## Redundancy ################## -function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { +export function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { // Base URL used by our HLS player return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid } -function generateWebVideoRedundancyUrl (file: MVideoFile) { - return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename -} - // ################## Meta data ################## -function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { +export function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { const path = '/api/v1/videos/' return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id } - -// --------------------------------------------------------------------------- - -export { - getLocalVideoFileMetadataUrl, - - generateWebVideoRedundancyUrl, - generateHLSRedundancyUrl -} diff --git a/server/core/middlewares/validators/redundancy.ts b/server/core/middlewares/validators/redundancy.ts index eac61bf44..6389ede24 100644 --- a/server/core/middlewares/validators/redundancy.ts +++ b/server/core/middlewares/validators/redundancy.ts @@ -18,52 +18,6 @@ import { ServerModel } from '../../models/server/server.js' import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from './shared/index.js' import { canVideoBeFederated } from '@server/lib/activitypub/videos/federate.js' -const videoFileRedundancyGetValidator = [ - isValidVideoIdParam('videoId'), - - param('resolution') - .customSanitizer(toIntOrNull) - .custom(exists), - param('fps') - .optional() - .customSanitizer(toIntOrNull) - .custom(exists), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - if (!canVideoBeFederated(res.locals.onlyVideo)) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) - - const video = res.locals.videoAll - - const paramResolution = req.params.resolution as unknown as number // We casted to int above - const paramFPS = req.params.fps as unknown as number // We casted to int above - - const videoFile = video.VideoFiles.find(f => { - return f.resolution === paramResolution && (!req.params.fps || paramFPS) - }) - - if (!videoFile) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video file not found.' - }) - } - res.locals.videoFile = videoFile - - const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) - if (!videoRedundancy) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video redundancy not found.' - }) - } - res.locals.videoRedundancy = videoRedundancy - - return next() - } -] - const videoPlaylistRedundancyGetValidator = [ isValidVideoIdParam('videoId'), @@ -192,7 +146,6 @@ const removeVideoRedundancyValidator = [ // --------------------------------------------------------------------------- export { - videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator, updateServerRedundancyValidator, listVideoRedundanciesValidator, diff --git a/server/core/models/redundancy/video-redundancy.ts b/server/core/models/redundancy/video-redundancy.ts index aa0126cd5..60c64fb21 100644 --- a/server/core/models/redundancy/video-redundancy.ts +++ b/server/core/models/redundancy/video-redundancy.ts @@ -1,8 +1,6 @@ import { - ActivityVideoUrlObject, CacheFileObject, - FileRedundancyInformation, - StreamingPlaylistRedundancyInformation, + RedundancyInformation, VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy, @@ -10,7 +8,6 @@ import { VideoRedundancyStrategyWithManual } from '@peertube/peertube-models' import { isTestInstance } from '@peertube/peertube-node-utils' -import { getVideoFileMimeType } from '@server/lib/video-file.js' import { getServerActor } from '@server/models/application/application.js' import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models/index.js' import sample from 'lodash-es/sample.js' @@ -47,16 +44,6 @@ export enum ScopeNames { @Scopes(() => ({ [ScopeNames.WITH_VIDEO]: { include: [ - { - model: VideoFileModel, - required: false, - include: [ - { - model: VideoModel, - required: true - } - ] - }, { model: VideoStreamingPlaylistModel, required: false, @@ -75,7 +62,7 @@ export enum ScopeNames { tableName: 'videoRedundancy', indexes: [ { - fields: [ 'videoFileId' ] + fields: [ 'videoStreamingPlaylistId' ] }, { fields: [ 'actorId' ] @@ -114,25 +101,13 @@ export class VideoRedundancyModel extends SequelizeModel { @Column strategy: string // Only used by us - @ForeignKey(() => VideoFileModel) - @Column - videoFileId: number - - @BelongsTo(() => VideoFileModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoFile: Awaited - @ForeignKey(() => VideoStreamingPlaylistModel) @Column videoStreamingPlaylistId: number @BelongsTo(() => VideoStreamingPlaylistModel, { foreignKey: { - allowNull: true + allowNull: false }, onDelete: 'cascade' }) @@ -154,91 +129,28 @@ export class VideoRedundancyModel extends SequelizeModel { static async removeFile (instance: VideoRedundancyModel) { if (!instance.isOwned()) return - if (instance.videoFileId) { - const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) + const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) - const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` - logger.info('Removing duplicated video file %s.', logIdentifier) + const videoUUID = videoStreamingPlaylist.Video.uuid + logger.info('Removing duplicated video streaming playlist %s.', videoUUID) - videoFile.Video.removeWebVideoFile(videoFile, true) - .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) - } - - if (instance.videoStreamingPlaylistId) { - const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) - - const videoUUID = videoStreamingPlaylist.Video.uuid - logger.info('Removing duplicated video streaming playlist %s.', videoUUID) - - videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true) - .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) - } + videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true) + .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) return undefined } - static async loadLocalByFileId (videoFileId: number): Promise { + static async listLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise { const actor = await getServerActor() const query = { where: { actorId: actor.id, - videoFileId + videoStreamingPlaylistId } } - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) - } - - static async listLocalByVideoId (videoId: number): Promise { - const actor = await getServerActor() - - const queryStreamingPlaylist = { - where: { - actorId: actor.id - }, - include: [ - { - model: VideoStreamingPlaylistModel.unscoped(), - required: true, - include: [ - { - model: VideoModel.unscoped(), - required: true, - where: { - id: videoId - } - } - ] - } - ] - } - - const queryFiles = { - where: { - actorId: actor.id - }, - include: [ - { - model: VideoFileModel, - required: true, - include: [ - { - model: VideoModel, - required: true, - where: { - id: videoId - } - } - ] - } - ] - } - - return Promise.all([ - VideoRedundancyModel.findAll(queryStreamingPlaylist), - VideoRedundancyModel.findAll(queryFiles) - ]).then(([ r1, r2 ]) => r1.concat(r2)) + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findAll(query) } static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise { @@ -274,6 +186,87 @@ export class VideoRedundancyModel extends SequelizeModel { return VideoRedundancyModel.findOne(query) } + // --------------------------------------------------------------------------- + // Select redundancy candidates + // --------------------------------------------------------------------------- + + static async findMostViewToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + + // On VideoModel! + const query = { + attributes: [ 'id', 'views' ], + limit: randomizedFactor, + order: getVideoSort('-views'), + where: { + ...this.buildVideoCandidateWhere(), + ...this.buildVideoIdsForDuplication(peertubeActor) + }, + include: [ + VideoRedundancyModel.buildRedundancyAllowedInclude(), + VideoRedundancyModel.buildStreamingPlaylistRequiredInclude() + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async findTrendingToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + + // On VideoModel! + const query = { + attributes: [ 'id', 'views' ], + subQuery: false, + group: 'VideoModel.id', + limit: randomizedFactor, + order: getVideoSort('-trending'), + where: { + ...this.buildVideoCandidateWhere(), + ...this.buildVideoIdsForDuplication(peertubeActor) + }, + include: [ + VideoRedundancyModel.buildRedundancyAllowedInclude(), + VideoRedundancyModel.buildStreamingPlaylistRequiredInclude(), + + VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + const peertubeActor = await getServerActor() + + // On VideoModel! + const query = { + attributes: [ 'id', 'publishedAt' ], + limit: randomizedFactor, + order: getVideoSort('-publishedAt'), + where: { + ...this.buildVideoCandidateWhere(), + ...this.buildVideoIdsForDuplication(peertubeActor), + + views: { + [Op.gte]: minViews + } + }, + include: [ + VideoRedundancyModel.buildRedundancyAllowedInclude(), + VideoRedundancyModel.buildStreamingPlaylistRequiredInclude(), + + // Required by publishedAt sort + { + model: ScheduleVideoUpdateModel.unscoped(), + required: false + } + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + static async isLocalByVideoUUIDExists (uuid: string) { const actor = await getServerActor() @@ -285,13 +278,12 @@ export class VideoRedundancyModel extends SequelizeModel { }, include: [ { - attributes: [], - model: VideoFileModel, + model: VideoStreamingPlaylistModel.unscoped(), required: true, include: [ { attributes: [], - model: VideoModel, + model: VideoModel.unscoped(), required: true, where: { uuid @@ -316,82 +308,49 @@ export class VideoRedundancyModel extends SequelizeModel { return VideoModel.loadWithFiles(id, undefined, !isTestInstance()) } - static async findMostViewToDuplicate (randomizedFactor: number) { - const peertubeActor = await getServerActor() - - // On VideoModel! - const query = { - attributes: [ 'id', 'views' ], - limit: randomizedFactor, - order: getVideoSort('-views'), - where: { - privacy: VideoPrivacy.PUBLIC, - isLive: false, - ...this.buildVideoIdsForDuplication(peertubeActor) - }, - include: [ - VideoRedundancyModel.buildServerRedundancyInclude() - ] + private static buildVideoCandidateWhere () { + return { + privacy: VideoPrivacy.PUBLIC, + remote: true, + isLive: false } - - return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } - static async findTrendingToDuplicate (randomizedFactor: number) { - const peertubeActor = await getServerActor() - - // On VideoModel! - const query = { - attributes: [ 'id', 'views' ], - subQuery: false, - group: 'VideoModel.id', - limit: randomizedFactor, - order: getVideoSort('-trending'), - where: { - privacy: VideoPrivacy.PUBLIC, - isLive: false, - ...this.buildVideoIdsForDuplication(peertubeActor) - }, + private static buildRedundancyAllowedInclude () { + return { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, include: [ - VideoRedundancyModel.buildServerRedundancyInclude(), - - VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) - ] - } - - return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) - } - - static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { - const peertubeActor = await getServerActor() - - // On VideoModel! - const query = { - attributes: [ 'id', 'publishedAt' ], - limit: randomizedFactor, - order: getVideoSort('-publishedAt'), - where: { - privacy: VideoPrivacy.PUBLIC, - isLive: false, - views: { - [Op.gte]: minViews - }, - ...this.buildVideoIdsForDuplication(peertubeActor) - }, - include: [ - VideoRedundancyModel.buildServerRedundancyInclude(), - - // Required by publishedAt sort { - model: ScheduleVideoUpdateModel.unscoped(), - required: false + attributes: [], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ServerModel.unscoped(), + required: true, + where: { + redundancyAllowed: true + } + } + ] } ] } - - return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) } + private static buildStreamingPlaylistRequiredInclude () { + return { + attributes: [], + required: true, + model: VideoStreamingPlaylistModel.unscoped() + } + } + + // --------------------------------------------------------------------------- + static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise { const expiredDate = new Date() expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) @@ -446,60 +405,38 @@ export class VideoRedundancyModel extends SequelizeModel { static async listLocalOfServer (serverId: number) { const actor = await getServerActor() - const buildVideoInclude = () => ({ - model: VideoModel, - required: true, - include: [ - { - attributes: [], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - where: { - serverId - } - } - ] - } - ] - }) const query = { where: { - [Op.and]: [ - { - actorId: actor.id - }, - { - [Op.or]: [ - { - '$VideoStreamingPlaylist.id$': { - [Op.ne]: null - } - }, - { - '$VideoFile.id$': { - [Op.ne]: null - } - } - ] - } - ] + actorId: actor.id }, include: [ - { - model: VideoFileModel.unscoped(), - required: false, - include: [ buildVideoInclude() ] - }, { model: VideoStreamingPlaylistModel.unscoped(), - required: false, - include: [ buildVideoInclude() ] + required: true, + include: [ + { + model: VideoModel, + required: true, + include: [ + { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + where: { + serverId + } + } + ] + } + ] + } + ] } ] } @@ -517,98 +454,60 @@ export class VideoRedundancyModel extends SequelizeModel { const { start, count, sort, target, strategy } = options const redundancyWhere: WhereOptions = {} const videosWhere: WhereOptions = {} - let redundancySqlSuffix = '' if (target === 'my-videos') { Object.assign(videosWhere, { remote: false }) } else if (target === 'remote-videos') { Object.assign(videosWhere, { remote: true }) Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) - redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' } if (strategy) { Object.assign(redundancyWhere, { strategy }) } - const videoFilterWhere = { - [Op.and]: [ - { - [Op.or]: [ - { - id: { - [Op.in]: literal( - '(' + - 'SELECT "videoId" FROM "videoFile" ' + - 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + - redundancySqlSuffix + - ')' - ) - } - }, - { - id: { - [Op.in]: literal( - '(' + - 'select "videoId" FROM "videoStreamingPlaylist" ' + - 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + - redundancySqlSuffix + - ')' - ) - } - } - ] - }, - - videosWhere - ] - } - // /!\ On video model /!\ const findOptions = { offset: start, limit: count, order: getSort(sort), + where: videosWhere, include: [ { - required: false, - model: VideoFileModel, - include: [ - { - model: VideoRedundancyModel.unscoped(), - required: false, - where: redundancyWhere - } - ] - }, - { - required: false, + required: true, model: VideoStreamingPlaylistModel.unscoped(), include: [ { model: VideoRedundancyModel.unscoped(), - required: false, + required: true, where: redundancyWhere }, { model: VideoFileModel, - required: false + required: true } ] } - ], - where: videoFilterWhere - } - - // /!\ On video model /!\ - const countOptions = { - where: videoFilterWhere + ] } return Promise.all([ VideoModel.findAll(findOptions), - VideoModel.count(countOptions) + VideoModel.count({ + where: { + ...videosWhere, + + id: { + [Op.in]: literal( + '(' + + 'SELECT "videoId" FROM "videoStreamingPlaylist" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + + ')' + ) + } + } + }) ]).then(([ data, total ]) => ({ total, data })) } @@ -617,22 +516,16 @@ export class VideoRedundancyModel extends SequelizeModel { const sql = `WITH "tmp" AS ` + `(` + - `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` + - `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + + `SELECT "videoStreamingFile"."size" AS "videoStreamingFileSize", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + `FROM "videoRedundancy" AS "videoRedundancy" ` + - `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` + `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` + `LEFT JOIN "videoFile" AS "videoStreamingFile" ` + `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` + `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` + - `), ` + - `"videoIds" AS (` + - `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` + - `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` + `) ` + `SELECT ` + - `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + - `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` + + `COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + + `COUNT(DISTINCT "videoStreamingVideoId") AS "totalVideos", ` + `COUNT(*) AS "totalVideoFiles" ` + `FROM "tmp"` @@ -647,22 +540,7 @@ export class VideoRedundancyModel extends SequelizeModel { } static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { - const filesRedundancies: FileRedundancyInformation[] = [] - const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] - - for (const file of video.VideoFiles) { - for (const redundancy of file.RedundancyVideos) { - filesRedundancies.push({ - id: redundancy.id, - fileUrl: redundancy.fileUrl, - strategy: redundancy.strategy, - createdAt: redundancy.createdAt, - updatedAt: redundancy.updatedAt, - expiresOn: redundancy.expiresOn, - size: file.size - }) - } - } + const streamingPlaylistsRedundancies: RedundancyInformation[] = [] for (const playlist of video.VideoStreamingPlaylists) { const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) @@ -687,25 +565,18 @@ export class VideoRedundancyModel extends SequelizeModel { uuid: video.uuid, redundancies: { - files: filesRedundancies, + files: [], streamingPlaylists: streamingPlaylistsRedundancies } } } getVideo () { - if (this.VideoFile?.Video) return this.VideoFile.Video - - if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video - - return undefined + return this.VideoStreamingPlaylist.Video } getVideoUUID () { - const video = this.getVideo() - if (!video) return undefined - - return video.uuid + return this.getVideo()?.uuid } isOwned () { @@ -713,37 +584,16 @@ export class VideoRedundancyModel extends SequelizeModel { } toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject { - if (this.VideoStreamingPlaylist) { - return { - id: this.url, - type: 'CacheFile' as 'CacheFile', - object: this.VideoStreamingPlaylist.Video.url, - expires: this.expiresOn ? this.expiresOn.toISOString() : null, - url: { - type: 'Link', - mediaType: 'application/x-mpegURL', - href: this.fileUrl - } - } - } - return { id: this.url, type: 'CacheFile' as 'CacheFile', - object: this.VideoFile.Video.url, - - expires: this.expiresOn - ? this.expiresOn.toISOString() - : null, - + object: this.VideoStreamingPlaylist.Video.url, + expires: this.expiresOn ? this.expiresOn.toISOString() : null, url: { type: 'Link', - mediaType: getVideoFileMimeType(this.VideoFile.extname, this.VideoFile.isAudio()), - href: this.fileUrl, - height: this.VideoFile.resolution, - size: this.VideoFile.size, - fps: this.VideoFile.fps - } as ActivityVideoUrlObject + mediaType: 'application/x-mpegURL', + href: this.fileUrl + } } } @@ -751,10 +601,6 @@ export class VideoRedundancyModel extends SequelizeModel { private static buildVideoIdsForDuplication (peertubeActor: MActor) { const notIn = literal( '(' + - `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` + - `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` + - `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + - `UNION ` + `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` + `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + @@ -767,29 +613,4 @@ export class VideoRedundancyModel extends SequelizeModel { } } } - - private static buildServerRedundancyInclude () { - return { - attributes: [], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ServerModel.unscoped(), - required: true, - where: { - redundancyAllowed: true - } - } - ] - } - ] - } - } } diff --git a/server/core/models/video/formatter/video-api-format.ts b/server/core/models/video/formatter/video-api-format.ts index 66b62b520..c90bb80fe 100644 --- a/server/core/models/video/formatter/video-api-format.ts +++ b/server/core/models/video/formatter/video-api-format.ts @@ -24,7 +24,7 @@ import { VIDEO_STATES } from '../../../initializers/constants.js' import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models/index.js' -import { MVideoFileRedundanciesOpt } from '../../../types/models/video/video-file.js' +import { MVideoFile } from '../../../types/models/video/video-file.js' import { sortByResolutionDesc } from './shared/index.js' export type VideoFormattingJSONOptions = { @@ -208,7 +208,7 @@ export function streamingPlaylistsModelToFormattedJSON ( export function videoFilesModelToFormattedJSON ( video: MVideoFormattable, - videoFiles: MVideoFileRedundanciesOpt[], + videoFiles: MVideoFile[], options: { includeMagnet?: boolean // default true } = {} diff --git a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts index 8ba5b0a75..f796f39e6 100644 --- a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts +++ b/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts @@ -293,19 +293,6 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery { } } - protected includeWebVideoRedundancies () { - this.addJoin( - 'LEFT OUTER JOIN "videoRedundancy" AS "VideoFiles->RedundancyVideos" ON ' + - '"VideoFiles"."id" = "VideoFiles->RedundancyVideos"."videoFileId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoFiles->RedundancyVideos', this.tables.getRedundancyAttributes()) - } - } - protected includeStreamingPlaylistRedundancies () { this.addJoin( 'LEFT OUTER JOIN "videoRedundancy" AS "VideoStreamingPlaylists->RedundancyVideos" ' + diff --git a/server/core/models/video/sql/video/shared/video-file-query-builder.ts b/server/core/models/video/sql/video/shared/video-file-query-builder.ts index b7d3e06d9..976ccb2f6 100644 --- a/server/core/models/video/sql/video/shared/video-file-query-builder.ts +++ b/server/core/models/video/sql/video/shared/video-file-query-builder.ts @@ -44,10 +44,6 @@ export class VideoFileQueryBuilder extends AbstractVideoQueryBuilder { this.includeWebVideoFiles() - if (options.includeRedundancy) { - this.includeWebVideoRedundancies() - } - this.whereId(options) this.query = this.buildQuery() diff --git a/server/core/models/video/sql/video/shared/video-model-builder.ts b/server/core/models/video/sql/video/shared/video-model-builder.ts index ead5b600c..2713568e8 100644 --- a/server/core/models/video/sql/video/shared/video-model-builder.ts +++ b/server/core/models/video/sql/video/shared/video-model-builder.ts @@ -167,7 +167,6 @@ export class VideoModelBuilder { const videoModel = this.videosMemo[row.id] this.addWebVideoFile(row, videoModel) - this.addRedundancy(row, 'VideoFiles', this.videoFileMemo[id]) } } @@ -314,7 +313,7 @@ export class VideoModelBuilder { this.videoFileMemo[id] = videoFileModel } - private addRedundancy (row: SQLRow, prefix: string, to: VideoFileModel | VideoStreamingPlaylistModel) { + private addRedundancy (row: SQLRow, prefix: string, to: VideoStreamingPlaylistModel) { if (!to.RedundancyVideos) to.RedundancyVideos = [] const redundancyPrefix = `${prefix}.RedundancyVideos` diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts index 6c5719231..101fa0607 100644 --- a/server/core/models/video/video-file.ts +++ b/server/core/models/video/video-file.ts @@ -34,7 +34,6 @@ import { Default, DefaultScope, ForeignKey, - HasMany, Is, Scopes, Table, UpdatedAt @@ -56,7 +55,6 @@ import { WEBSERVER } from '../../initializers/constants.js' import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file.js' -import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' import { SequelizeModel, doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js' import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js' import { VideoModel } from './video.js' @@ -252,15 +250,6 @@ export class VideoFileModel extends SequelizeModel { }) VideoStreamingPlaylist: Awaited - @HasMany(() => VideoRedundancyModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE', - hooks: true - }) - RedundancyVideos: Awaited[] - static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist.bind(VideoFileModel), { promise: true, max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, @@ -331,11 +320,11 @@ export class VideoFileModel extends SequelizeModel { } static loadWithMetadata (id: number) { - return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) + return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) } - static loadWithVideo (id: number) { - return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id) + static loadWithVideo (id: number, transaction?: Transaction) { + return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id, { transaction }) } static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts index 3e2b76763..5ea13618f 100644 --- a/server/core/models/video/video.ts +++ b/server/core/models/video/video.ts @@ -40,7 +40,7 @@ import { ModelCache } from '@server/models/shared/model-cache.js' import { MVideoSource } from '@server/types/models/video/video-source.js' import Bluebird from 'bluebird' import { remove } from 'fs-extra/esm' -import { FindOptions, IncludeOptions, Includeable, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' +import { FindOptions, Includeable, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' import { AfterCreate, AfterDestroy, @@ -114,7 +114,6 @@ import { AccountModel } from '../account/account.js' import { ActorImageModel } from '../actor/actor-image.js' import { ActorModel } from '../actor/actor.js' import { VideoAutomaticTagModel } from '../automatic-tag/video-automatic-tag.js' -import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' import { ServerModel } from '../server/server.js' import { TrackerModel } from '../server/tracker.js' import { VideoTrackerModel } from '../server/video-tracker.js' @@ -308,53 +307,30 @@ export type ForAPIOptions = { } ] }, - [ScopeNames.WITH_WEB_VIDEO_FILES]: (withRedundancies = false) => { - let subInclude: any[] = [] - - if (withRedundancies === true) { - subInclude = [ - { - attributes: [ 'fileUrl' ], - model: VideoRedundancyModel.unscoped(), - required: false - } - ] - } - + [ScopeNames.WITH_WEB_VIDEO_FILES]: () => { return { include: [ { model: VideoFileModel, separate: true, - required: false, - include: subInclude + required: false } ] } }, - [ScopeNames.WITH_STREAMING_PLAYLISTS]: (withRedundancies = false) => { - const subInclude: IncludeOptions[] = [ - { - model: VideoFileModel, - required: false - } - ] - - if (withRedundancies === true) { - subInclude.push({ - attributes: [ 'fileUrl' ], - model: VideoRedundancyModel.unscoped(), - required: false - }) - } - + [ScopeNames.WITH_STREAMING_PLAYLISTS]: () => { return { include: [ { model: VideoStreamingPlaylistModel.unscoped(), required: false, separate: true, - include: subInclude + include: [ + { + model: VideoFileModel, + required: false + } + ] } ] } @@ -2015,19 +1991,19 @@ export class VideoModel extends SequelizeModel { // --------------------------------------------------------------------------- - removeWebVideoFile (videoFile: MVideoFile, isRedundancy = false) { - const filePath = isRedundancy - ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) - : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) + removeWebVideoFile (videoFile: MVideoFile) { + const filePath = VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) - const promises: Promise[] = [ remove(filePath) ] - if (!isRedundancy) promises.push(videoFile.removeTorrent()) + const promises: Promise[] = [ + remove(filePath), + videoFile.removeTorrent() + ] if (videoFile.storage === FileStorage.OBJECT_STORAGE) { promises.push(removeWebVideoObjectStorage(videoFile)) } - logger.debug(`Removing files associated to web video ${videoFile.filename}`, { videoFile, isRedundancy, ...lTags(this.uuid) }) + logger.debug(`Removing files associated to web video ${videoFile.filename}`, { videoFile, ...lTags(this.uuid) }) return Promise.all(promises) } diff --git a/server/core/types/models/video/video-file.ts b/server/core/types/models/video/video-file.ts index 8431b6f5a..f0e85992c 100644 --- a/server/core/types/models/video/video-file.ts +++ b/server/core/types/models/video/video-file.ts @@ -1,14 +1,13 @@ -import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { PickWith } from '@peertube/peertube-typescript-utils' import { VideoFileModel } from '../../../models/video/video-file.js' -import { MVideo, MVideoUUID } from './video.js' -import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy.js' import { MStreamingPlaylist, MStreamingPlaylistVideo } from './video-streaming-playlist.js' +import { MVideo, MVideoUUID } from './video.js' type Use = PickWith // ############################################################################ -export type MVideoFile = Omit +export type MVideoFile = Omit export type MVideoFileVideo = MVideoFile & @@ -26,14 +25,6 @@ export type MVideoFileVideoUUID = MVideoFile & Use<'Video', MVideoUUID> -export type MVideoFileRedundanciesAll = - MVideoFile & - PickWithOpt - -export type MVideoFileRedundanciesOpt = - MVideoFile & - PickWithOpt - export function isStreamingPlaylistFile (file: any): file is MVideoFileStreamingPlaylist { return !!file.videoStreamingPlaylistId } diff --git a/server/core/types/models/video/video-redundancy.ts b/server/core/types/models/video/video-redundancy.ts index 55014cdf6..3cf06f5c8 100644 --- a/server/core/types/models/video/video-redundancy.ts +++ b/server/core/types/models/video/video-redundancy.ts @@ -1,36 +1,25 @@ -import { VideoFileModel } from '@server/models/video/video-file.js' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy.js' -import { MVideoUrl } from './video.js' -import { MVideoFile, MVideoFileVideo } from './video-file.js' import { MStreamingPlaylistVideo } from './video-streaming-playlist.js' +import { MVideoUrl } from './video.js' type Use = PickWith // ############################################################################ -export type MVideoRedundancy = Omit +export type MVideoRedundancy = Omit export type MVideoRedundancyFileUrl = Pick // ############################################################################ -export type MVideoRedundancyFile = - MVideoRedundancy & - Use<'VideoFile', MVideoFile> - -export type MVideoRedundancyFileVideo = - MVideoRedundancy & - Use<'VideoFile', MVideoFileVideo> - export type MVideoRedundancyStreamingPlaylistVideo = MVideoRedundancy & Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> export type MVideoRedundancyVideo = MVideoRedundancy & - Use<'VideoFile', MVideoFileVideo> & Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> // ############################################################################ @@ -39,5 +28,4 @@ export type MVideoRedundancyVideo = export type MVideoRedundancyAP = MVideoRedundancy & - PickWithOpt> & PickWithOpt> diff --git a/server/core/types/models/video/video.ts b/server/core/types/models/video/video.ts index de462176b..e3ec16d9d 100644 --- a/server/core/types/models/video/video.ts +++ b/server/core/types/models/video/video.ts @@ -18,7 +18,7 @@ import { MChannelHostOnly, MChannelUserId } from './video-channel.js' -import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file.js' +import { MVideoFile } from './video-file.js' import { MVideoLive } from './video-live.js' import { MStreamingPlaylistFiles, @@ -181,7 +181,7 @@ export type MVideoAP = Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & Use<'VideoCaptions', MVideoCaptionLanguageUrl[]> & Use<'VideoBlacklist', MVideoBlacklistUnfederated> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'VideoFiles', MVideoFile[]> & Use<'Thumbnails', MThumbnail[]> & Use<'VideoLive', MVideoLive> & Use<'Storyboard', MStoryboard> @@ -197,7 +197,7 @@ export type MVideoDetails = Use<'Thumbnails', MThumbnail[]> & Use<'UserVideoHistories', MUserVideoHistoryTime[]> & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundancies[]> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'VideoFiles', MVideoFile[]> & Use<'Trackers', MTrackerUrl[]> export type MVideoForUser = @@ -209,7 +209,6 @@ export type MVideoForUser = export type MVideoForRedundancyAPI = MVideo & - Use<'VideoFiles', MVideoFileRedundanciesAll[]> & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]> // ############################################################################ @@ -230,5 +229,5 @@ export type MVideoFormattableDetails = Use<'VideoChannel', MChannelFormattable> & Use<'Tags', MTag[]> & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'VideoFiles', MVideoFile[]> & PickWithOpt diff --git a/server/scripts/prune-storage.ts b/server/scripts/prune-storage.ts index 9356be5be..84a1fec37 100755 --- a/server/scripts/prune-storage.ts +++ b/server/scripts/prune-storage.ts @@ -303,11 +303,8 @@ class FSPruner { return !!redundancy } - const file = await VideoFileModel.loadByFilename(basename(filePath)) - if (!file) return false - - const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) - return !!redundancy + // WebTorrent support redundancy has been removed from PeerTube + return false } } diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 2712710ca..5d3c8561e 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -8502,10 +8502,6 @@ components: redundancies: type: object properties: - files: - type: array - items: - $ref: '#/components/schemas/FileRedundancyInformation' streamingPlaylists: type: array items: diff --git a/yarn.lock b/yarn.lock index 863b643d0..8e5559dcf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2460,27 +2460,18 @@ dependencies: defer-to-connect "^2.0.1" -"@thaunknown/idb-chunk-store@^1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@thaunknown/idb-chunk-store/-/idb-chunk-store-1.0.4.tgz#901c0c51a07c0c91e50da681c8179f57f384fe72" - integrity sha512-4/XDQZHKHyJCGeqnVjHyqeAXClZJ9l90rRvoTslUiuvwTGAUpIb3poL0LfGJEdSuWV+zzGdDjIm/3L4x6crwbg== +"@thaunknown/simple-peer@^10.0.11", "@thaunknown/simple-peer@^10.0.8": + version "10.0.11" + resolved "https://registry.yarnpkg.com/@thaunknown/simple-peer/-/simple-peer-10.0.11.tgz#0b5ec1ed8b1c6aaddf846ce2274604d511c1648a" + integrity sha512-A5MdmtZ6HUzRa4gwPOS4jG+09HvpTv2rFo4kk7Vwveo2ELm+WmbO124ZrJrQnZc2D7z2Q3AWKSitjl9OKXO88g== dependencies: - idb "^7.1.1" - queue-microtask "^1.2.3" - -"@thaunknown/simple-peer@^9.12.1": - version "9.12.1" - resolved "https://registry.yarnpkg.com/@thaunknown/simple-peer/-/simple-peer-9.12.1.tgz#c712335a1043f85ac305a54c8c33abc181e26c74" - integrity sha512-IS5BXvXx7cvBAzaxqotJf4s4rJCPk5JABLK6Gbnn7oAmWVcH4hYABabBBrvvJtv/xyUqR4v/H3LalnGRJJfEog== - dependencies: - debug "^4.3.2" + debug "^4.3.7" err-code "^3.0.1" - get-browser-rtc "^1.1.0" - queue-microtask "^1.2.3" - streamx "^2.13.2" - uint8-util "^2.1.9" + streamx "^2.20.1" + uint8-util "^2.2.5" + webrtc-polyfill "^1.1.10" -"@thaunknown/simple-websocket@^9.1.0": +"@thaunknown/simple-websocket@^9.1.3": version "9.1.3" resolved "https://registry.yarnpkg.com/@thaunknown/simple-websocket/-/simple-websocket-9.1.3.tgz#843065027c6cf4470fb08ca78dbf9e48afc56ea6" integrity sha512-pf/FCJsgWtLJiJmIpiSI7acOZVq3bIQCpnNo222UFc8Ph1lOUOTpe6LoYhhiOSKB9GUaWJEVUtZ+sK1/aBgU5Q== @@ -2491,7 +2482,7 @@ uint8-util "^2.2.5" ws "^8.17.1" -"@thaunknown/thirty-two@^1.0.3": +"@thaunknown/thirty-two@^1.0.3", "@thaunknown/thirty-two@^1.0.5": version "1.0.5" resolved "https://registry.yarnpkg.com/@thaunknown/thirty-two/-/thirty-two-1.0.5.tgz#c41c1756e150854cb7a264f343c0de2d0ea7cbb8" integrity sha512-Q53KyCXweV1CS62EfqtPDqfpksn5keQ59PGqzzkK+g8Vif1jB4inoBCcs/BUSdsqddhE3G+2Fn+4RX3S6RqT0A== @@ -3667,18 +3658,18 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== -bitfield@^4.0.0, bitfield@^4.1.0: +bitfield@^4.0.0, bitfield@^4.1.0, bitfield@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/bitfield/-/bitfield-4.2.0.tgz#fecb620bbe38e16526fbb62048f6c4af712ace91" integrity sha512-kUTatQb/mBd8uhvdLrUkouGDBUQiJaIOvPlptUwOWp6MFqih4d1MiVf0m3ATxfZSzu+LjW/awFeABltYa62uIA== -bittorrent-dht@^11.0.5: - version "11.0.8" - resolved "https://registry.yarnpkg.com/bittorrent-dht/-/bittorrent-dht-11.0.8.tgz#623b489c12e6ac141e3e9c7ce71e555edbd285cb" - integrity sha512-hWNmv297wWLpTAkdhG15MJLDXkPXaG//9jRoT62WHja565fjlZojO3WZ7vlzbGRp7o58jnd1fx78dZlpq5d3zA== +bittorrent-dht@^11.0.9: + version "11.0.9" + resolved "https://registry.yarnpkg.com/bittorrent-dht/-/bittorrent-dht-11.0.9.tgz#cd4c169e9dabc7d54856fb7971a9ab164d683160" + integrity sha512-aM6m9zvIGi8lMANaxUWcF3yytUxloUCc4gzqa0SOvo22FyeNDHecOXccw6FIZyk4I0IN9KDCj7We+n+RpqnYgg== dependencies: bencode "^4.0.0" - debug "^4.3.7" + debug "^4.4.0" k-bucket "^5.1.0" k-rpc "^5.1.0" last-one-wins "^1.0.4" @@ -3694,55 +3685,63 @@ bittorrent-lsd@^2.0.0: chrome-dgram "^3.0.6" debug "^4.2.0" -bittorrent-peerid@^1.3.3, bittorrent-peerid@^1.3.6: +bittorrent-peerid@^1.3.6: version "1.3.6" resolved "https://registry.yarnpkg.com/bittorrent-peerid/-/bittorrent-peerid-1.3.6.tgz#3688705a64937a8176ac2ded1178fc7bd91b61db" integrity sha512-VyLcUjVMEOdSpHaCG/7odvCdLbAB1y3l9A2V6WIje24uV7FkJPrQrH/RrlFmKxP89pFVDEnE+YlHaFujlFIZsg== -bittorrent-protocol@^4.1.11: - version "4.1.15" - resolved "https://registry.yarnpkg.com/bittorrent-protocol/-/bittorrent-protocol-4.1.15.tgz#1ea471df34b4d1d4be59dea5a0c0f8815c113894" - integrity sha512-41W08svaxGrNtxwMl7DbOcYnp44wcNs1B4szSfdLNjCRQH7yWdGdSOTNOvEi+FtsRVyNWabadM6IZLbhj5SS2w== +bittorrent-protocol@^4.1.16: + version "4.1.16" + resolved "https://registry.yarnpkg.com/bittorrent-protocol/-/bittorrent-protocol-4.1.16.tgz#fa9536eafe7e8abab941e5da853b09da6668de60" + integrity sha512-93t8h77uAyD8BGSpBo8SqxYyKBA/xgv9N8+WghnXpH2I+JmlmJmddUt8nugPRgj/LNuL1VrWJ26jhYhiVWpRaQ== dependencies: bencode "^4.0.0" bitfield "^4.1.0" - debug "^4.3.7" + debug "^4.4.0" rc4 "^0.1.5" streamx "^2.15.1" throughput "^1.0.1" uint8-util "^2.2.5" unordered-array-remove "^1.0.2" -bittorrent-tracker@^10.0.12: - version "10.0.12" - resolved "https://registry.yarnpkg.com/bittorrent-tracker/-/bittorrent-tracker-10.0.12.tgz#084fb250317f69033f5f1c4ed6a9cddf6b9acf61" - integrity sha512-EYQEwhOYkrRiiwkCFcM9pbzJInsAe7UVmUgevW133duwlZzjwf5ABwDE7pkkmNRS6iwN0b8LbI/94q16dYqiow== +bittorrent-tracker@^11.2.1: + version "11.2.1" + resolved "https://registry.yarnpkg.com/bittorrent-tracker/-/bittorrent-tracker-11.2.1.tgz#b45ff4bd70c2c582bc60d4a8bb6b5bdcb487df9a" + integrity sha512-SffBgHzNrhn+HBwdRD2st+TYJOs2LhF3ljJFPCYGv592LpGtPxw41UZHTUeY5muWnQl+wopcU8qXM9UEk2WKrA== dependencies: - "@thaunknown/simple-peer" "^9.12.1" - "@thaunknown/simple-websocket" "^9.1.0" + "@thaunknown/simple-peer" "^10.0.8" + "@thaunknown/simple-websocket" "^9.1.3" bencode "^4.0.0" - bittorrent-peerid "^1.3.3" + bittorrent-peerid "^1.3.6" chrome-dgram "^3.0.6" - clone "^2.0.0" compact2string "^1.4.1" - debug "^4.1.1" - ip "^1.1.5" + cross-fetch-ponyfill "^1.0.3" + debug "^4.3.4" + ip "^2.0.1" lru "^3.1.0" - minimist "^1.2.5" + minimist "^1.2.8" once "^1.4.0" queue-microtask "^1.2.3" random-iterate "^1.0.1" run-parallel "^1.2.0" run-series "^1.1.9" - simple-get "^4.0.0" - socks "^2.0.0" - string2compact "^2.0.0" - uint8-util "^2.1.9" + socks "^2.8.3" + string2compact "^2.0.1" + uint8-util "^2.2.5" unordered-array-remove "^1.0.2" - ws "^8.0.0" + ws "^8.17.0" optionalDependencies: - bufferutil "^4.0.3" - utf-8-validate "^5.0.5" + bufferutil "^4.0.8" + utf-8-validate "^6.0.4" + +bl@^4.0.3: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" + integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== + dependencies: + buffer "^5.5.0" + inherits "^2.0.4" + readable-stream "^3.4.0" block-iterator@^1.1.1: version "1.1.1" @@ -3852,7 +3851,7 @@ buffer@5.6.0: base64-js "^1.0.2" ieee754 "^1.1.4" -buffer@^5.2.0, buffer@^5.2.1: +buffer@^5.2.0, buffer@^5.2.1, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -3868,7 +3867,7 @@ buffer@^6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" -bufferutil@^4.0.3, bufferutil@^4.0.8: +bufferutil@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/bufferutil/-/bufferutil-4.0.8.tgz#1de6a71092d65d7766c4d8a522b261a6e787e8ea" integrity sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw== @@ -4114,6 +4113,11 @@ chokidar@^3.4.2, chokidar@^3.5.3: optionalDependencies: fsevents "~2.3.2" +chownr@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" + integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + chownr@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" @@ -4194,11 +4198,6 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" -clone@^2.0.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" - integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w== - cluster-key-slot@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" @@ -4472,6 +4471,24 @@ create-torrent@^6.0.15: run-parallel "^1.2.0" uint8-util "^2.2.5" +create-torrent@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/create-torrent/-/create-torrent-6.1.0.tgz#c8ed4e5f1575729bf5ebba141b81e4479c77ec54" + integrity sha512-War593HCsg4TotHgMGWTJqnDHN0pmEU2RM13xUzzSZ78TpRNOC2bbcsC5yMO3pqIkedHEWFzYNqH1yhwuuBYTg== + dependencies: + bencode "^4.0.0" + block-iterator "^1.1.1" + fast-readable-async-iterator "^2.0.0" + is-file "^1.0.0" + join-async-iterator "^1.1.1" + junk "^4.0.1" + minimist "^1.2.8" + once "^1.4.0" + piece-length "^2.0.1" + queue-microtask "^1.2.3" + run-parallel "^1.2.0" + uint8-util "^2.2.5" + cron-parser@^4.6.0: version "4.9.0" resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5" @@ -4599,7 +4616,7 @@ debug@2.6.9: dependencies: ms "2.0.0" -debug@4, debug@^4.1.1, debug@^4.2.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@^4.3.5, debug@^4.3.7, debug@~4.3.1, debug@~4.3.2, debug@~4.3.4: +debug@4, debug@^4.2.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@^4.3.5, debug@^4.3.7, debug@~4.3.1, debug@~4.3.2, debug@~4.3.4: version "4.3.7" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== @@ -4613,6 +4630,13 @@ debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.0.tgz#2b3f2aea2ffeb776477460267377dc8710faba8a" + integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA== + dependencies: + ms "^2.1.3" + decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -4988,7 +5012,7 @@ encoding-japanese@2.1.0: resolved "https://registry.yarnpkg.com/encoding-japanese/-/encoding-japanese-2.1.0.tgz#5d3c2b652c84ca563783b86907bf5cdfe9a597e2" integrity sha512-58XySVxUgVlBikBTbQ8WdDxBDHIdXucB16LO5PBHR8t75D54wQrNo4cg+58+R1CtJfKnsVsvt9XlteRaR8xw1w== -end-of-stream@^1.0.0, end-of-stream@^1.1.0: +end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== @@ -5561,6 +5585,11 @@ exif-parser@^0.1.12: resolved "https://registry.yarnpkg.com/exif-parser/-/exif-parser-0.1.12.tgz#58a9d2d72c02c1f6f02a0ef4a9166272b7760922" integrity sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw== +expand-template@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" + integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== + expand-tilde@^2.0.0, expand-tilde@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-2.0.2.tgz#97e801aa052df02454de46b02bf621642cdc8502" @@ -5953,6 +5982,11 @@ fs-chunk-store@^4.1.0: run-parallel "^1.1.2" thunky "^1.0.1" +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== + fs-extra@^11.1.0: version "11.2.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.2.0.tgz#e70e17dfad64232287d01929399e0ea7c86b0e5b" @@ -5982,7 +6016,7 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsa-chunk-store@^1.1.5: +fsa-chunk-store@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/fsa-chunk-store/-/fsa-chunk-store-1.3.0.tgz#e43a6104a01dbff1e9a13fecd8a715dcfcc42987" integrity sha512-0WCfuxqqSB6Tz/g7Ar/nwAxMoigXaIXuvfrnLIEFYIA9uc6w9eNaHuBGzU1X3lyM4cpLKCOTUmKAA/gCiTvzMQ== @@ -6029,11 +6063,6 @@ gauge@^3.0.0: strip-ansi "^6.0.1" wide-align "^1.1.2" -get-browser-rtc@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz#d1494e299b00f33fc8e9d6d3343ba4ba99711a2c" - integrity sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ== - get-caller-file@^2.0.1, get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" @@ -6121,6 +6150,11 @@ gifwrap@^0.10.1: image-q "^4.0.0" omggif "^1.0.10" +github-from-package@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" + integrity sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw== + glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -6494,16 +6528,6 @@ human-signals@^8.0.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-8.0.0.tgz#2d3d63481c7c2319f0373428b01ffe30da6df852" integrity sha512-/1/GPCpDUCCYwlERiYjxoczfP0zfvZMU/OWgQPMya9AbAE24vseigFdhAMObpc8Q4lc/kjutPfUddDYyAmejnA== -hybrid-chunk-store@^1.2.2: - version "1.2.6" - resolved "https://registry.yarnpkg.com/hybrid-chunk-store/-/hybrid-chunk-store-1.2.6.tgz#161f506bc49899c03937cfc2e2b0946a5bcc8eac" - integrity sha512-D8DkY6FT+exjw4b6uQ8z5QfUokcIb0YYPHaa/zpBdFIoS1CS7mjM4wnd2mGoo2XUeM5Y10C23AXOQRExoifPbA== - dependencies: - "@thaunknown/idb-chunk-store" "^1.0.4" - cache-chunk-store "^3.2.2" - fsa-chunk-store "^1.1.5" - memory-chunk-store "^1.3.5" - hyperid@^3.0.0: version "3.3.0" resolved "https://registry.yarnpkg.com/hyperid/-/hyperid-3.3.0.tgz#2042bb296b7f1d5ba0797a5705469af0899c8556" @@ -6546,11 +6570,6 @@ iconv-lite@0.6.3: dependencies: safer-buffer ">= 2.1.2 < 3.0.0" -idb@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/idb/-/idb-7.1.1.tgz#d910ded866d32c7ced9befc5bfdf36f572ced72b" - integrity sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ== - ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -6685,11 +6704,6 @@ ip-set@^2.1.0: dependencies: ip "^2.0.1" -ip@^1.1.5: - version "1.1.9" - resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.9.tgz#8dfbcc99a754d07f425310b86a99546b1151e396" - integrity sha512-cyRxvOEpNHNtchU3Ln9KC/auJgup87llfQpQ+t5ghoC/UhL16SWzbueiCsdTnWmqAWl7LadfuwhlqmtOaqMHdQ== - ip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.1.tgz#e8f3595d33a3ea66490204234b77636965307105" @@ -7579,10 +7593,10 @@ lru@^3.1.0: dependencies: inherits "^2.0.1" -lt_donthave@^2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/lt_donthave/-/lt_donthave-2.0.3.tgz#ec8149a4adce1d2b6d9d8a8657520808f24d5f83" - integrity sha512-wC1ATeT+y6CRZ7RFm6LFFuQvAa1rWVeW6KZa7VqTYuqA5yS+69ddSrtKMjPqQ2Vh+kX2jY5wwxaEjgXAuLlXeA== +lt_donthave@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/lt_donthave/-/lt_donthave-2.0.4.tgz#94460f5c2498826c833c90010bfcda32d473147f" + integrity sha512-VIKjdxflF8+6vFb3t8LQ4czRYvw6OyxPLDr5YV5qOieu4qwl0wX2DA18WyaHJjBKyKSHXvdo1JcrrUag5MmMiA== dependencies: debug "^4.2.0" unordered-array-remove "^1.0.2" @@ -7608,6 +7622,15 @@ magnet-uri@^7.0.5: bep53-range "^2.0.0" uint8-util "^2.1.9" +magnet-uri@^7.0.7: + version "7.0.7" + resolved "https://registry.yarnpkg.com/magnet-uri/-/magnet-uri-7.0.7.tgz#ffc6c7e731f6d8cae2c9b17593a4f92243f4dbde" + integrity sha512-z/+dB2NQsXaDuxVBjoPLpZT8ePaacUmoontoFheRBl++nALHYs4qV9MmhTur9e4SaMbkCR/uPX43UMzEOoeyaw== + dependencies: + "@thaunknown/thirty-two" "^1.0.5" + bep53-range "^2.0.0" + uint8-util "^2.2.5" + mailparser-mit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/mailparser-mit/-/mailparser-mit-1.0.0.tgz#19df8436c2a02e1d34a03ec518a2eb065e0a94a4" @@ -7843,7 +7866,7 @@ minimatch@^9.0.4: dependencies: brace-expansion "^2.0.1" -minimist@^1.1.0, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8: +minimist@^1.1.0, minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.6, minimist@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -7873,6 +7896,11 @@ minizlib@^2.1.1: minipass "^3.0.0" yallist "^4.0.0" +mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + mkdirp@^0.5.4: version "0.5.6" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" @@ -8038,6 +8066,11 @@ nanoid@^3.3.7: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +napi-build-utils@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-2.0.0.tgz#13c22c0187fcfccce1461844136372a47ddc027e" + integrity sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA== + napi-macros@^2.0.0: version "2.2.2" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.2.2.tgz#817fef20c3e0e40a963fbf7b37d1600bd0201044" @@ -8078,6 +8111,13 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-abi@^3.3.0: + version "3.73.0" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.73.0.tgz#4459ea77e71969edba8588387eecb05e2c2cff3b" + integrity sha512-z8iYzQGBu35ZkTQ9mtR8RqugJZ9RCLn8fv3d7LsgDBzOijGQP3RdKTX4LA7LXw03ZhU5z0l4xfhIMgSES31+cg== + dependencies: + semver "^7.3.5" + node-abort-controller@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" @@ -8093,11 +8133,24 @@ node-cleanup@^2.1.2: resolved "https://registry.yarnpkg.com/node-cleanup/-/node-cleanup-2.1.2.tgz#7ac19abd297e09a7f72a71545d951b517e4dde2c" integrity sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw== +node-datachannel@^v0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/node-datachannel/-/node-datachannel-0.12.0.tgz#7700767bf178bc6d258560c78181b56abb800e44" + integrity sha512-pZ9FsVZpHdUKqyWynuCc9IBLkZPJMpDzpNk4YNPCizbIXHYifpYeWqSF35REHGIWi9JMCf11QzapsyQGo/Y4Ig== + dependencies: + node-domexception "^2.0.1" + prebuild-install "^7.0.1" + node-domexception@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== +node-domexception@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-2.0.1.tgz#83b0d101123b5bbf91018fd569a58b88ae985e5b" + integrity sha512-M85rnSC7WQ7wnfQTARPT4LrK7nwCHLdDFOCcItZMhTQjyCebJH8GciKqYJNgaOFZs9nFmTmd/VMyi3OW5jA47w== + node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -8538,6 +8591,18 @@ parse-torrent@^11.0.14: queue-microtask "^1.2.3" uint8-util "^2.2.5" +parse-torrent@^11.0.18: + version "11.0.18" + resolved "https://registry.yarnpkg.com/parse-torrent/-/parse-torrent-11.0.18.tgz#2ae7e52160fd0e59b6f1e539fb3670a71882d7a6" + integrity sha512-C1igbmTrQQuKlspAfP1wcLaOPlvtu5qi4pMdPoCCfepHmxDOk8iArJ2J1yblLx11UefZJUaKEPSxIwMdG11SuA== + dependencies: + bencode "^4.0.0" + cross-fetch-ponyfill "^1.0.3" + get-stdin "^9.0.0" + magnet-uri "^7.0.7" + queue-microtask "^1.2.3" + uint8-util "^2.2.5" + parse5-htmlparser2-tree-adapter@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz#b5a806548ed893a43e24ccb42fbb78069311e81b" @@ -8862,6 +8927,24 @@ postgres-range@^1.1.1: resolved "https://registry.yarnpkg.com/postgres-range/-/postgres-range-1.1.4.tgz#a59c5f9520909bcec5e63e8cf913a92e4c952863" integrity sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w== +prebuild-install@^7.0.1: + version "7.1.3" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.3.tgz#d630abad2b147443f20a212917beae68b8092eec" + integrity sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug== + dependencies: + detect-libc "^2.0.0" + expand-template "^2.0.3" + github-from-package "0.0.0" + minimist "^1.2.3" + mkdirp-classic "^0.5.3" + napi-build-utils "^2.0.0" + node-abi "^3.3.0" + pump "^3.0.0" + rc "^1.2.7" + simple-get "^4.0.0" + tar-fs "^2.0.0" + tunnel-agent "^0.6.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -9074,7 +9157,7 @@ pump@^2.0.0: end-of-stream "^1.1.0" once "^1.3.1" -pump@^3.0.0: +pump@^3.0.0, pump@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.2.tgz#836f3edd6bc2ee599256c924ffe0d88573ddcbf8" integrity sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw== @@ -9199,7 +9282,7 @@ rc4@^0.1.5: resolved "https://registry.yarnpkg.com/rc4/-/rc4-0.1.5.tgz#08c6e04a0168f6eb621c22ab6cb1151bd9f4a64d" integrity sha512-xdDTNV90z5x5u25Oc871Xnvu7yAr4tV7Eluh0VSvrhUkry39q1k+zkz7xroqHbRq+8PiazySHJPArqifUvz9VA== -rc@^1.2.8: +rc@^1.2.7, rc@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== @@ -9256,7 +9339,7 @@ readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.2.2, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.0.2, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0, readable-stream@^3.6.2: +readable-stream@^3.0.2, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0, readable-stream@^3.6.2: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== @@ -9911,7 +9994,7 @@ socket.io@^4.5.4: socket.io-adapter "~2.5.2" socket.io-parser "~4.2.4" -socks@^2.0.0: +socks@^2.8.3: version "2.8.3" resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.3.tgz#1ebd0f09c52ba95a09750afe3f3f9f724a800cb5" integrity sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw== @@ -10066,7 +10149,18 @@ streamsearch@^1.1.0: resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== -streamx@^2.10.3, streamx@^2.13.2, streamx@^2.15.0, streamx@^2.15.1, streamx@^2.17.0, streamx@^2.20.0: +streamx@2.21.1: + version "2.21.1" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.21.1.tgz#f02979d8395b6b637d08a589fb514498bed55845" + integrity sha512-PhP9wUnFLa+91CPy3N6tiQsK+gnYyUNuk15S3YG/zjYE7RuPeCjJngqnzpC31ow0lzBHQ+QGO4cNJnd0djYUsw== + dependencies: + fast-fifo "^1.3.2" + queue-tick "^1.0.1" + text-decoder "^1.1.0" + optionalDependencies: + bare-events "^2.2.0" + +streamx@^2.10.3, streamx@^2.15.0, streamx@^2.15.1, streamx@^2.17.0, streamx@^2.20.0: version "2.20.2" resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.20.2.tgz#6a8911959d6f307c19781a1d19ecd94b5f042d78" integrity sha512-aDGDLU+j9tJcUdPGOaHmVF1u/hhI+CsGkT02V3OKlHDV7IukOI+nTWAGkiZEKCO35rWN1wIr4tS7YFr1f4qSvA== @@ -10077,6 +10171,16 @@ streamx@^2.10.3, streamx@^2.13.2, streamx@^2.15.0, streamx@^2.15.1, streamx@^2.1 optionalDependencies: bare-events "^2.2.0" +streamx@^2.20.1: + version "2.22.0" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.22.0.tgz#cd7b5e57c95aaef0ff9b2aef7905afa62ec6e4a7" + integrity sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw== + dependencies: + fast-fifo "^1.3.2" + text-decoder "^1.1.0" + optionalDependencies: + bare-events "^2.2.0" + string-argv@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" @@ -10142,7 +10246,7 @@ string.prototype.trimstart@^1.0.8: define-properties "^1.2.1" es-object-atoms "^1.0.0" -string2compact@^2.0.0, string2compact@^2.0.1: +string2compact@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/string2compact/-/string2compact-2.0.1.tgz#a640e70413e8875c3fc34de6184f57abe8b34868" integrity sha512-Bm/T8lHMTRXw+u83LE+OW7fXmC/wM+Mbccfdo533ajSBNxddDHlRrvxE49NdciGHgXkUQM5WYskJ7uTkbBUI0A== @@ -10296,6 +10400,27 @@ swagger-cli@^4.0.2: dependencies: "@apidevtools/swagger-cli" "4.0.4" +tar-fs@^2.0.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.2.tgz#425f154f3404cb16cb8ff6e671d45ab2ed9596c5" + integrity sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.1.4" + +tar-stream@^2.1.4: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar-stream@^3.0.0: version "3.1.7" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.7.tgz#24b3fb5eabada19fe7338ed6d26e5f7c482e792b" @@ -10476,21 +10601,21 @@ toposort-class@^1.0.1: resolved "https://registry.yarnpkg.com/toposort-class/-/toposort-class-1.0.1.tgz#7ffd1f78c8be28c3ba45cd4e1a3f5ee193bd9988" integrity sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg== -torrent-discovery@^10.0.16: - version "10.0.16" - resolved "https://registry.yarnpkg.com/torrent-discovery/-/torrent-discovery-10.0.16.tgz#e9f5948201ecf1ffeb04923760b4543168aa35df" - integrity sha512-HUvCgL3JAyk9VKUfFBOD7Fx/MWVNmjiCjaEOEc6P7ijm2BpPWpdjlXydP+/12f/NB3T4ItuyMjcGdPseGGjNTw== +torrent-discovery@^11.0.15: + version "11.0.15" + resolved "https://registry.yarnpkg.com/torrent-discovery/-/torrent-discovery-11.0.15.tgz#e3f0cfa5859163344fa51b8ec5853488deb07568" + integrity sha512-O5kCZ/PDcK0PMD5lH4VdwUrL4Wfe1Kt3pjcrMp3yieNQq/ZcnLuae6jnjSvpzoa7DxpYc5OqhkiIOYGyvj1tbA== dependencies: - bittorrent-dht "^11.0.5" + bittorrent-dht "^11.0.9" bittorrent-lsd "^2.0.0" - bittorrent-tracker "^10.0.12" - debug "^4.3.4" + bittorrent-tracker "^11.2.1" + debug "^4.4.0" run-parallel "^1.2.0" -torrent-piece@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/torrent-piece/-/torrent-piece-3.0.0.tgz#395e37c39e62dec75ed601f79c90e4a62e639be7" - integrity sha512-j0tRX7qq22nIuVFF57Tg/wAvFq79F1eM9pcMxY+b0qCCe7yXJnIrqF+Q5YEJ94tNisDnJzcqDHNrPmD9X/yAIg== +torrent-piece@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/torrent-piece/-/torrent-piece-3.0.1.tgz#3f5899aac991256618812d066d7917f809a1ba21" + integrity sha512-EvCqfOkNm3PXqgaGPVVmp0JlGC8fDpH+8Yt5uUiF4oCrAqy3htyUFxK1DJpneWfg1fFdeTKsstxLxQUrHpmocA== dependencies: uint8-util "^2.1.9" @@ -10554,6 +10679,13 @@ tsx@^4.7.1: optionalDependencies: fsevents "~2.3.3" +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== + dependencies: + safe-buffer "^5.0.1" + tv4@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/tv4/-/tv4-1.3.0.tgz#d020c846fadd50c855abb25ebaecc68fc10f7963" @@ -10661,7 +10793,7 @@ uid-safe@2.1.5: dependencies: random-bytes "~1.0.0" -uint8-util@^2.1.3, uint8-util@^2.1.9, uint8-util@^2.2.2, uint8-util@^2.2.4, uint8-util@^2.2.5: +uint8-util@^2.1.3, uint8-util@^2.1.9, uint8-util@^2.2.2, uint8-util@^2.2.5: version "2.2.5" resolved "https://registry.yarnpkg.com/uint8-util/-/uint8-util-2.2.5.tgz#f1a8ff800e4e10a3ac1c82ee3667c99245123896" integrity sha512-/QxVQD7CttWpVUKVPz9znO+3Dd4BdTSnFQ7pv/4drVhC9m4BaL2LFHTkJn6EsYoxT79VDq/2Gg8L0H22PrzyMw== @@ -10764,13 +10896,6 @@ ut_pex@^4.0.4: compact2string "^1.4.1" string2compact "^2.0.1" -utf-8-validate@^5.0.5: - version "5.0.10" - resolved "https://registry.yarnpkg.com/utf-8-validate/-/utf-8-validate-5.0.10.tgz#d7d10ea39318171ca982718b6b96a8d2442571a2" - integrity sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ== - dependencies: - node-gyp-build "^4.3.0" - utf-8-validate@^6.0.4: version "6.0.5" resolved "https://registry.yarnpkg.com/utf-8-validate/-/utf-8-validate-6.0.5.tgz#8087d39902be2cc15bdb21a426697ff256d65aab" @@ -10887,47 +11012,55 @@ webidl-conversions@^3.0.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== -webtorrent@2.1.27: - version "2.1.27" - resolved "https://registry.yarnpkg.com/webtorrent/-/webtorrent-2.1.27.tgz#a6748edd1ea43da7e5f9f3645fc327e45dcf2d55" - integrity sha512-LkAcAOReF82MH9hB64m8xTxTPFHc6cqVGE0Kg9Icpp697wc/rCQbiTtuoOGSzs4u6sMoGRc21iqoktrUq70Zyg== +webrtc-polyfill@^1.1.10: + version "1.1.10" + resolved "https://registry.yarnpkg.com/webrtc-polyfill/-/webrtc-polyfill-1.1.10.tgz#1a140c42afd9bcd041a63174810795b35be5e26d" + integrity sha512-sOn0bj3/noUdzQX7rvk0jFbBurqWDGGo2ipl+WfgoOe/x3cxbGLk/ZUY+WHCISSlLaIeBumi1X3wxQZnUESExQ== + dependencies: + node-datachannel "^v0.12.0" + node-domexception "^1.0.0" + +webtorrent@2.5.17: + version "2.5.17" + resolved "https://registry.yarnpkg.com/webtorrent/-/webtorrent-2.5.17.tgz#df5dcda706996fc07ce09d5c1160ea7ff04bd601" + integrity sha512-0YKibmeq4rq9CqGF5PopZVjyrXS7IW6M/kQ4pRo5DiFoco7OQm9Hyh+0SGGJSfkDBgCurXPnId84rvaoi3u3hw== dependencies: "@silentbot1/nat-api" "^0.4.7" - "@thaunknown/simple-peer" "^9.12.1" + "@thaunknown/simple-peer" "^10.0.11" "@webtorrent/http-node" "^1.3.0" addr-to-ip-port "^2.0.0" - bitfield "^4.1.0" - bittorrent-dht "^11.0.5" - bittorrent-protocol "^4.1.11" + bitfield "^4.2.0" + bittorrent-dht "^11.0.9" + bittorrent-protocol "^4.1.16" cache-chunk-store "^3.2.2" chunk-store-iterator "^1.0.3" cpus "^1.0.3" - create-torrent "^6.0.15" + create-torrent "^6.1.0" cross-fetch-ponyfill "^1.0.3" - debug "^4.3.4" + debug "^4.4.0" escape-html "^1.0.3" fs-chunk-store "^4.1.0" - hybrid-chunk-store "^1.2.2" + fsa-chunk-store "^1.3.0" immediate-chunk-store "^2.2.0" join-async-iterator "^1.1.1" load-ip-set "^3.0.1" - lt_donthave "^2.0.0" + lt_donthave "^2.0.4" memory-chunk-store "^1.3.5" mime "^3.0.0" once "^1.4.0" - parse-torrent "^11.0.14" - pump "^3.0.0" + parse-torrent "^11.0.18" + pump "^3.0.2" queue-microtask "^1.2.3" random-iterate "^1.0.1" range-parser "^1.2.1" run-parallel "^1.2.0" run-parallel-limit "^1.1.0" speed-limiter "^1.0.2" - streamx "^2.15.1" + streamx "2.21.1" throughput "^1.0.1" - torrent-discovery "^10.0.16" - torrent-piece "^3.0.0" - uint8-util "^2.2.4" + torrent-discovery "^11.0.15" + torrent-piece "^3.0.1" + uint8-util "^2.2.5" unordered-array-remove "^1.0.2" ut_metadata "^4.0.3" ut_pex "^4.0.4"