forks
/
peertube
1
0
Fork 0
Code Releases Activity

Fix redundancy with HLS only files

This commit is contained in:
Chocobozzz 2021-02-01 11:18:50 +01:00
parent 33c7131be5
commit 7448551fe5
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
3 changed files with 131 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
server/models/redundancy/video-redundancy.ts
View File

@ -1,5 +1,5 @@
import { sample } from 'lodash'
import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
import { col, FindOptions, fn, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
import {
AllowNull,
BeforeDestroy,
@ -15,7 +15,7 @@ import {
UpdatedAt
} from 'sequelize-typescript'
import { getServerActor } from '@server/models/application/application'
import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
import {
FileRedundancyInformation,
@ -36,6 +36,7 @@ import { VideoModel } from '../video/video'
import { VideoChannelModel } from '../video/video-channel'
import { VideoFileModel } from '../video/video-file'
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
import { forEachSeries } from 'async'
export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
@ -261,6 +262,8 @@ export class VideoRedundancyModel extends Model {
}
static async findMostViewToDuplicate (randomizedFactor: number) {
const peertubeActor = await getServerActor()
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
@ -268,10 +271,10 @@ export class VideoRedundancyModel extends Model {
order: getVideoSort('-views'),
where: {
privacy: VideoPrivacy.PUBLIC,
isLive: false
isLive: false,
...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude()
]
}
@ -280,6 +283,8 @@ export class VideoRedundancyModel extends Model {
}
static async findTrendingToDuplicate (randomizedFactor: number) {
const peertubeActor = await getServerActor()
// On VideoModel!
const query = {
attributes: [ 'id', 'views' ],
@ -289,10 +294,10 @@ export class VideoRedundancyModel extends Model {
order: getVideoSort('-trending'),
where: {
privacy: VideoPrivacy.PUBLIC,
isLive: false
isLive: false,
...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude(),
VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
@ -303,6 +308,8 @@ export class VideoRedundancyModel extends Model {
}
static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
const peertubeActor = await getServerActor()
// On VideoModel!
const query = {
attributes: [ 'id', 'publishedAt' ],
@ -313,10 +320,10 @@ export class VideoRedundancyModel extends Model {
isLive: false,
views: {
[Op.gte]: minViews
}
},
...this.buildVideoIdsForDuplication(peertubeActor)
},
include: [
await VideoRedundancyModel.buildVideoFileForDuplication(),
VideoRedundancyModel.buildServerRedundancyInclude()
]
}
@ -573,31 +580,34 @@ export class VideoRedundancyModel extends Model {
static async getStats (strategy: VideoRedundancyStrategyWithManual) {
const actor = await getServerActor()
const query: FindOptions = {
raw: true,
attributes: [
[ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
[ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
[ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
],
where: {
strategy,
actorId: actor.id
},
include: [
{
attributes: [],
model: VideoFileModel,
required: true
}
]
}
const sql = `WITH "tmp" AS ` +
`(` +
`SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
`"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
`FROM "videoRedundancy" AS "videoRedundancy" ` +
`LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
`LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
`LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
`ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
`WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
`), ` +
`"videoIds" AS (` +
`SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
`UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
`) ` +
`SELECT ` +
`COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
`(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
`COUNT(*) AS "totalVideoFiles" ` +
`FROM "tmp"`
return VideoRedundancyModel.findOne(query)
.then((r: any) => ({
totalUsed: parseAggregateResult(r.totalUsed),
totalVideos: r.totalVideos,
totalVideoFiles: r.totalVideoFiles
return VideoRedundancyModel.sequelize.query<any>(sql, {
replacements: { strategy, actorId: actor.id },
type: QueryTypes.SELECT
}).then(([ row ]) => ({
totalUsed: parseAggregateResult(row.totalUsed),
totalVideos: row.totalVideos,
totalVideoFiles: row.totalVideoFiles
}))
}
@ -692,26 +702,25 @@ export class VideoRedundancyModel extends Model {
}
// Don't include video files we already duplicated
private static async buildVideoFileForDuplication () {
const actor = await getServerActor()
private static buildVideoIdsForDuplication (peertubeActor: MActor) {
const notIn = literal(
'(' +
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
`SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
`INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
`WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
`UNION ` +
`SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
`INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
`WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
')'
)
return {
attributes: [],
model: VideoFileModel,
required: true,
where: {
id: {
[Op.notIn]: notIn
}
}
}
}
private static buildServerRedundancyInclude () {
return {

90
server/tests/api/redundancy/redundancy.ts
View File

@ -21,6 +21,8 @@ import {
ServerInfo,
setAccessTokensToServers,
unfollow,
updateCustomConfig,
updateCustomSubConfig,
uploadVideo,
viewVideo,
wait,
@ -60,7 +62,7 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
}
async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) {
async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
const strategies: any[] = []
if (strategy !== null) {
@ -75,6 +77,9 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, add
const config = {
transcoding: {
webtorrent: {
enabled: withWebtorrent
},
hls: {
enabled: true
}
@ -253,7 +258,7 @@ async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
return stat
}
async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) {
async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual) {
const stat = await checkStatsGlobal(strategy)
expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
@ -261,7 +266,7 @@ async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManu
expect(stat.totalVideos).to.equal(1)
}
async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) {
async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
const stat = await checkStatsGlobal(strategy)
expect(stat.totalUsed).to.equal(0)
@ -313,7 +318,7 @@ describe('Test videos redundancy', function () {
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
await checkStatsWithoutRedundancy(strategy)
})
it('Should enable redundancy on server 1', function () {
@ -329,7 +334,7 @@ describe('Test videos redundancy', function () {
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
await checkStatsWith1Redundancy(strategy)
})
it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
@ -363,7 +368,7 @@ describe('Test videos redundancy', function () {
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
await checkStatsWithoutRedundancy(strategy)
})
it('Should enable redundancy on server 1', function () {
@ -379,7 +384,7 @@ describe('Test videos redundancy', function () {
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
await checkStatsWith1Redundancy(strategy)
})
it('Should unfollow on server 1 and remove duplicated videos', async function () {
@ -413,7 +418,7 @@ describe('Test videos redundancy', function () {
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
await checkStatsWithoutRedundancy(strategy)
})
it('Should enable redundancy on server 1', function () {
@ -429,7 +434,7 @@ describe('Test videos redundancy', function () {
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed(strategy)
await checkStatsWithoutRedundancy(strategy)
})
it('Should view 2 times the first video to have > min_views config', async function () {
@ -451,7 +456,7 @@ describe('Test videos redundancy', function () {
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
await checkStatsWith1Redundancy(strategy)
})
it('Should remove the video and the redundancy files', async function () {
@ -471,6 +476,65 @@ describe('Test videos redundancy', function () {
})
})
describe('With only HLS files', function () {
const strategy = 'recently-added'
before(async function () {
this.timeout(120000)
await flushAndRunServers(strategy, { min_views: 3 }, false)
})
it('Should have 0 playlist redundancy on the first video', async function () {
await check1WebSeed()
await check0PlaylistRedundancies()
})
it('Should enable redundancy on server 1', function () {
return enableRedundancyOnServer1()
})
it('Should still have 0 redundancy on the first video', async function () {
this.timeout(80000)
await waitJobs(servers)
await wait(15000)
await waitJobs(servers)
await check0PlaylistRedundancies()
await checkStatsWithoutRedundancy(strategy)
})
it('Should have 1 redundancy on the first video', async function () {
this.timeout(160000)
await viewVideo(servers[0].url, video1Server2UUID)
await viewVideo(servers[2].url, video1Server2UUID)
await wait(10000)
await waitJobs(servers)
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 1)
await waitJobs(servers)
await check1PlaylistRedundancies()
await checkStatsWith1Redundancy(strategy)
})
it('Should remove the video and the redundancy files', async function () {
this.timeout(20000)
await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
await waitJobs(servers)
for (const server of servers) {
await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
}
})
})
describe('With manual strategy', function () {
before(function () {
this.timeout(120000)
@ -481,7 +545,7 @@ describe('Test videos redundancy', function () {
it('Should have 1 webseed on the first video', async function () {
await check1WebSeed()
await check0PlaylistRedundancies()
await checkStatsWith1Webseed('manual')
await checkStatsWithoutRedundancy('manual')
})
it('Should create a redundancy on first video', async function () {
@ -501,7 +565,7 @@ describe('Test videos redundancy', function () {
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed('manual')
await checkStatsWith1Redundancy('manual')
})
it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
@ -619,7 +683,7 @@ describe('Test videos redundancy', function () {
await check2Webseeds()
await check1PlaylistRedundancies()
await checkStatsWith2Webseed(strategy)
await checkStatsWith1Redundancy(strategy)
const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' })
video2Server2UUID = res.body.video.uuid

2
shared/extra-utils/videos/videos.ts
View File

@ -338,7 +338,7 @@ async function checkVideoFilesWereRemoved (
const files = await readdir(directoryPath)
for (const file of files) {
expect(file).to.not.contain(videoUUID)
expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID)
}
}
}