forks/peertube
1
0
Fork 0
Code Releases Activity

Add redundancy stats

Browse source
This commit is contained in:
Chocobozzz 2018-09-14 14:57:59 +02:00
parent cfc16a6db8
commit 4b5384f6e7
No known key found for this signature in database
GPG key ID: 583A612D890159BE
8 changed files with 132 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

20
server/controllers/api/overviews.ts
View file

@ -21,6 +21,16 @@ export { overviewsRouter }
// ---------------------------------------------------------------------------
const buildSamples = memoizee(async function () {
const [ categories, channels, tags ] = await Promise.all([
VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
])
return { categories, channels, tags }
}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
// This endpoint could be quite long, but we cache it
async function getVideosOverview (req: express.Request, res: express.Response) {
const attributes = await buildSamples()
@ -45,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
return res.json(result)
}
const buildSamples = memoizee(async function () {
const [ categories, channels, tags ] = await Promise.all([
VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
])
return { categories, channels, tags }
}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
async function getVideosByTag (tag: string, res: express.Response) {
const videos = await getVideos(res, { tagsOneOf: [ tag ] })

14
server/controllers/api/server/stats.ts
View file

@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user'
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { VideoModel } from '../../../models/video/video'
import { VideoCommentModel } from '../../../models/video/video-comment'
import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
import { cacheRoute } from '../../../middlewares/cache'
const statsRouter = express.Router()
statsRouter.get('/stats',
asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
asyncMiddleware(getStats)
)
@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr
const { totalUsers } = await UserModel.getStats()
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
const videosRedundancyStats = await Promise.all(
CONFIG.REDUNDANCY.VIDEOS.map(r => {
return VideoRedundancyModel.getStats(r.strategy)
.then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
})
)
const data: ServerStats = {
totalLocalVideos,
totalLocalVideoViews,
@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr
totalVideoComments,
totalUsers,
totalInstanceFollowers,
totalInstanceFollowing
totalInstanceFollowing,
videosRedundancy: videosRedundancyStats
}
return res.json(data).end()

3
server/initializers/constants.ts
View file

@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = {
},
ACTIVITY_PUB: {
VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
}
},
STATS: '4 hours'
}
// ---------------------------------------------------------------------------

31
server/models/redundancy/video-redundancy.ts
View file

@ -245,6 +245,37 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
.findAll(query)
}
static async getStats (strategy: VideoRedundancyStrategy) {
const actor = await getServerActor()
const query = {
raw: true,
attributes: [
[ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
[ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ],
[ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ]
],
where: {
strategy,
actorId: actor.id
},
include: [
{
attributes: [],
model: VideoFileModel,
required: true
}
]
}
return VideoRedundancyModel.find(query as any) // FIXME: typings
.then((r: any) => ({
totalUsed: parseInt(r.totalUsed.toString(), 10),
totalVideos: r.totalVideos,
totalVideoFiles: r.totalVideoFiles
}))
}
toActivityPubObject (): CacheFileObject {
return {
id: this.url,

75
server/tests/api/server/redundancy.ts
View file

@ -23,6 +23,8 @@ import { ActorFollow } from '../../../../shared/models/actors'
import { readdir } from 'fs-extra'
import { join } from 'path'
import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
import { getStats } from '../../utils/server/stats'
import { ServerStats } from '../../../../shared/models/server/server-stats.model'
const expect = chai.expect
@ -79,16 +81,32 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
await waitJobs(servers)
}
async function check1WebSeed () {
async function check1WebSeed (strategy: VideoRedundancyStrategy) {
const webseeds = [
'http://localhost:9002/static/webseed/' + video1Server2UUID
]
for (const server of servers) {
const res = await getVideo(server.url, video1Server2UUID)
{
const res = await getVideo(server.url, video1Server2UUID)
const video: VideoDetails = res.body
video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
const video: VideoDetails = res.body
video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
}
{
const res = await getStats(server.url)
const data: ServerStats = res.body
expect(data.videosRedundancy).to.have.lengthOf(1)
const stat = data.videosRedundancy[0]
expect(stat.strategy).to.equal(strategy)
expect(stat.totalSize).to.equal(102400)
expect(stat.totalUsed).to.equal(0)
expect(stat.totalVideoFiles).to.equal(0)
expect(stat.totalVideos).to.equal(0)
}
}
}
@ -107,7 +125,7 @@ async function enableRedundancy () {
expect(server2.following.hostRedundancyAllowed).to.be.true
}
async function check2Webseeds () {
async function check2Webseeds (strategy: VideoRedundancyStrategy) {
await waitJobs(servers)
await wait(15000)
await waitJobs(servers)
@ -118,12 +136,14 @@ async function check2Webseeds () {
]
for (const server of servers) {
const res = await getVideo(server.url, video1Server2UUID)
{
const res = await getVideo(server.url, video1Server2UUID)
const video: VideoDetails = res.body
const video: VideoDetails = res.body
for (const file of video.files) {
checkMagnetWebseeds(file, webseeds)
for (const file of video.files) {
checkMagnetWebseeds(file, webseeds)
}
}
}
@ -133,6 +153,20 @@ async function check2Webseeds () {
for (const resolution of [ 240, 360, 480, 720 ]) {
expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
}
{
const res = await getStats(servers[0].url)
const data: ServerStats = res.body
expect(data.videosRedundancy).to.have.lengthOf(1)
const stat = data.videosRedundancy[0]
expect(stat.strategy).to.equal(strategy)
expect(stat.totalSize).to.equal(102400)
expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
expect(stat.totalVideoFiles).to.equal(4)
expect(stat.totalVideos).to.equal(1)
}
}
async function cleanServers () {
@ -142,15 +176,16 @@ async function cleanServers () {
describe('Test videos redundancy', function () {
describe('With most-views strategy', function () {
const strategy = 'most-views'
before(function () {
this.timeout(120000)
return runServers('most-views')
return runServers(strategy)
})
it('Should have 1 webseed on the first video', function () {
return check1WebSeed()
return check1WebSeed(strategy)
})
it('Should enable redundancy on server 1', function () {
@ -160,7 +195,7 @@ describe('Test videos redundancy', function () {
it('Should have 2 webseed on the first video', function () {
this.timeout(40000)
return check2Webseeds()
return check2Webseeds(strategy)
})
after(function () {
@ -169,15 +204,16 @@ describe('Test videos redundancy', function () {
})
describe('With trending strategy', function () {
const strategy = 'trending'
before(function () {
this.timeout(120000)
return runServers('trending')
return runServers(strategy)
})
it('Should have 1 webseed on the first video', function () {
return check1WebSeed()
return check1WebSeed(strategy)
})
it('Should enable redundancy on server 1', function () {
@ -187,7 +223,7 @@ describe('Test videos redundancy', function () {
it('Should have 2 webseed on the first video', function () {
this.timeout(40000)
return check2Webseeds()
return check2Webseeds(strategy)
})
after(function () {
@ -196,15 +232,16 @@ describe('Test videos redundancy', function () {
})
describe('With recently added strategy', function () {
const strategy = 'recently-added'
before(function () {
this.timeout(120000)
return runServers('recently-added', { minViews: 3 })
return runServers(strategy, { minViews: 3 })
})
it('Should have 1 webseed on the first video', function () {
return check1WebSeed()
return check1WebSeed(strategy)
})
it('Should enable redundancy on server 1', function () {
@ -218,7 +255,7 @@ describe('Test videos redundancy', function () {
await wait(15000)
await waitJobs(servers)
return check1WebSeed()
return check1WebSeed(strategy)
})
it('Should view 2 times the first video', async function () {
@ -234,7 +271,7 @@ describe('Test videos redundancy', function () {
it('Should have 2 webseed on the first video', function () {
this.timeout(40000)
return check2Webseeds()
return check2Webseeds(strategy)
})
after(function () {

5
server/tests/api/server/stats.ts
View file

@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs'
const expect = chai.expect
describe('Test stats', function () {
describe('Test stats (excluding redundancy)', function () {
let servers: ServerInfo[] = []
before(async function () {
@ -65,6 +65,7 @@ describe('Test stats', function () {
expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(2)
expect(data.totalInstanceFollowing).to.equal(1)
expect(data.videosRedundancy).to.have.lengthOf(0)
})
it('Should have the correct stats on instance 2', async function () {
@ -79,6 +80,7 @@ describe('Test stats', function () {
expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(1)
expect(data.totalInstanceFollowing).to.equal(1)
expect(data.videosRedundancy).to.have.lengthOf(0)
})
it('Should have the correct stats on instance 3', async function () {
@ -93,6 +95,7 @@ describe('Test stats', function () {
expect(data.totalVideos).to.equal(1)
expect(data.totalInstanceFollowing).to.equal(1)
expect(data.totalInstanceFollowers).to.equal(0)
expect(data.videosRedundancy).to.have.lengthOf(0)
})
after(async function () {

7
server/tests/utils/server/stats.ts
View file

@ -1,11 +1,16 @@
import { makeGetRequest } from '../'
function getStats (url: string) {
function getStats (url: string, useCache = false) {
const path = '/api/v1/server/stats'
const query = {
t: useCache ? undefined : new Date().getTime()
}
return makeGetRequest({
url,
path,
query,
statusCodeExpected: 200
})
}

10
shared/models/server/server-stats.model.ts
View file

@ -1,3 +1,5 @@
import { VideoRedundancyStrategy } from '../redundancy'
export interface ServerStats {
totalUsers: number
totalLocalVideos: number
@ -9,4 +11,12 @@ export interface ServerStats {
totalInstanceFollowers: number
totalInstanceFollowing: number
videosRedundancy: {
strategy: VideoRedundancyStrategy
totalSize: number
totalUsed: number
totalVideoFiles: number
totalVideos: number
}[]
}