diff --git a/.travis.yml b/.travis.yml index 7670cb7c0..3a73e4fc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ matrix: - env: TEST_SUITE=api-1 - env: TEST_SUITE=api-2 - env: TEST_SUITE=api-3 + - env: TEST_SUITE=api-4 - env: TEST_SUITE=cli - env: TEST_SUITE=lint - env: TEST_SUITE=jest diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1363041..d2bd98f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## v1.1.0-alpha.1 + +We released this alpha version because some admins/users need some moderation tools we implemented in recent weeks. +This release could contain bugs. Don't expect a stable v1.1.0 until December :) + +### Scripts + + * Use DB information from config/production.yaml in upgrade script ([@ldidry](https://github.com/ldidry)) + * Add REPL script ([@McFlat](https://github.com/mcflat)) + +### Docker + + * Add search and import settings env settings env variables ([@kaiyou](https://github.com/kaiyou)) + * Add docker dev image ([@am97](https://github.com/am97)) + +### Features + + * Automatically resume videos if the user is logged in + * Hide automatically the menu when the window is resized ([@BO41](https://github.com/BO41)) + * Remove confirm modal for JavaScript/CSS injection ([@scanlime](https://github.com/scanlime)) + * Set bitrate limits for transcoding ([@Nutomic](https://github.com/nutomic)) + * Add moderation tools in the account page + * Add bulk actions in users table (Delete/Ban for now) + * Add search filter in admin users table + * Add search filter in admin following + * Add search filter in admin followers + * Add ability to list all local videos + * Add ability for users to mute an account or an instance + * Add ability for administrators to mute an account or an instance + * Rename "News" category to "News & Politics" ([@daker](https://github.com/daker)) + * Add explicit error message when changing video ownership ([@lucas-dclrcq](https://github.com/lucas-dclrcq)) + * Improve description of the HTTP video import feature ([@rigelk](https://github.com/rigelk)) + + ## v1.0.0 ### SECURITY diff --git a/scripts/travis.sh b/scripts/travis.sh index 628039ab7..ae4a9f926 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -31,6 +31,9 @@ elif [ "$1" = "api-2" ]; then elif [ "$1" = "api-3" ]; then npm run build:server mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts +elif [ "$1" = "api-3" ]; then + npm run build:server + mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-4.ts elif [ "$1" = "lint" ]; then npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts" diff --git a/server/tests/api/index-4.ts b/server/tests/api/index-4.ts new file mode 100644 index 000000000..8e69b95a6 --- /dev/null +++ b/server/tests/api/index-4.ts @@ -0,0 +1 @@ +import './redundancy' diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index 2d996dbf9..bc140f860 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts @@ -2,3 +2,4 @@ import './index-1' import './index-2' import './index-3' +import './index-4' diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts new file mode 100644 index 000000000..8e69b95a6 --- /dev/null +++ b/server/tests/api/redundancy/index.ts @@ -0,0 +1 @@ +import './redundancy' diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts new file mode 100644 index 000000000..1960854b6 --- /dev/null +++ b/server/tests/api/redundancy/redundancy.ts @@ -0,0 +1,483 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { VideoDetails } from '../../../../shared/models/videos' +import { + doubleFollow, + flushAndRunMultipleServers, + getFollowingListPaginationAndSort, + getVideo, + immutableAssign, + killallServers, makeGetRequest, + root, + ServerInfo, + setAccessTokensToServers, unfollow, + uploadVideo, + viewVideo, + wait, + waitUntilLog, + checkVideoFilesWereRemoved, removeVideo +} from '../../utils' +import { waitJobs } from '../../utils/server/jobs' +import * as magnetUtil from 'magnet-uri' +import { updateRedundancy } from '../../utils/server/redundancy' +import { ActorFollow } from '../../../../shared/models/actors' +import { readdir } from 'fs-extra' +import { join } from 'path' +import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' +import { getStats } from '../../utils/server/stats' +import { ServerStats } from '../../../../shared/models/server/server-stats.model' + +const expect = chai.expect + +let servers: ServerInfo[] = [] +let video1Server2UUID: string + +function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { + const parsed = magnetUtil.decode(file.magnetUri) + + for (const ws of baseWebseeds) { + const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) + expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined + } + + expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) +} + +async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { + const config = { + redundancy: { + videos: { + check_interval: '5 seconds', + strategies: [ + immutableAssign({ + min_lifetime: '1 hour', + strategy: strategy, + size: '100KB' + }, additionalParams) + ] + } + } + } + servers = await flushAndRunMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + + await viewVideo(servers[ 1 ].url, video1Server2UUID) + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[ 0 ], servers[ 1 ]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[ 0 ], servers[ 2 ]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[ 1 ], servers[ 2 ]) + + await waitJobs(servers) +} + +async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + const webseeds = [ + 'http://localhost:9002/static/webseed/' + videoUUID + ] + + for (const server of servers) { + { + const res = await getVideo(server.url, videoUUID) + + const video: VideoDetails = res.body + for (const f of video.files) { + checkMagnetWebseeds(f, webseeds, server) + } + } + } +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.be.at.least(1).and.below(102401) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) +} + +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) +} + +async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + const webseeds = [ + 'http://localhost:9001/static/webseed/' + videoUUID, + 'http://localhost:9002/static/webseed/' + videoUUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + + const video: VideoDetails = res.body + + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds, server) + + // Only servers 1 and 2 have the video + if (server.serverNumber !== 3) { + await makeGetRequest({ + url: server.url, + statusCodeExpected: 200, + path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) + } + } + } + + for (const directory of [ 'test1', 'test2' ]) { + const files = await readdir(join(root(), directory, 'videos')) + expect(files).to.have.length.at.least(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined + } + } +} + +async function enableRedundancyOnServer1 () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) + + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true +} + +async function disableRedundancyOnServer1 () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) + + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.false +} + +async function cleanServers () { + killallServers(servers) +} + +describe('Test videos redundancy', function () { + + describe('With most-views strategy', function () { + const strategy = 'most-views' + + before(function () { + this.timeout(120000) + + return runServers(strategy) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + it('Should undo redundancy on server 1 and remove duplicated videos', async function () { + this.timeout(40000) + + await disableRedundancyOnServer1() + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed(strategy) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + }) + + after(function () { + return cleanServers() + }) + }) + + describe('With trending strategy', function () { + const strategy = 'trending' + + before(function () { + this.timeout(120000) + + return runServers(strategy) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + it('Should unfollow on server 1 and remove duplicated videos', async function () { + this.timeout(40000) + + await unfollow(servers[0].url, servers[0].accessToken, servers[1]) + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed(strategy) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + }) + + after(function () { + return cleanServers() + }) + }) + + describe('With recently added strategy', function () { + const strategy = 'recently-added' + + before(function () { + this.timeout(120000) + + return runServers(strategy, { min_views: 3 }) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should still have 1 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should view 2 times the first video to have > min_views config', async function () { + this.timeout(40000) + + await viewVideo(servers[ 0 ].url, video1Server2UUID) + await viewVideo(servers[ 2 ].url, video1Server2UUID) + + await wait(10000) + await waitJobs(servers) + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + it('Should remove the video and the redundancy files', async function () { + this.timeout(20000) + + await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber) + } + }) + + after(function () { + return cleanServers() + }) + }) + + describe('Test expiration', function () { + const strategy = 'recently-added' + + async function checkContains (servers: ServerInfo[], str: string) { + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + const video: VideoDetails = res.body + + for (const f of video.files) { + expect(f.magnetUri).to.contain(str) + } + } + } + + async function checkNotContains (servers: ServerInfo[], str: string) { + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + const video: VideoDetails = res.body + + for (const f of video.files) { + expect(f.magnetUri).to.not.contain(str) + } + } + } + + before(async function () { + this.timeout(120000) + + await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + }) + + it('Should still have 2 webseeds after 10 seconds', async function () { + this.timeout(40000) + + await wait(10000) + + try { + await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') + } catch { + // Maybe a server deleted a redundancy in the scheduler + await wait(2000) + + await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') + } + }) + + it('Should stop server 1 and expire video redundancy', async function () { + this.timeout(40000) + + killallServers([ servers[0] ]) + + await wait(10000) + + await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') + }) + + after(function () { + return killallServers([ servers[1], servers[2] ]) + }) + }) + + describe('Test file replacement', function () { + let video2Server2UUID: string + const strategy = 'recently-added' + + before(async function () { + this.timeout(120000) + + await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + }) + + it('Should cache video 2 webseed on the first video', async function () { + this.timeout(50000) + + await waitJobs(servers) + + await wait(7000) + + try { + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) + } catch { + await wait(3000) + + try { + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) + } catch { + await wait(5000) + + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) + } + } + }) + + after(function () { + return cleanServers() + }) + }) +}) diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index c74c68a33..eeb8b7a28 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts @@ -3,7 +3,6 @@ import './email' import './follows' import './handle-down' import './jobs' -import './redundancy' import './reverse-proxy' import './stats' import './tracker'