From 26fcf2efebc681104d8e181da42b9ec112a8d28e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 28 Jan 2020 11:07:23 +0100 Subject: [PATCH] Add redundancy CLI --- server/tests/cli/peertube.ts | 75 ++++++++++- server/tools/cli.ts | 19 ++- server/tools/package.json | 5 +- server/tools/peertube-auth.ts | 7 +- server/tools/peertube-plugins.ts | 25 +--- server/tools/peertube-redundancy.ts | 194 ++++++++++++++++++++++++++++ server/tools/peertube.ts | 1 + server/tools/yarn.lock | 23 ++-- 8 files changed, 308 insertions(+), 41 deletions(-) create mode 100644 server/tools/peertube-redundancy.ts diff --git a/server/tests/cli/peertube.ts b/server/tests/cli/peertube.ts index b8c0b1f79..09cbcdb65 100644 --- a/server/tests/cli/peertube.ts +++ b/server/tests/cli/peertube.ts @@ -6,15 +6,15 @@ import { addVideoChannel, buildAbsoluteFixturePath, cleanupTests, - createUser, + createUser, doubleFollow, execCLI, flushAndRunServer, - getEnvCli, + getEnvCli, getLocalIdByUUID, getVideo, getVideosList, getVideosListWithToken, removeVideo, ServerInfo, - setAccessTokensToServers, + setAccessTokensToServers, uploadVideo, uploadVideoAndGetId, userLogin, waitJobs } from '../../../shared/extra-utils' @@ -210,6 +210,75 @@ describe('Test CLI wrapper', function () { }) }) + describe('Manage video redundancies', function () { + let anotherServer: ServerInfo + let video1Server2: number + let servers: ServerInfo[] + + before(async function () { + this.timeout(120000) + + anotherServer = await flushAndRunServer(2) + await setAccessTokensToServers([ anotherServer ]) + + await doubleFollow(server, anotherServer) + + servers = [ server, anotherServer ] + await waitJobs(servers) + + const uuid = (await uploadVideoAndGetId({ server: anotherServer, videoName: 'super video' })).uuid + await waitJobs(servers) + + video1Server2 = await getLocalIdByUUID(server.url, uuid) + }) + + it('Should add a redundancy', async function () { + this.timeout(60000) + + const env = getEnvCli(server) + + const params = `add --video ${video1Server2}` + + await execCLI(`${env} ${cmd} redundancy ${params}`) + + await waitJobs(servers) + }) + + it('Should list redundancies', async function () { + this.timeout(60000) + + { + const env = getEnvCli(server) + + const params = `list-my-redundancies` + const stdout = await execCLI(`${env} ${cmd} redundancy ${params}`) + + expect(stdout).to.contain('super video') + expect(stdout).to.contain(`localhost:${server.port}`) + } + }) + + it('Should remove a redundancy', async function () { + this.timeout(60000) + + const env = getEnvCli(server) + + const params = `remove --video ${video1Server2}` + + await execCLI(`${env} ${cmd} redundancy ${params}`) + + await waitJobs(servers) + + { + const env = getEnvCli(server) + const params = `list-my-redundancies` + const stdout = await execCLI(`${env} ${cmd} redundancy ${params}`) + + expect(stdout).to.not.contain('super video') + } + }) + }) + after(async function () { this.timeout(10000) diff --git a/server/tools/cli.ts b/server/tools/cli.ts index 58e2445ac..ba80872fb 100644 --- a/server/tools/cli.ts +++ b/server/tools/cli.ts @@ -6,6 +6,8 @@ import { getVideoChannel } from '../../shared/extra-utils/videos/video-channels' import { Command } from 'commander' import { VideoChannel, VideoPrivacy } from '../../shared/models/videos' import { createLogger, format, transports } from 'winston' +import { getAccessToken, getMyUserInformation } from '@shared/extra-utils' +import { User, UserRole } from '@shared/models' let configName = 'PeerTube/CLI' if (isTestInstance()) configName += `-${getAppNumber()}` @@ -14,6 +16,19 @@ const config = require('application-config')(configName) const version = require('../../../package.json').version +async function getAdminTokenOrDie (url: string, username: string, password: string) { + const accessToken = await getAccessToken(url, username, password) + const resMe = await getMyUserInformation(url, accessToken) + const me: User = resMe.body + + if (me.role !== UserRole.ADMINISTRATOR) { + console.error('You must be an administrator.') + process.exit(-1) + } + + return accessToken +} + interface Settings { remotes: any[], default: number @@ -222,5 +237,7 @@ export { getServerCredentials, buildCommonVideoOptions, - buildVideoAttributesFromCommander + buildVideoAttributesFromCommander, + + getAdminTokenOrDie } diff --git a/server/tools/package.json b/server/tools/package.json index 40959d76e..06ad31cab 100644 --- a/server/tools/package.json +++ b/server/tools/package.json @@ -4,11 +4,12 @@ "private": true, "dependencies": { "application-config": "^1.0.1", - "cli-table": "^0.3.1", + "cli-table3": "^0.5.1", "netrc-parser": "^3.1.6", "webtorrent-hybrid": "^4.0.1" }, "summon": { "silent": true - } + }, + "devDependencies": {} } diff --git a/server/tools/peertube-auth.ts b/server/tools/peertube-auth.ts index 6597a5c36..acac75034 100644 --- a/server/tools/peertube-auth.ts +++ b/server/tools/peertube-auth.ts @@ -6,8 +6,7 @@ import * as prompt from 'prompt' import { getNetrc, getSettings, writeSettings } from './cli' import { isUserUsernameValid } from '../helpers/custom-validators/users' import { getAccessToken, login } from '../../shared/extra-utils' - -const Table = require('cli-table') +import * as CliTable3 from 'cli-table3' async function delInstance (url: string) { const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) @@ -108,10 +107,10 @@ program .action(async () => { const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) - const table = new Table({ + const table = new CliTable3({ head: ['instance', 'login'], colWidths: [30, 30] - }) + }) as CliTable3.HorizontalTable settings.remotes.forEach(element => { if (!netrc.machines[element]) return diff --git a/server/tools/peertube-plugins.ts b/server/tools/peertube-plugins.ts index e40606107..b341c14c1 100644 --- a/server/tools/peertube-plugins.ts +++ b/server/tools/peertube-plugins.ts @@ -3,15 +3,11 @@ registerTSPaths() import * as program from 'commander' import { PluginType } from '../../shared/models/plugins/plugin.type' -import { getAccessToken } from '../../shared/extra-utils/users/login' -import { getMyUserInformation } from '../../shared/extra-utils/users/users' import { installPlugin, listPlugins, uninstallPlugin, updatePlugin } from '../../shared/extra-utils/server/plugins' -import { getServerCredentials } from './cli' -import { User, UserRole } from '../../shared/models/users' +import { getAdminTokenOrDie, getServerCredentials } from './cli' import { PeerTubePlugin } from '../../shared/models/plugins/peertube-plugin.model' import { isAbsolute } from 'path' - -const Table = require('cli-table') +import * as CliTable3 from 'cli-table3' program .name('plugins') @@ -82,10 +78,10 @@ async function pluginsListCLI () { }) const plugins: PeerTubePlugin[] = res.body.data - const table = new Table({ + const table = new CliTable3({ head: ['name', 'version', 'homepage'], colWidths: [ 50, 10, 50 ] - }) + }) as CliTable3.HorizontalTable for (const plugin of plugins) { const npmName = plugin.type === PluginType.PLUGIN @@ -192,16 +188,3 @@ async function uninstallPluginCLI (options: any) { console.log('Plugin uninstalled.') process.exit(0) } - -async function getAdminTokenOrDie (url: string, username: string, password: string) { - const accessToken = await getAccessToken(url, username, password) - const resMe = await getMyUserInformation(url, accessToken) - const me: User = resMe.body - - if (me.role !== UserRole.ADMINISTRATOR) { - console.error('Cannot list plugins if you are not administrator.') - process.exit(-1) - } - - return accessToken -} diff --git a/server/tools/peertube-redundancy.ts b/server/tools/peertube-redundancy.ts new file mode 100644 index 000000000..a71f48104 --- /dev/null +++ b/server/tools/peertube-redundancy.ts @@ -0,0 +1,194 @@ +import { registerTSPaths } from '../helpers/register-ts-paths' +registerTSPaths() + +import * as program from 'commander' +import { getAdminTokenOrDie, getServerCredentials } from './cli' +import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' +import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy } from '@shared/extra-utils/server/redundancy' +import validator from 'validator' +import bytes = require('bytes') +import * as CliTable3 from 'cli-table3' +import { parse } from 'url' +import { uniq } from 'lodash' + +program + .name('plugins') + .usage('[command] [options]') + +program + .command('list-remote-redundancies') + .description('List remote redundancies on your videos') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .action(() => listRedundanciesCLI('my-videos')) + +program + .command('list-my-redundancies') + .description('List your redundancies of remote videos') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .action(() => listRedundanciesCLI('remote-videos')) + +program + .command('add') + .description('Duplicate a video in your redundancy system') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-v, --video ', 'Video id to duplicate') + .action((options) => addRedundancyCLI(options)) + +program + .command('remove') + .description('Remove a video from your redundancies') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-v, --video ', 'Video id to remove from redundancies') + .action((options) => removeRedundancyCLI(options)) + +if (!process.argv.slice(2).length) { + program.outputHelp() +} + +program.parse(process.argv) + +// ---------------------------------------------------------------------------- + +async function listRedundanciesCLI (target: VideoRedundanciesTarget) { + const { url, username, password } = await getServerCredentials(program) + const accessToken = await getAdminTokenOrDie(url, username, password) + + const redundancies = await listVideoRedundanciesData(url, accessToken, target) + + const table = new CliTable3({ + head: [ 'video id', 'video name', 'video url', 'files', 'playlists', 'by instances', 'total size' ] + }) as CliTable3.HorizontalTable + + for (const redundancy of redundancies) { + const webtorrentFiles = redundancy.redundancies.files + const streamingPlaylists = redundancy.redundancies.streamingPlaylists + + let totalSize = '' + if (target === 'remote-videos') { + const tmp = webtorrentFiles.concat(streamingPlaylists) + .reduce((a, b) => a + b.size, 0) + + totalSize = bytes(tmp) + } + + const instances = uniq( + webtorrentFiles.concat(streamingPlaylists) + .map(r => r.fileUrl) + .map(u => parse(u).host) + ) + + table.push([ + redundancy.id.toString(), + redundancy.name, + redundancy.url, + webtorrentFiles.length, + streamingPlaylists.length, + instances.join('\n'), + totalSize + ]) + } + + console.log(table.toString()) + process.exit(0) +} + +async function addRedundancyCLI (options: { videoId: number }) { + const { url, username, password } = await getServerCredentials(program) + const accessToken = await getAdminTokenOrDie(url, username, password) + + if (!options[ 'video' ] || validator.isInt('' + options[ 'video' ]) === false) { + console.error('You need to specify the video id to duplicate and it should be a number.\n') + program.outputHelp() + process.exit(-1) + } + + try { + await addVideoRedundancy({ + url, + accessToken, + videoId: options[ 'video' ] + }) + + console.log('Video will be duplicated by your instance!') + + process.exit(0) + } catch (err) { + if (err.message.includes(409)) { + console.error('This video is already duplicated by your instance.') + } else if (err.message.includes(404)) { + console.error('This video id does not exist.') + } else { + console.error(err) + } + + process.exit(-1) + } +} + +async function removeRedundancyCLI (options: { videoId: number }) { + const { url, username, password } = await getServerCredentials(program) + const accessToken = await getAdminTokenOrDie(url, username, password) + + if (!options[ 'video' ] || validator.isInt('' + options[ 'video' ]) === false) { + console.error('You need to specify the video id to remove from your redundancies.\n') + program.outputHelp() + process.exit(-1) + } + + const videoId = parseInt(options[ 'video' ] + '', 10) + + let redundancies = await listVideoRedundanciesData(url, accessToken, 'my-videos') + let videoRedundancy = redundancies.find(r => videoId === r.id) + + if (!videoRedundancy) { + redundancies = await listVideoRedundanciesData(url, accessToken, 'remote-videos') + videoRedundancy = redundancies.find(r => videoId === r.id) + } + + if (!videoRedundancy) { + console.error('Video redundancy not found.') + process.exit(-1) + } + + try { + const ids = videoRedundancy.redundancies.files + .concat(videoRedundancy.redundancies.streamingPlaylists) + .map(r => r.id) + + for (const id of ids) { + await removeVideoRedundancy({ + url, + accessToken, + redundancyId: id + }) + } + + console.log('Video redundancy removed!') + + process.exit(0) + } catch (err) { + console.error(err) + process.exit(-1) + } +} + +async function listVideoRedundanciesData (url: string, accessToken: string, target: VideoRedundanciesTarget) { + const res = await listVideoRedundancies({ + url, + accessToken, + start: 0, + count: 100, + sort: 'name', + target + }) + + return res.body.data as VideoRedundancy[] +} diff --git a/server/tools/peertube.ts b/server/tools/peertube.ts index fc85c4210..9883bbf05 100644 --- a/server/tools/peertube.ts +++ b/server/tools/peertube.ts @@ -22,6 +22,7 @@ program .command('watch', 'watch a video in the terminal ✩°｡⋆').alias('w') .command('repl', 'initiate a REPL to access internals') .command('plugins [action]', 'manage instance plugins/themes').alias('p') + .command('redundancy [action]', 'manage instance redundancies').alias('r') /* Not Yet Implemented */ program diff --git a/server/tools/yarn.lock b/server/tools/yarn.lock index 28756cbc2..ccd716a51 100644 --- a/server/tools/yarn.lock +++ b/server/tools/yarn.lock @@ -347,12 +347,15 @@ chunk-store-stream@^4.0.0: block-stream2 "^2.0.0" readable-stream "^3.4.0" -cli-table@^0.3.1: - version "0.3.1" - resolved "https://registry.yarnpkg.com/cli-table/-/cli-table-0.3.1.tgz#f53b05266a8b1a0b934b3d0821e6e2dc5914ae23" - integrity sha1-9TsFJmqLGguTSz0IIebi3FkUriM= +cli-table3@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.5.1.tgz#0252372d94dfc40dbd8df06005f48f31f656f202" + integrity sha512-7Qg2Jrep1S/+Q3EceiZtQcDPWxhAvBw+ERf1162v4sikJrvojMHFqXt8QIVha8UlH9rgU0BeWPytZ9/TzYqlUw== dependencies: - colors "1.0.3" + object-assign "^4.1.0" + string-width "^2.1.1" + optionalDependencies: + colors "^1.1.2" clivas@^0.2.0: version "0.2.0" @@ -364,10 +367,10 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= -colors@1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b" - integrity sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs= +colors@^1.1.2: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" + integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== common-tags@^1.8.0: version "1.8.0" @@ -1609,7 +1612,7 @@ string-width@^1.0.1: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" -"string-width@^1.0.2 || 2": +"string-width@^1.0.2 || 2", string-width@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==