WIP plugins: install/uninstall

package.json

@@ -32,6 +32,8 @@
     "clean:server:test": "scripty",
     "watch:client": "scripty",
     "watch:server": "scripty",
+    "plugin:install": "node ./dist/scripts/plugin/install.js",
+    "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js",
     "danger:clean:dev": "scripty",
     "danger:clean:prod": "scripty",
     "danger:clean:modules": "scripty",

scripts/plugin/install.ts

@@ -0,0 +1,39 @@
+import { initDatabaseModels } from '../../server/initializers/database'
+import * as program from 'commander'
+import { PluginManager } from '../../server/lib/plugins/plugin-manager'
+import { isAbsolute } from 'path'
+
+program
+  .option('-n, --pluginName [pluginName]', 'Plugin name to install')
+  .option('-v, --pluginVersion [pluginVersion]', 'Plugin version to install')
+  .option('-p, --pluginPath [pluginPath]', 'Path of the plugin you want to install')
+  .parse(process.argv)
+
+if (!program['pluginName'] && !program['pluginPath']) {
+  console.error('You need to specify a plugin name with the desired version, or a plugin path.')
+  process.exit(-1)
+}
+
+if (program['pluginName'] && !program['pluginVersion']) {
+  console.error('You need to specify a the version of the plugin you want to install.')
+  process.exit(-1)
+}
+
+if (program['pluginPath'] && !isAbsolute(program['pluginPath'])) {
+  console.error('Plugin path should be absolute.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const toInstall = program['pluginName'] || program['pluginPath']
+  await PluginManager.Instance.install(toInstall, program['pluginVersion'], !!program['pluginPath'])
+}

server.ts

@@ -1,4 +1,6 @@
 // FIXME: https://github.com/nodejs/node/pull/16853
+import { PluginManager } from './server/lib/plugins/plugin-manager'
+
 require('tls').DEFAULT_ECDH_CURVE = 'auto'
 
 import { isTestInstance } from './server/helpers/core-utils'
@@ -259,6 +261,8 @@ async function startApplication () {
   updateStreamingPlaylistsInfohashesIfNeeded()
     .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
 
+  await PluginManager.Instance.registerPlugins()
+
   // Make server listening
   server.listen(port, hostname, () => {
     logger.info('Server listening on %s:%d', hostname, port)

server/helpers/core-utils.ts

@@ -10,7 +10,7 @@ import { isAbsolute, join } from 'path'
 import * as pem from 'pem'
 import { URL } from 'url'
 import { truncate } from 'lodash'
-import { exec } from 'child_process'
+import { exec, ExecOptions } from 'child_process'
 
 const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
   if (!oldObject || typeof oldObject !== 'object') {
@@ -204,6 +204,16 @@ function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex')
   return createHash('sha1').update(str).digest(encoding)
 }
 
+function execShell (command: string, options?: ExecOptions) {
+  return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
+    exec(command, options, (err, stdout, stderr) => {
+      if (err) return rej({ err, stdout, stderr })
+
+      return res({ stdout, stderr })
+    })
+  })
+}
+
 function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
   return function promisified (): Promise<A> {
     return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -269,6 +279,7 @@ export {
   sanitizeUrl,
   sanitizeHost,
   buildPath,
+  execShell,
   peertubeTruncate,
 
   sha256,

server/helpers/custom-validators/misc.ts

@@ -9,7 +9,7 @@ function exists (value: any) {
 function isSafePath (p: string) {
   return exists(p) &&
     (p + '').split(sep).every(part => {
-      return [ '', '.', '..' ].includes(part) === false
+      return [ '..' ].includes(part) === false
     })
 }
 

server/helpers/custom-validators/plugins.ts

@@ -17,6 +17,13 @@ function isPluginNameValid (value: string) {
     validator.matches(value, /^[a-z\-]+$/)
 }
 
+function isNpmPluginNameValid (value: string) {
+  return exists(value) &&
+    validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
+    validator.matches(value, /^[a-z\-]+$/) &&
+    (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-'))
+}
+
 function isPluginDescriptionValid (value: string) {
   return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION)
 }
@@ -55,7 +62,7 @@ function isCSSPathsValid (css: any[]) {
 }
 
 function isPackageJSONValid (packageJSON: PluginPackageJson, pluginType: PluginType) {
-  return isPluginNameValid(packageJSON.name) &&
+  return isNpmPluginNameValid(packageJSON.name) &&
     isPluginDescriptionValid(packageJSON.description) &&
     isPluginEngineValid(packageJSON.engine) &&
     isUrlValid(packageJSON.homepage) &&
@@ -78,5 +85,6 @@ export {
   isPluginVersionValid,
   isPluginNameValid,
   isPluginDescriptionValid,
-  isLibraryCodeValid
+  isLibraryCodeValid,
+  isNpmPluginNameValid
 }

server/initializers/database.ts

@@ -37,6 +37,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
 import { VideoPlaylistModel } from '../models/video/video-playlist'
 import { VideoPlaylistElementModel } from '../models/video/video-playlist-element'
 import { ThumbnailModel } from '../models/video/thumbnail'
+import { PluginModel } from '../models/server/plugin'
 import { QueryTypes, Transaction } from 'sequelize'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
@@ -107,7 +108,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoStreamingPlaylistModel,
     VideoPlaylistModel,
     VideoPlaylistElementModel,
-    ThumbnailModel
+    ThumbnailModel,
+    PluginModel
   ])
 
   // Check extensions exist in the database

server/lib/plugins/plugin-manager.ts

@@ -1,7 +1,7 @@
 import { PluginModel } from '../../models/server/plugin'
 import { logger } from '../../helpers/logger'
 import { RegisterHookOptions } from '../../../shared/models/plugins/register.model'
-import { join } from 'path'
+import { basename, join } from 'path'
 import { CONFIG } from '../../initializers/config'
 import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins'
 import { PluginPackageJson } from '../../../shared/models/plugins/plugin-package-json.model'
@@ -9,6 +9,7 @@ import { PluginLibrary } from '../../../shared/models/plugins/plugin-library.mod
 import { createReadStream, createWriteStream } from 'fs'
 import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants'
 import { PluginType } from '../../../shared/models/plugins/plugin.type'
+import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn'
 
 export interface RegisteredPlugin {
   name: string
@@ -84,11 +85,63 @@ export class PluginManager {
     await plugin.unregister()
   }
 
+  async install (toInstall: string, version: string, fromDisk = false) {
+    let plugin: PluginModel
+    let name: string
+
+    logger.info('Installing plugin %s.', toInstall)
+
+    try {
+      fromDisk
+        ? await installNpmPluginFromDisk(toInstall)
+        : await installNpmPlugin(toInstall, version)
+
+      name = fromDisk ? basename(toInstall) : toInstall
+      const pluginType = name.startsWith('peertube-theme-') ? PluginType.THEME : PluginType.PLUGIN
+      const pluginName = this.normalizePluginName(name)
+
+      const packageJSON = this.getPackageJSON(pluginName, pluginType)
+      if (!isPackageJSONValid(packageJSON, pluginType)) {
+        throw new Error('PackageJSON is invalid.')
+      }
+
+      [ plugin ] = await PluginModel.upsert({
+        name: pluginName,
+        description: packageJSON.description,
+        type: pluginType,
+        version: packageJSON.version,
+        enabled: true,
+        uninstalled: false,
+        peertubeEngine: packageJSON.engine.peertube
+      }, { returning: true })
+    } catch (err) {
+      logger.error('Cannot install plugin %s, removing it...', toInstall, { err })
+
+      try {
+        await removeNpmPlugin(name)
+      } catch (err) {
+        logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err })
+      }
+
+      throw err
+    }
+
+    logger.info('Successful installation of plugin %s.', toInstall)
+
+    await this.registerPluginOrTheme(plugin)
+  }
+
+  async uninstall (packageName: string) {
+    await PluginModel.uninstall(this.normalizePluginName(packageName))
+
+    await removeNpmPlugin(packageName)
+  }
+
   private async registerPluginOrTheme (plugin: PluginModel) {
     logger.info('Registering plugin or theme %s.', plugin.name)
 
-    const pluginPath = join(CONFIG.STORAGE.PLUGINS_DIR, plugin.name, plugin.version)
+    const packageJSON = this.getPackageJSON(plugin.name, plugin.type)
-    const packageJSON: PluginPackageJson = require(join(pluginPath, 'package.json'))
+    const pluginPath = this.getPluginPath(plugin.name, plugin.type)
 
     if (!isPackageJSONValid(packageJSON, plugin.type)) {
       throw new Error('Package.JSON is invalid.')
@@ -124,6 +177,7 @@ export class PluginManager {
     }
 
     const library: PluginLibrary = require(join(pluginPath, packageJSON.library))
+
     if (!isLibraryCodeValid(library)) {
       throw new Error('Library code is not valid (miss register or unregister function)')
     }
@@ -163,6 +217,22 @@ export class PluginManager {
     })
   }
 
+  private getPackageJSON (pluginName: string, pluginType: PluginType) {
+    const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json')
+
+    return require(pluginPath) as PluginPackageJson
+  }
+
+  private getPluginPath (pluginName: string, pluginType: PluginType) {
+    const prefix = pluginType === PluginType.PLUGIN ? 'peertube-plugin-' : 'peertube-theme-'
+
+    return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', prefix + pluginName)
+  }
+
+  private normalizePluginName (name: string) {
+    return name.replace(/^peertube-((theme)|(plugin))-/, '')
+  }
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }

server/lib/plugins/yarn.ts

@@ -0,0 +1,61 @@
+import { execShell } from '../../helpers/core-utils'
+import { logger } from '../../helpers/logger'
+import { isNpmPluginNameValid, isPluginVersionValid } from '../../helpers/custom-validators/plugins'
+import { CONFIG } from '../../initializers/config'
+import { outputJSON, pathExists } from 'fs-extra'
+import { join } from 'path'
+
+async function installNpmPlugin (name: string, version: string) {
+  // Security check
+  checkNpmPluginNameOrThrow(name)
+  checkPluginVersionOrThrow(version)
+
+  const toInstall = `${name}@${version}`
+  await execYarn('add ' + toInstall)
+}
+
+async function installNpmPluginFromDisk (path: string) {
+  await execYarn('add file:' + path)
+}
+
+async function removeNpmPlugin (name: string) {
+  checkNpmPluginNameOrThrow(name)
+
+  await execYarn('remove ' + name)
+}
+
+// ############################################################################
+
+export {
+  installNpmPlugin,
+  installNpmPluginFromDisk,
+  removeNpmPlugin
+}
+
+// ############################################################################
+
+async function execYarn (command: string) {
+  try {
+    const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR
+    const pluginPackageJSON = join(pluginDirectory, 'package.json')
+
+    // Create empty package.json file if needed
+    if (!await pathExists(pluginPackageJSON)) {
+      await outputJSON(pluginPackageJSON, {})
+    }
+
+    await execShell(`yarn ${command}`, { cwd: pluginDirectory })
+  } catch (result) {
+    logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr })
+
+    throw result.err
+  }
+}
+
+function checkNpmPluginNameOrThrow (name: string) {
+  if (!isNpmPluginNameValid(name)) throw new Error('Invalid NPM plugin name to install')
+}
+
+function checkPluginVersionOrThrow (name: string) {
+  if (!isPluginVersionValid(name)) throw new Error('Invalid NPM plugin version to install')
+}

server/models/server/plugin.ts

@@ -42,7 +42,6 @@ export class PluginModel extends Model<PluginModel> {
   uninstalled: boolean
 
   @AllowNull(false)
-  @Is('PluginPeertubeEngine', value => throwIfNotValid(value, isPluginVersionValid, 'peertubeEngine'))
   @Column
   peertubeEngine: string
 
@@ -76,4 +75,14 @@ export class PluginModel extends Model<PluginModel> {
     return PluginModel.findAll(query)
   }
 
+  static uninstall (pluginName: string) {
+    const query = {
+      where: {
+        name: pluginName
+      }
+    }
+
+    return PluginModel.update({ enabled: false, uninstalled: true }, query)
+  }
+
 }

shared/models/plugins/plugin-package-json.model.ts

@@ -1,5 +1,6 @@
 export type PluginPackageJson = {
   name: string
+  version: string
   description: string
   engine: { peertube: string },