Automatically rebuild native modules on ABI change

server.ts

@@ -138,6 +138,7 @@ import { ServerConfigManager } from '@server/lib/server-config-manager'
 import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { isTestOrDevInstance } from './server/helpers/core-utils'
 import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
+import { ApplicationModel } from '@server/models/application/application'
 
 // ----------- Command line -----------
 
@@ -330,12 +331,17 @@ async function startApplication () {
   server.listen(port, hostname, async () => {
     if (cliOptions.plugins) {
       try {
+        await PluginManager.Instance.rebuildNativePluginsIfNeeded()
+
         await PluginManager.Instance.registerPluginsAndThemes()
       } catch (err) {
         logger.error('Cannot register plugins and themes.', { err })
       }
     }
 
+    ApplicationModel.updateNodeVersions()
+      .catch(err => logger.error('Cannot update node versions.', { err }))
+
     logger.info('HTTP server listening on %s:%d', hostname, port)
     logger.info('Web server: %s', WEBSERVER.URL)
 

server/helpers/utils.ts

@@ -4,7 +4,7 @@ import { join } from 'path'
 import { sha256 } from '@shared/extra-utils'
 import { ResultList } from '@shared/models'
 import { CONFIG } from '../initializers/config'
-import { execPromise, execPromise2, randomBytesPromise } from './core-utils'
+import { randomBytesPromise } from './core-utils'
 import { logger } from './logger'
 
 function deleteFileAndCatch (path: string) {
@@ -44,29 +44,6 @@ function getSecureTorrentName (originalName: string) {
   return sha256(originalName) + '.torrent'
 }
 
-async function getServerCommit () {
-  try {
-    const tag = await execPromise2(
-      '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
-      { stdio: [ 0, 1, 2 ] }
-    )
-
-    if (tag) return tag.replace(/^v/, '')
-  } catch (err) {
-    logger.debug('Cannot get version from git tags.', { err })
-  }
-
-  try {
-    const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
-
-    if (version) return version.toString().trim()
-  } catch (err) {
-    logger.debug('Cannot get version from git HEAD.', { err })
-  }
-
-  return ''
-}
-
 /**
  * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
  * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
@@ -88,7 +65,6 @@ export {
   generateRandomString,
   getFormattedObjects,
   getSecureTorrentName,
-  getServerCommit,
   generateVideoImportTmpPath,
   getUUIDFromFilename
 }

server/helpers/version.ts

@@ -0,0 +1,36 @@
+import { execPromise, execPromise2 } from './core-utils'
+import { logger } from './logger'
+
+async function getServerCommit () {
+  try {
+    const tag = await execPromise2(
+      '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
+      { stdio: [ 0, 1, 2 ] }
+    )
+
+    if (tag) return tag.replace(/^v/, '')
+  } catch (err) {
+    logger.debug('Cannot get version from git tags.', { err })
+  }
+
+  try {
+    const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
+
+    if (version) return version.toString().trim()
+  } catch (err) {
+    logger.debug('Cannot get version from git HEAD.', { err })
+  }
+
+  return ''
+}
+
+function getNodeABIVersion () {
+  const version = process.versions.modules
+
+  return parseInt(version)
+}
+
+export {
+  getServerCommit,
+  getNodeABIVersion
+}

server/initializers/constants.ts

@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 720
+const LAST_MIGRATION_VERSION = 725
 
 // ---------------------------------------------------------------------------
 

server/initializers/installer.ts

@@ -1,6 +1,8 @@
 import { ensureDir, readdir, remove } from 'fs-extra'
 import passwordGenerator from 'password-generator'
 import { join } from 'path'
+import { isTestOrDevInstance } from '@server/helpers/core-utils'
+import { getNodeABIVersion } from '@server/helpers/version'
 import { UserRole } from '@shared/models'
 import { logger } from '../helpers/logger'
 import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'
@@ -10,7 +12,6 @@ import { applicationExist, clientsExist, usersExist } from './checker-after-init
 import { CONFIG } from './config'
 import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
 import { sequelizeTypescript } from './database'
-import { isTestOrDevInstance } from '@server/helpers/core-utils'
 
 async function installApplication () {
   try {
@@ -175,7 +176,9 @@ async function createApplicationIfNotExist () {
   logger.info('Creating application account.')
 
   const application = await ApplicationModel.create({
-    migrationVersion: LAST_MIGRATION_VERSION
+    migrationVersion: LAST_MIGRATION_VERSION,
+    nodeVersion: process.version,
+    nodeABIVersion: getNodeABIVersion()
   })
 
   return createApplicationActor(application.id)

server/initializers/migrations/0725-node-version.ts

@@ -0,0 +1,66 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+  db: any
+}): Promise<void> {
+  const { transaction } = utils
+
+  {
+    const data = {
+      type: Sequelize.STRING,
+      defaultValue: null,
+      allowNull: true
+    }
+    await utils.queryInterface.addColumn('application', 'nodeVersion', data, { transaction })
+  }
+
+  {
+    const data = {
+      type: Sequelize.STRING,
+      defaultValue: null,
+      allowNull: true
+    }
+    await utils.queryInterface.addColumn('application', 'nodeABIVersion', data, { transaction })
+  }
+
+  {
+    const query = `UPDATE "application" SET "nodeVersion" = '${process.version}'`
+    await utils.sequelize.query(query, { transaction })
+  }
+
+  {
+    const nodeABIVersion = parseInt(process.versions.modules)
+    const query = `UPDATE "application" SET "nodeABIVersion" = ${nodeABIVersion}`
+    await utils.sequelize.query(query, { transaction })
+  }
+
+  {
+    const data = {
+      type: Sequelize.STRING,
+      defaultValue: null,
+      allowNull: false
+    }
+    await utils.queryInterface.changeColumn('application', 'nodeVersion', data, { transaction })
+  }
+
+  {
+    const data = {
+      type: Sequelize.STRING,
+      defaultValue: null,
+      allowNull: false
+    }
+    await utils.queryInterface.changeColumn('application', 'nodeABIVersion', data, { transaction })
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

server/lib/plugins/plugin-manager.ts

@@ -3,6 +3,7 @@ import { createReadStream, createWriteStream } from 'fs'
 import { ensureDir, outputFile, readJSON } from 'fs-extra'
 import { basename, join } from 'path'
 import { decachePlugin } from '@server/helpers/decache'
+import { ApplicationModel } from '@server/models/application/application'
 import { MOAuthTokenUser, MUser } from '@server/types/models'
 import { getCompleteLocale } from '@shared/core-utils'
 import {
@@ -23,7 +24,7 @@ import { PluginModel } from '../../models/server/plugin'
 import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins'
 import { ClientHtml } from '../client-html'
 import { RegisterHelpers } from './register-helpers'
-import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn'
+import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn'
 
 export interface RegisteredPlugin {
   npmName: string
@@ -384,6 +385,12 @@ export class PluginManager implements ServerHook {
     logger.info('Plugin %s uninstalled.', npmName)
   }
 
+  async rebuildNativePluginsIfNeeded () {
+    if (!await ApplicationModel.nodeABIChanged()) return
+
+    return rebuildNativePlugins()
+  }
+
   // ###################### Private register ######################
 
   private async registerPluginOrTheme (plugin: PluginModel) {

server/lib/plugins/yarn.ts

@@ -31,11 +31,16 @@ async function removeNpmPlugin (name: string) {
   await execYarn('remove ' + name)
 }
 
+async function rebuildNativePlugins () {
+  await execYarn('install --pure-lockfile')
+}
+
 // ############################################################################
 
 export {
   installNpmPlugin,
   installNpmPluginFromDisk,
+  rebuildNativePlugins,
   removeNpmPlugin
 }
 

server/lib/server-config-manager.ts

@@ -1,4 +1,4 @@
-import { getServerCommit } from '@server/helpers/utils'
+import { getServerCommit } from '@server/helpers/version'
 import { CONFIG, isEmailEnabled } from '@server/initializers/config'
 import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants'
 import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup'

server/models/application/application.ts

@@ -1,5 +1,6 @@
 import memoizee from 'memoizee'
 import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript'
+import { getNodeABIVersion } from '@server/helpers/version'
 import { AttributesOnly } from '@shared/typescript-utils'
 import { AccountModel } from '../account/account'
 
@@ -37,6 +38,14 @@ export class ApplicationModel extends Model<Partial<AttributesOnly<ApplicationMo
   @Column
   latestPeerTubeVersion: string
 
+  @AllowNull(false)
+  @Column
+  nodeVersion: string
+
+  @AllowNull(false)
+  @Column
+  nodeABIVersion: number
+
   @HasOne(() => AccountModel, {
     foreignKey: {
       allowNull: true
@@ -52,4 +61,17 @@ export class ApplicationModel extends Model<Partial<AttributesOnly<ApplicationMo
   static load () {
     return ApplicationModel.findOne()
   }
+
+  static async nodeABIChanged () {
+    const application = await this.load()
+
+    return application.nodeABIVersion !== getNodeABIVersion()
+  }
+
+  static async updateNodeVersions () {
+    const application = await this.load()
+
+    application.nodeABIVersion = getNodeABIVersion()
+    application.nodeVersion = process.version
+  }
 }

server/tests/api/server/plugins.ts

@@ -2,6 +2,8 @@
 
 import 'mocha'
 import * as chai from 'chai'
+import { pathExists, remove } from 'fs-extra'
+import { join } from 'path'
 import { testHelloWorldRegisteredSettings } from '@server/tests/shared'
 import { wait } from '@shared/core-utils'
 import { HttpStatusCode, PluginType } from '@shared/models'
@@ -9,6 +11,7 @@ import {
   cleanupTests,
   createSingleServer,
   killallServers,
+  makeGetRequest,
   PeerTubeServer,
   PluginsCommand,
   setAccessTokensToServers
@@ -349,6 +352,35 @@ describe('Test plugins', function () {
     await check()
   })
 
+  it('Should rebuild native modules on Node ABI change', async function () {
+    await command.install({ path: PluginsCommand.getPluginTestPath('-native') })
+
+    await makeGetRequest({
+      url: server.url,
+      path: '/plugins/test-native/router',
+      expectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+
+    const query = `UPDATE "application" SET "nodeABIVersion" = 1`
+    await server.sql.updateQuery(query)
+
+    const baseNativeModule = server.servers.buildDirectory(join('plugins', 'node_modules', 'a-native-example'))
+    await remove(join(baseNativeModule, 'build'))
+    await remove(join(baseNativeModule, 'prebuilds'))
+
+    await server.kill()
+    await server.run()
+
+    await pathExists(join(baseNativeModule, 'build'))
+    await pathExists(join(baseNativeModule, 'prebuilds'))
+
+    await makeGetRequest({
+      url: server.url,
+      path: '/plugins/test-native/router',
+      expectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  })
+
   after(async function () {
     await cleanupTests([ server ])
   })

server/tests/fixtures/peertube-plugin-test-native/main.js

@@ -0,0 +1,21 @@
+const print = require('a-native-example')
+
+async function register ({ getRouter }) {
+  print('hello world')
+
+  const router = getRouter()
+
+  router.get('/', (req, res) => {
+    print('hello world')
+    res.sendStatus(204)
+  })
+}
+
+async function unregister () {
+  return
+}
+
+module.exports = {
+  register,
+  unregister
+}

server/tests/fixtures/peertube-plugin-test-native/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "peertube-plugin-test-native",
+  "version": "0.0.1",
+  "description": "Plugin test-native",
+  "engine": {
+    "peertube": ">=4.3.0"
+  },
+  "keywords": [
+    "peertube",
+    "plugin"
+  ],
+  "homepage": "https://github.com/Chocobozzz/PeerTube",
+  "author": "Chocobozzz",
+  "bugs": "https://github.com/Chocobozzz/PeerTube/issues",
+  "library": "./main.js",
+  "staticDirs": {},
+  "css": [],
+  "clientScripts": [],
+  "translations": {},
+  "dependencies": {
+    "a-native-example": "^1.0.0"
+  }
+}