Add migration (for db, folders...) mechanism

server.js

@@ -32,6 +32,7 @@ if (miss.length !== 0) {
 // ----------- PeerTube modules -----------
 const customValidators = require('./server/helpers/custom-validators')
 const installer = require('./server/initializers/installer')
+const migrator = require('./server/initializers/migrator')
 const mongoose = require('mongoose')
 const routes = require('./server/controllers')
 const utils = require('./server/helpers/utils')
@@ -127,31 +128,36 @@ app.use(function (err, req, res, next) {
 installer.installApplication(function (err) {
   if (err) throw err
 
-  // Create/activate the webtorrent module
-  webtorrent.create(function () {
-    function cleanForExit () {
-      utils.cleanForExit(webtorrent.app)
-    }
+  // Run the migration scripts if needed
+  migrator.migrate(function (err) {
+    if (err) throw err
 
-    function exitGracefullyOnSignal () {
-      process.exit(-1)
-    }
+    // Create/activate the webtorrent module
+    webtorrent.create(function () {
+      function cleanForExit () {
+        utils.cleanForExit(webtorrent.app)
+      }
 
-    process.on('exit', cleanForExit)
-    process.on('SIGINT', exitGracefullyOnSignal)
-    process.on('SIGTERM', exitGracefullyOnSignal)
+      function exitGracefullyOnSignal () {
+        process.exit(-1)
+      }
 
-    // ----------- Make the server listening -----------
-    server.listen(port, function () {
-      // Activate the pool requests
-      Request.activate()
+      process.on('exit', cleanForExit)
+      process.on('SIGINT', exitGracefullyOnSignal)
+      process.on('SIGTERM', exitGracefullyOnSignal)
 
-      Video.seedAllExisting(function (err) {
-        if (err) throw err
+      // ----------- Make the server listening -----------
+      server.listen(port, function () {
+        // Activate the pool requests
+        Request.activate()
 
-        logger.info('Seeded all the videos')
-        logger.info('Server listening on port %d', port)
-        app.emit('ready')
+        Video.seedAllExisting(function (err) {
+          if (err) throw err
+
+          logger.info('Seeded all the videos')
+          logger.info('Server listening on port %d', port)
+          app.emit('ready')
+        })
       })
     })
   })

server/initializers/constants.js

@@ -54,6 +54,18 @@ const FRIEND_SCORE = {
   MAX: 1000
 }
 
+const MONGO_MIGRATION_SCRIPTS = [
+  {
+    script: '0005-create-application',
+    version: 5
+  },
+  {
+    script: '0010-users-password',
+    version: 10
+  }
+]
+const LAST_MONGO_SCHEMA_VERSION = 10
+
 // Time to wait between requests to the friends (10 min)
 let REQUESTS_INTERVAL = 600000
 
@@ -121,11 +133,13 @@ module.exports = {
   CONFIG: CONFIG,
   CONSTRAINTS_FIELDS: CONSTRAINTS_FIELDS,
   FRIEND_SCORE: FRIEND_SCORE,
-  REQUESTS_INTERVAL: REQUESTS_INTERVAL,
+  LAST_MONGO_SCHEMA_VERSION: LAST_MONGO_SCHEMA_VERSION,
+  MONGO_MIGRATION_SCRIPTS: MONGO_MIGRATION_SCRIPTS,
   OAUTH_LIFETIME: OAUTH_LIFETIME,
   PAGINATION_COUNT_DEFAULT: PAGINATION_COUNT_DEFAULT,
   PODS_SCORE: PODS_SCORE,
   REQUESTS_IN_PARALLEL: REQUESTS_IN_PARALLEL,
+  REQUESTS_INTERVAL: REQUESTS_INTERVAL,
   REQUESTS_LIMIT: REQUESTS_LIMIT,
   RETRY_REQUESTS: RETRY_REQUESTS,
   SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS,

server/initializers/database.js

@@ -6,6 +6,7 @@ const constants = require('../initializers/constants')
 const logger = require('../helpers/logger')
 
 // Bootstrap models
+require('../models/application')
 require('../models/user')
 require('../models/oauth-client')
 require('../models/oauth-token')

server/initializers/migrations/0005-create-application.js

@@ -0,0 +1,17 @@
+/*
+  Create the application collection in MongoDB.
+  Used to store the actual MongoDB scheme version.
+*/
+
+const mongoose = require('mongoose')
+
+const Application = mongoose.model('Application')
+
+exports.up = function (callback) {
+  const application = new Application()
+  application.save(callback)
+}
+
+exports.down = function (callback) {
+  throw new Error('Not implemented.')
+}

server/initializers/migrations/0010-users-password.js

@@ -0,0 +1,23 @@
+/*
+  Convert plain user password to encrypted user password.
+*/
+
+const mongoose = require('mongoose')
+
+const User = mongoose.model('User')
+
+exports.up = function (callback) {
+  User.list(function (err, users) {
+    if (err) return callback(err)
+
+    users.forEach(function (user) {
+      user.save()
+    })
+
+    return callback(null)
+  })
+}
+
+exports.down = function (callback) {
+  throw new Error('Not implemented.')
+}

server/initializers/migrator.js

@@ -0,0 +1,56 @@
+'use strict'
+
+const eachSeries = require('async/eachSeries')
+const mongoose = require('mongoose')
+const path = require('path')
+
+const constants = require('./constants')
+const logger = require('../helpers/logger')
+
+const Application = mongoose.model('Application')
+
+const migrator = {
+  migrate: migrate
+}
+
+function migrate (callback) {
+  Application.loadMongoSchemaVersion(function (err, actualVersion) {
+    if (err) return callback(err)
+
+    // If there are a new mongo schemas
+    if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) {
+      logger.info('Begin migrations.')
+
+      eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) {
+        const versionScript = entity.version
+
+        // Do not execute old migration scripts
+        if (versionScript <= actualVersion) return callbackEach(null)
+
+        // Load the migration module and run it
+        const migrationScriptName = entity.script
+        logger.info('Executing %s migration script.', migrationScriptName)
+
+        const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
+        migrationScript.up(function (err) {
+          if (err) return callbackEach(err)
+
+          // Update the new mongo version schema
+          Application.updateMongoSchemaVersion(versionScript, callbackEach)
+        })
+      }, function (err) {
+        if (err) return callback(err)
+
+        logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION)
+        return callback(null)
+      })
+    } else {
+      return callback(null)
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+module.exports = migrator
+

server/models/application.js

@@ -0,0 +1,31 @@
+const mongoose = require('mongoose')
+
+// ---------------------------------------------------------------------------
+
+const ApplicationSchema = mongoose.Schema({
+  mongoSchemaVersion: {
+    type: Number,
+    default: 0
+  }
+})
+
+ApplicationSchema.statics = {
+  loadMongoSchemaVersion: loadMongoSchemaVersion,
+  updateMongoSchemaVersion: updateMongoSchemaVersion
+}
+
+mongoose.model('Application', ApplicationSchema)
+
+// ---------------------------------------------------------------------------
+
+function loadMongoSchemaVersion (callback) {
+  return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) {
+    const version = data ? data.mongoSchemaVersion : 0
+
+    return callback(err, version)
+  })
+}
+
+function updateMongoSchemaVersion (newVersion, callback) {
+  return this.update({}, { mongoSchemaVersion: newVersion }, callback)
+}

server/models/user.js

@@ -28,6 +28,7 @@ UserSchema.methods = {
 UserSchema.statics = {
   countTotal: countTotal,
   getByUsername: getByUsername,
+  list: list,
   listForApi: listForApi,
   loadById: loadById,
   loadByUsername: loadByUsername
@@ -71,6 +72,10 @@ function getByUsername (username) {
   return this.findOne({ username: username })
 }
 
+function list (callback) {
+  return this.find(callback)
+}
+
 function listForApi (start, count, sort, callback) {
   const query = {}
   return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback)