From 00d6a41e46e4c4948b0d5b4cf21433150a57c067 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 27 Dec 2016 18:33:38 +0100 Subject: [PATCH] Add script to migrate from mongodb to postgresql Usage: NODE_ENV=production ./scripts/mongo-to-postgre.js --mongo-database peertube-prod --- scripts/danger/clean/cleaner.js | 10 +- scripts/mongo-to-postgre.js | 244 ++++++++++++++++++++++++++++++ server/helpers/peertube-crypto.js | 1 - 3 files changed, 250 insertions(+), 5 deletions(-) create mode 100755 scripts/mongo-to-postgre.js diff --git a/scripts/danger/clean/cleaner.js b/scripts/danger/clean/cleaner.js index d1e218145..1a1e3dee7 100644 --- a/scripts/danger/clean/cleaner.js +++ b/scripts/danger/clean/cleaner.js @@ -1,3 +1,4 @@ +const eachSeries = require('async/eachSeries') const rimraf = require('rimraf') const constants = require('../../../server/initializers/constants') @@ -10,14 +11,15 @@ db.init(true, function () { console.info('Tables of %s deleted.', db.sequelize.config.database) const STORAGE = constants.CONFIG.STORAGE - Object.keys(STORAGE).forEach(function (storage) { + eachSeries(Object.keys(STORAGE), function (storage, callbackEach) { const storageDir = STORAGE[storage] rimraf(storageDir, function (err) { - if (err) throw err - - console.info('Deleting %s.', storageDir) + console.info('%s deleted.', storageDir) + return callbackEach(err) }) + }, function () { + process.exit(0) }) }) }) diff --git a/scripts/mongo-to-postgre.js b/scripts/mongo-to-postgre.js new file mode 100755 index 000000000..4a581b46a --- /dev/null +++ b/scripts/mongo-to-postgre.js @@ -0,0 +1,244 @@ +#!/usr/bin/env node + +'use strict' + +// TODO: document this script + +const program = require('commander') +const eachSeries = require('async/eachSeries') +const series = require('async/series') +const waterfall = require('async/waterfall') +const fs = require('fs') +const path = require('path') +const MongoClient = require('mongodb').MongoClient + +const constants = require('../server/initializers/constants') + +program + .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost') + .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017') + .option('-md, --mongo-database [dbname]', 'MongoDB database') + .parse(process.argv) + +if (!program.mongoDatabase) { + console.error('The mongodb database is mandatory.') + process.exit(-1) +} + +const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase +const dbSequelize = require('../server/initializers/database') + +console.log('Connecting to ' + mongoUrl) +MongoClient.connect(mongoUrl, function (err, dbMongo) { + if (err) throw err + + console.log('Connected to ' + mongoUrl) + + const videoMongo = dbMongo.collection('videos') + const userMongo = dbMongo.collection('users') + const podMongo = dbMongo.collection('pods') + + podMongo.count(function (err, podsLength) { + if (err) throw err + + if (podsLength > 0) { + console.error('You need to quit friends first.') + process.exit(-1) + } + + console.log('Connecting to ' + dbSequelize.sequelize.config.database) + dbSequelize.init(true, function (err) { + if (err) throw err + + console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database) + + series([ + function (next) { + dbSequelize.sequelize.sync({ force: true }).asCallback(next) + }, + + function (next) { + migrateVideos(videoMongo, dbSequelize, next) + }, + + function (next) { + migrateUsers(userMongo, dbSequelize, next) + } + ], function (err) { + if (err) console.error(err) + + process.exit(0) + }) + }) + }) +}) + +// --------------------------------------------------------------------------- + +function migrateUsers (userMongo, dbSequelize, callback) { + userMongo.find().toArray(function (err, mongoUsers) { + if (err) return callback(err) + + eachSeries(mongoUsers, function (mongoUser, callbackEach) { + console.log('Migrating user %s', mongoUser.username) + + const userData = { + username: mongoUser.username, + password: mongoUser.password, + role: mongoUser.role + } + const options = { + hooks: false + } + + dbSequelize.User.create(userData, options).asCallback(callbackEach) + }, callback) + }) +} + +function migrateVideos (videoMongo, dbSequelize, finalCallback) { + videoMongo.find().toArray(function (err, mongoVideos) { + if (err) return finalCallback(err) + + eachSeries(mongoVideos, function (mongoVideo, callbackEach) { + console.log('Migrating video %s.', mongoVideo.name) + + waterfall([ + + function startTransaction (callback) { + dbSequelize.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreatePod (t, callback) { + if (mongoVideo.remoteId === null) return callback(null, t, null) + + const query = { + where: { + host: mongoVideo.podHost + }, + defaults: { + host: mongoVideo.podHost + }, + transaction: t + } + + dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, t, result[0]) + }) + }, + + function findOrCreateAuthor (t, pod, callback) { + const podId = pod ? pod.id : null + const username = mongoVideo.author + + const query = { + where: { + podId, + name: username + }, + defaults: { + podId, + name: username + }, + transaction: t + } + + dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, t, result[0]) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = mongoVideo.tags + const tagInstances = [] + + eachSeries(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: mongoVideo.name, + remoteId: mongoVideo.remoteId, + extname: mongoVideo.extname, + infoHash: mongoVideo.magnet.infoHash, + description: mongoVideo.description, + authorId: author.id, + duration: mongoVideo.duration, + createdAt: mongoVideo.createdDate + } + + const video = dbSequelize.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function moveVideoFile (t, tagInstances, video, callback) { + const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR + const src = path.join(basePath, mongoVideo._id.toString()) + video.extname + const dst = path.join(basePath, video.id) + video.extname + fs.rename(src, dst, function (err) { + if (err) return callback(err) + + return callback(null, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + // Abort transaction? + if (t) t.rollback() + + return callbackEach(err) + } + + // Commit transaction + t.commit() + + return callbackEach() + }) + }, finalCallback) + }) +} diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 302ddca58..610cb16cd 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -1,7 +1,6 @@ 'use strict' const bcrypt = require('bcrypt') -const crypto = require('crypto') const fs = require('fs') const openssl = require('openssl-wrapper') const ursa = require('ursa')