Create CI migration task on GitLab side

lib/ci/migrate/builds.rb

@@ -0,0 +1,29 @@
+module Ci
+  module Migrate
+    class Builds
+      attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
+
+      def initialize
+        @app_builds_dir = Settings.gitlab_ci.builds_path
+        @backup_dir = Gitlab.config.backup.path
+        @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
+      end
+
+      def restore
+        backup_existing_builds_dir
+
+        FileUtils.mkdir_p(app_builds_dir, mode: 0700)
+        unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
+          abort 'Restore failed'.red
+        end
+      end
+
+      def backup_existing_builds_dir
+        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+        if File.exists?(app_builds_dir)
+          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+        end
+      end
+    end
+  end
+end

lib/ci/migrate/database.rb

@@ -9,32 +9,32 @@ module Ci
         @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       end
 
-      def restore(ci_dump)
-        puts 'Deleting all CI related data ... '
-        truncate_ci_tables
+      def restore
+        decompress_rd, decompress_wr = IO.pipe
+        decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+        decompress_wr.close
 
-        puts 'Restoring CI data ... '
-        case config["adapter"]
+        restore_pid = case config["adapter"]
                         when /^mysql/ then
-            print "Restoring MySQL database #{config['database']} ... "
+                          $progress.print "Restoring MySQL database #{config['database']} ... "
                           # Workaround warnings from MySQL 5.6 about passwords on cmd line
                           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-            system('mysql', *mysql_args, config['database'], in: ci_dump)
+                          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
                         when "postgresql" then
-            puts "Restoring PostgreSQL database #{config['database']} ... "
+                          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
                           pg_env
-            system('psql', config['database'], '-f', ci_dump)
+                          spawn('psql', config['database'], in: decompress_rd)
                       end
+        decompress_rd.close
+
+        success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+        abort 'Restore failed' unless success
       end
 
       protected
 
-      def truncate_ci_tables
-        c = ActiveRecord::Base.connection
-        c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-          puts "Deleting data from #{table}..."
-          c.execute("DELETE FROM #{table}")
-        end
+      def db_file_name
+        File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
       end
 
       def mysql_args

lib/ci/migrate/manager.rb

@@ -0,0 +1,72 @@
+module Ci
+  module Migrate
+    class Manager
+      VERSION = '8.0.0.pre'
+
+      def cleanup
+        $progress.print "Deleting tmp directories ... "
+
+        backup_contents.each do |dir|
+          next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
+          if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+            $progress.puts "done".green
+          else
+            puts "deleting tmp directory '#{dir}' failed".red
+            abort 'Backup failed'
+          end
+        end
+      end
+
+      def unpack
+        Dir.chdir(Gitlab.config.backup.path)
+
+        # check for existing backups in the backup dir
+        file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+        puts "no backups found" if file_list.count == 0
+
+        if file_list.count > 1 && ENV["BACKUP"].nil?
+          puts "Found more than one backup, please specify which one you want to restore:"
+          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+          exit 1
+        end
+
+        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+        unless File.exists?(tar_file)
+          puts "The specified CI backup doesn't exist!"
+          exit 1
+        end
+
+        $progress.print "Unpacking backup ... "
+
+        unless Kernel.system(*%W(tar -xf #{tar_file}))
+          puts "unpacking backup failed".red
+          exit 1
+        else
+          $progress.puts "done".green
+        end
+
+        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+        # restoring mismatching backups can lead to unexpected problems
+        if settings[:gitlab_version] != VERSION
+          puts "GitLab CI version mismatch:".red
+          puts "  Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
+          exit 1
+        end
+      end
+
+      private
+
+      def backup_contents
+        ["db", "builds", "backup_information.yml"]
+      end
+
+      def settings
+        @settings ||= YAML.load_file("backup_information.yml")
+      end
+    end
+  end
+end
+

lib/tasks/ci/migrate.rake

@@ -1,40 +1,49 @@
 namespace :ci do
   desc 'GitLab | Import and migrate CI database'
   task migrate: :environment do
+    warn_user_is_not_gitlab
+    configure_cron_mode
+
     unless ENV['force'] == 'yes'
-      puts "This will truncate all CI tables and restore it from provided backup."
-      puts "You will lose any previous CI data stored in the database."
+      puts 'This will remove all CI related data and restore it from the provided backup.'
       ask_to_continue
-      puts ""
+      puts ''
     end
 
-    Rake::Task["ci:migrate:db"].invoke
-    Rake::Task["ci:migrate:autoincrements"].invoke
-    Rake::Task["ci:migrate:tags"].invoke
-    Rake::Task["ci:migrate:services"].invoke
+    migrate = Ci::Migrate::Manager.new
+    migrate.unpack
+
+    Rake::Task['ci:migrate:db'].invoke
+    Rake::Task['ci:migrate:builds'].invoke
+    Rake::Task['ci:migrate:tags'].invoke
+    Rake::Task['ci:migrate:services'].invoke
+
+    migrate.cleanup
   end
 
   namespace :migrate do
     desc 'GitLab | Import CI database'
     task db: :environment do
-      if ENV["CI_DUMP"].nil?
-        puts "No CI SQL dump specified:"
-        puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
-        exit 1
+      configure_cron_mode
+      $progress.puts 'Restoring database ... '.blue
+      Ci::Migrate::Database.new.restore
+      $progress.puts 'done'.green
     end
 
-      ci_dump = ENV["CI_DUMP"]
-      unless File.exists?(ci_dump)
-        puts "The specified sql dump doesn't exist!"
-        exit 1
-      end
-
-      ::Ci::Migrate::Database.new.restore(ci_dump)
+    desc 'GitLab | Import CI builds'
+    task builds: :environment do
+      configure_cron_mode
+      $progress.puts 'Restoring builds ... '.blue
+      Ci::Migrate::Builds.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI tags'
     task tags: :environment do
+      configure_cron_mode
+      $progress.puts 'Migrating tags ... '.blue
       ::Ci::Migrate::Tags.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI auto-increments'
@@ -56,8 +65,10 @@ namespace :ci do
 
     desc 'GitLab | Migrate CI services'
     task services: :environment do
+      $progress.puts 'Migrating services ... '.blue
       c = ActiveRecord::Base.connection
       c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
+      $progress.puts 'done'.green
     end
   end
 end