Merge branch 'optimize-ci-tags-migration' into 'master'

CI migration rake task for GitLab CE

- Uses backup format for final GitLab CI 8.0
- Highly optimized tags migration SQL queries


See merge request !1365

app/views/admin/application_settings/_form.html.haml

@@ -131,7 +131,7 @@
         .checkbox
           = f.label :ci_enabled do
             = f.check_box :ci_enabled
-            Enable Continuous Integration
+            Disable to prevent CI usage until rake ci:migrate is run (8.0 only)
 
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'

app/views/ci/projects/disabled.html.haml

@@ -1 +1 @@
-Continuous Integration has been disabled. Please ask your administrator to enable it.
+Continuous Integration has been disabled for time of the migration.

lib/ci/migrate/builds.rb

@@ -0,0 +1,29 @@
+module Ci
+  module Migrate
+    class Builds
+      attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
+
+      def initialize
+        @app_builds_dir = Settings.gitlab_ci.builds_path
+        @backup_dir = Gitlab.config.backup.path
+        @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
+      end
+
+      def restore
+        backup_existing_builds_dir
+
+        FileUtils.mkdir_p(app_builds_dir, mode: 0700)
+        unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
+          abort 'Restore failed'.red
+        end
+      end
+
+      def backup_existing_builds_dir
+        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+        if File.exists?(app_builds_dir)
+          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+        end
+      end
+    end
+  end
+end

lib/ci/migrate/database.rb

@@ -9,32 +9,32 @@ module Ci
         @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       end
 
-      def restore(ci_dump)
+      def restore
-        puts 'Deleting all CI related data ... '
+        decompress_rd, decompress_wr = IO.pipe
-        truncate_ci_tables
+        decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+        decompress_wr.close
 
-        puts 'Restoring CI data ... '
+        restore_pid = case config["adapter"]
-        case config["adapter"]
                         when /^mysql/ then
-            print "Restoring MySQL database #{config['database']} ... "
+                          $progress.print "Restoring MySQL database #{config['database']} ... "
                           # Workaround warnings from MySQL 5.6 about passwords on cmd line
                           ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-            system('mysql', *mysql_args, config['database'], in: ci_dump)
+                          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
                         when "postgresql" then
-            puts "Restoring PostgreSQL database #{config['database']} ... "
+                          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
                           pg_env
-            system('psql', config['database'], '-f', ci_dump)
+                          spawn('psql', config['database'], in: decompress_rd)
                       end
+        decompress_rd.close
+
+        success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+        abort 'Restore failed' unless success
       end
 
       protected
 
-      def truncate_ci_tables
+      def db_file_name
-        c = ActiveRecord::Base.connection
+        File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
-        c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-          puts "Deleting data from #{table}..."
-          c.execute("DELETE FROM #{table}")
-        end
       end
 
       def mysql_args

lib/ci/migrate/manager.rb

@@ -0,0 +1,72 @@
+module Ci
+  module Migrate
+    class Manager
+      VERSION = '8.0.0.pre'
+
+      def cleanup
+        $progress.print "Deleting tmp directories ... "
+
+        backup_contents.each do |dir|
+          next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
+          if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+            $progress.puts "done".green
+          else
+            puts "deleting tmp directory '#{dir}' failed".red
+            abort 'Backup failed'
+          end
+        end
+      end
+
+      def unpack
+        Dir.chdir(Gitlab.config.backup.path)
+
+        # check for existing backups in the backup dir
+        file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+        puts "no backups found" if file_list.count == 0
+
+        if file_list.count > 1 && ENV["BACKUP"].nil?
+          puts "Found more than one backup, please specify which one you want to restore:"
+          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+          exit 1
+        end
+
+        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+        unless File.exists?(tar_file)
+          puts "The specified CI backup doesn't exist!"
+          exit 1
+        end
+
+        $progress.print "Unpacking backup ... "
+
+        unless Kernel.system(*%W(tar -xf #{tar_file}))
+          puts "unpacking backup failed".red
+          exit 1
+        else
+          $progress.puts "done".green
+        end
+
+        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+        # restoring mismatching backups can lead to unexpected problems
+        if settings[:gitlab_version] != VERSION
+          puts "GitLab CI version mismatch:".red
+          puts "  Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
+          exit 1
+        end
+      end
+
+      private
+
+      def backup_contents
+        ["db", "builds", "backup_information.yml"]
+      end
+
+      def settings
+        @settings ||= YAML.load_file("backup_information.yml")
+      end
+    end
+  end
+end
+

lib/ci/migrate/tags.rb

@@ -4,45 +4,38 @@ module Ci
   module Migrate
     class Tags
       def restore
-        puts 'Migrating tags for Runners... '
+        puts 'Inserting tags...'
-        list_objects('Runner').each do |id|
+        connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
-          putc '.'
+          begin
-          runner = Ci::Runner.find_by_id(id)
+            connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
-          if runner
+          rescue ActiveRecord::RecordNotUnique
-            tags = list_tags('Runner', id)
-            runner.update_attributes(tag_list: tags)
           end
         end
-        puts ''
 
-        puts 'Migrating tags for Builds... '
+        ActiveRecord::Base.transaction do
-        list_objects('Build').each do |id|
+          puts 'Deleting old taggings...'
-          putc '.'
+          connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
-          build = Ci::Build.find_by_id(id)
+
-          if build
+          puts 'Inserting taggings...'
-            tags = list_tags('Build', id)
+          connection.execute(
-            build.update_attributes(tag_list: tags)
+            'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
+              "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
+              'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
+              'JOIN tags ON tags.name = ci_tags.name '
+          )
+
+          puts 'Resetting counters... '
+          connection.execute(
+            'UPDATE tags SET ' +
+              'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
+          )
         end
       end
-        puts ''
-      end
 
       protected
 
-      def list_objects(type)
+      def connection
-        ids = ActiveRecord::Base.connection.select_all(
+        ActiveRecord::Base.connection
-          "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
-        )
-        ids.map { |id| id['taggable_id'] }
-      end
-
-      def list_tags(type, id)
-        tags = ActiveRecord::Base.connection.select_all(
-          'select ci_tags.name from ci_tags ' +
-            'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
-            "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
-        )
-        tags.map { |tag| tag['name'] }
       end
     end
   end

lib/tasks/ci/migrate.rake

@@ -1,40 +1,56 @@
 namespace :ci do
   desc 'GitLab | Import and migrate CI database'
   task migrate: :environment do
+    warn_user_is_not_gitlab
+    configure_cron_mode
+
     unless ENV['force'] == 'yes'
-      puts "This will truncate all CI tables and restore it from provided backup."
+      puts 'This will remove all CI related data and restore it from the provided backup.'
-      puts "You will lose any previous CI data stored in the database."
       ask_to_continue
-      puts ""
+      puts ''
     end
 
-    Rake::Task["ci:migrate:db"].invoke
+    # disable CI for time of migration
-    Rake::Task["ci:migrate:autoincrements"].invoke
+    enable_ci(false)
-    Rake::Task["ci:migrate:tags"].invoke
+
-    Rake::Task["ci:migrate:services"].invoke
+    # unpack archives
+    migrate = Ci::Migrate::Manager.new
+    migrate.unpack
+
+    Rake::Task['ci:migrate:db'].invoke
+    Rake::Task['ci:migrate:builds'].invoke
+    Rake::Task['ci:migrate:tags'].invoke
+    Rake::Task['ci:migrate:services'].invoke
+
+    # enable CI for time of migration
+    enable_ci(true)
+
+    migrate.cleanup
   end
 
   namespace :migrate do
     desc 'GitLab | Import CI database'
     task db: :environment do
-      if ENV["CI_DUMP"].nil?
+      configure_cron_mode
-        puts "No CI SQL dump specified:"
+      $progress.puts 'Restoring database ... '.blue
-        puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
+      Ci::Migrate::Database.new.restore
-        exit 1
+      $progress.puts 'done'.green
     end
 
-      ci_dump = ENV["CI_DUMP"]
+    desc 'GitLab | Import CI builds'
-      unless File.exists?(ci_dump)
+    task builds: :environment do
-        puts "The specified sql dump doesn't exist!"
+      configure_cron_mode
-        exit 1
+      $progress.puts 'Restoring builds ... '.blue
-      end
+      Ci::Migrate::Builds.new.restore
-
+      $progress.puts 'done'.green
-      ::Ci::Migrate::Database.new.restore(ci_dump)
     end
 
     desc 'GitLab | Migrate CI tags'
     task tags: :environment do
+      configure_cron_mode
+      $progress.puts 'Migrating tags ... '.blue
       ::Ci::Migrate::Tags.new.restore
+      $progress.puts 'done'.green
     end
 
     desc 'GitLab | Migrate CI auto-increments'
@@ -56,8 +72,16 @@ namespace :ci do
 
     desc 'GitLab | Migrate CI services'
     task services: :environment do
+      $progress.puts 'Migrating services ... '.blue
       c = ActiveRecord::Base.connection
       c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
+      $progress.puts 'done'.green
     end
   end
+
+  def enable_ci(enabled)
+    settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
+    settings.ci_enabled = enabled
+    settings.save!
+  end
 end