Merge branch 'backstage/gb/build-stages-catch-up-migration' into 'master'

Fully migrate build stages again

Closes #38756

See merge request gitlab-org/gitlab-ce!15741

changelogs/unreleased/feature-gb-pipeline-variable-expressions.yml

@@ -0,0 +1,5 @@
+---
+title: Add catch-up background migration to migrate pipeline stages
+merge_request: 15741
+author:
+type: performance

db/post_migrate/20180212101828_add_tmp_partial_null_index_to_builds.rb

@@ -0,0 +1,14 @@
+class AddTmpPartialNullIndexToBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL',
+                                          name: 'tmp_id_partial_null_index')
+  end
+
+  def down
+    remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index')
+  end
+end

db/post_migrate/20180212101928_schedule_build_stage_migration.rb

@@ -0,0 +1,29 @@
+class ScheduleBuildStageMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateBuildStage'.freeze
+  BATCH_SIZE = 500
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_builds'
+  end
+
+  def up
+    disable_statement_timeout
+
+    Build.where('stage_id IS NULL').tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # noop
+  end
+end

db/post_migrate/20180212102028_remove_tmp_partial_null_index_from_builds.rb

@@ -0,0 +1,14 @@
+class RemoveTmpPartialNullIndexFromBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name(:ci_builds, 'tmp_id_partial_null_index')
+  end
+
+  def down
+    add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL',
+                                          name: 'tmp_id_partial_null_index')
+  end
+end

lib/gitlab/background_migration/migrate_build_stage.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class MigrateBuildStage
+      module Migratable
+        class Stage < ActiveRecord::Base
+          self.table_name = 'ci_stages'
+        end
+
+        class Build < ActiveRecord::Base
+          self.table_name = 'ci_builds'
+
+          def ensure_stage!(attempts: 2)
+            find_stage || create_stage!
+          rescue ActiveRecord::RecordNotUnique
+            retry if (attempts -= 1) > 0
+            raise
+          end
+
+          def find_stage
+            Stage.find_by(name: self.stage || 'test',
+                          pipeline_id: self.commit_id,
+                          project_id: self.project_id)
+          end
+
+          def create_stage!
+            Stage.create!(name: self.stage || 'test',
+                          pipeline_id: self.commit_id,
+                          project_id: self.project_id)
+          end
+        end
+      end
+
+      def perform(start_id, stop_id)
+        stages = Migratable::Build.where('stage_id IS NULL')
+          .where('id BETWEEN ? AND ?', start_id, stop_id)
+          .map { |build| build.ensure_stage! }
+          .compact.map(&:id)
+
+        MigrateBuildStageIdReference.new.perform(start_id, stop_id)
+        MigrateStageStatus.new.perform(stages.min, stages.max)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateBuildStage, :migration, schema: 20180212101928 do
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  STATUSES = { created: 0, pending: 1, running: 2, success: 3,
+               failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+
+  before do
+    projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+
+    jobs.create!(id: 1, commit_id: 1, project_id: 123,
+                 stage_idx: 2, stage: 'build', status: :success)
+    jobs.create!(id: 2, commit_id: 1, project_id: 123,
+                 stage_idx: 2, stage: 'build', status: :success)
+    jobs.create!(id: 3, commit_id: 1, project_id: 123,
+                 stage_idx: 1, stage: 'test', status: :failed)
+    jobs.create!(id: 4, commit_id: 1, project_id: 123,
+                 stage_idx: 1, stage: 'test', status: :success)
+    jobs.create!(id: 5, commit_id: 1, project_id: 123,
+                 stage_idx: 3, stage: 'deploy', status: :pending)
+    jobs.create!(id: 6, commit_id: 1, project_id: 123,
+                 stage_idx: 3, stage: nil, status: :pending)
+  end
+
+  it 'correctly migrates builds stages' do
+    expect(stages.count).to be_zero
+
+    described_class.new.perform(1, 6)
+
+    expect(stages.count).to eq 3
+    expect(stages.all.pluck(:name)).to match_array %w[test build deploy]
+    expect(jobs.where(stage_id: nil)).to be_one
+    expect(jobs.find_by(stage_id: nil).id).to eq 6
+    expect(stages.all.pluck(:status)).to match_array [STATUSES[:success],
+                                                      STATUSES[:failed],
+                                                      STATUSES[:pending]]
+  end
+
+  it 'recovers from unique constraint violation only twice' do
+    allow(described_class::Migratable::Stage)
+      .to receive(:find_by).and_return(nil)
+
+    expect(described_class::Migratable::Stage)
+      .to receive(:find_by).exactly(3).times
+
+    expect { described_class.new.perform(1, 6) }
+      .to raise_error ActiveRecord::RecordNotUnique
+  end
+end

spec/migrations/schedule_build_stage_migration_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180212101928_schedule_build_stage_migration')
+
+describe ScheduleBuildStageMigration, :migration do
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+
+    projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce')
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    stages.create!(id: 1, project_id: 123, pipeline_id: 1, name: 'test')
+
+    jobs.create!(id: 11, commit_id: 1, project_id: 123, stage_id: nil)
+    jobs.create!(id: 206, commit_id: 1, project_id: 123, stage_id: nil)
+    jobs.create!(id: 3413, commit_id: 1, project_id: 123, stage_id: nil)
+    jobs.create!(id: 4109, commit_id: 1, project_id: 123, stage_id: 1)
+  end
+
+  it 'schedules delayed background migrations in batches in bulk' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 11, 11)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 206, 206)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 3413, 3413)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+end