kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Schedule stages statuses bg migrations in batches

Browse source
This commit is contained in:
Grzegorz Bizon 2017-07-10 15:42:19 +02:00
parent 6120dc261a
commit 7082530d55
3 changed files with 16 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
db/post_migrate/20170630111158_migrate_stages_statuses.rb
View file

@ -6,21 +6,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration
disable_ddl_transaction! disable_ddl_transaction!
BATCH_SIZE = 10000 BATCH_SIZE = 10000
RANGE_SIZE = 1000
MIGRATION = 'MigrateStageStatus'.freeze MIGRATION = 'MigrateStageStatus'.freeze
class Stage < ActiveRecord::Base class Stage < ActiveRecord::Base
self.table_name = 'ci_stages' self.table_name = 'ci_stages'
include ::EachBatch
end end
def up def up
index = 1 Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index|
relation.each_batch(of: RANGE_SIZE) do |batch|
Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| range = relation.pluck('MIN(id)', 'MAX(id)').first
jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] }
schedule = index * 5.minutes schedule = index * 5.minutes
index += 1
BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range)
end
end end
end end

5
lib/gitlab/background_migration/migrate_stage_status.rb
View file

@ -58,7 +58,7 @@ module Gitlab
end end
end end
def perform(id) def perform(start_id, stop_id)
status_sql = Build status_sql = Build
.where('ci_builds.commit_id = ci_stages.pipeline_id') .where('ci_builds.commit_id = ci_stages.pipeline_id')
.where('ci_builds.stage = ci_stages.name') .where('ci_builds.stage = ci_stages.name')
@ -66,7 +66,8 @@ module Gitlab
sql = <<-SQL sql = <<-SQL
UPDATE ci_stages SET status = (#{status_sql}) UPDATE ci_stages SET status = (#{status_sql})
WHERE ci_stages.id = #{id.to_i} WHERE ci_stages.status IS NULL
AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
SQL SQL
ActiveRecord::Base.connection.execute(sql) ActiveRecord::Base.connection.execute(sql)

8
spec/migrations/migrate_stages_statuses_spec.rb
View file

@ -12,6 +12,7 @@ describe MigrateStagesStatuses, :migration do
before do before do
stub_const("#{described_class.name}::BATCH_SIZE", 2) stub_const("#{described_class.name}::BATCH_SIZE", 2)
stub_const("#{described_class.name}::RANGE_SIZE", 1)
projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1') projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2') projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2')
@ -49,9 +50,10 @@ describe MigrateStagesStatuses, :migration do
Timecop.freeze do Timecop.freeze do
migrate! migrate!
expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1) puts BackgroundMigrationWorker.jobs.inspect
expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2) expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 1, 1)
expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3) expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, 2, 2)
expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, 3, 3)
expect(BackgroundMigrationWorker.jobs.size).to eq 3 expect(BackgroundMigrationWorker.jobs.size).to eq 3
end end
end end