kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Merge branch '63667-hashed-storage-migration-count-correctly' into 'master'

Browse source 
Display the amount for Hashed Storage migration/rollback correctly

Closes #63667

See merge request gitlab-org/gitlab-ce!29996
This commit is contained in:
Douglas Barbosa Alexandre 2019-07-11 16:57:04 +00:00
commit 06b8fe5607
5 changed files with 116 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Display the correct amount of projects being migrated/rolled-back to Hashed Storage when specifying ranges
merge_request: 29996
author:
type: fixed

6
lib/gitlab/hashed_storage/rake_helper.rb
View file

@ -19,8 +19,12 @@ module Gitlab
ENV['ID_TO']
end
def self.using_ranges?
!range_from.nil? && !range_to.nil?
end
def self.range_single_item?
!range_from.nil? && range_from == range_to
using_ranges? && range_from == range_to
end
# rubocop: disable CodeReuse/ActiveRecord

80
lib/tasks/gitlab/storage.rake
View file

@ -3,50 +3,44 @@ namespace :gitlab do
desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
task migrate_to_hashed: :environment do
if Gitlab::Database.read_only?
warn 'This task requires database write access. Exiting.'
next
abort 'This task requires database write access. Exiting.'
end
storage_migrator = Gitlab::HashedStorage::Migrator.new
helper = Gitlab::HashedStorage::RakeHelper
if storage_migrator.rollback_pending?
warn "There is already a rollback operation in progress, " \
abort "There is already a rollback operation in progress, " \
"running a migration at the same time may have unexpected consequences."
next
end
if helper.range_single_item?
project = Project.with_unmigrated_storage.find_by(id: helper.range_from)
unless project
warn "There are no projects requiring storage migration with ID=#{helper.range_from}"
next
abort "There are no projects requiring storage migration with ID=#{helper.range_from}"
end
puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..."
storage_migrator.migrate(project)
else
legacy_projects_count = if helper.using_ranges?
Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count
else
Project.with_unmigrated_storage.count
end
next
end
if legacy_projects_count == 0
abort 'There are no projects requiring storage migration. Nothing to do!'
end
legacy_projects_count = Project.with_unmigrated_storage.count
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
if legacy_projects_count == 0
warn 'There are no projects requiring storage migration. Nothing to do!'
helper.project_id_batches_migration do |start, finish|
storage_migrator.bulk_schedule_migration(start: start, finish: finish)
next
end
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
helper.project_id_batches_migration do |start, finish|
storage_migrator.bulk_schedule_migration(start: start, finish: finish)
print '.'
print '.'
end
end
puts ' Done!'
@ -55,50 +49,44 @@ namespace :gitlab do
desc 'GitLab | Storage | Rollback existing projects to Legacy Storage'
task rollback_to_legacy: :environment do
if Gitlab::Database.read_only?
warn 'This task requires database write access. Exiting.'
next
abort 'This task requires database write access. Exiting.'
end
storage_migrator = Gitlab::HashedStorage::Migrator.new
helper = Gitlab::HashedStorage::RakeHelper
if storage_migrator.migration_pending?
warn "There is already a migration operation in progress, " \
abort "There is already a migration operation in progress, " \
"running a rollback at the same time may have unexpected consequences."
next
end
if helper.range_single_item?
project = Project.with_storage_feature(:repository).find_by(id: helper.range_from)
unless project
warn "There are no projects that can be rolledback with ID=#{helper.range_from}"
next
abort "There are no projects that can be rolledback with ID=#{helper.range_from}"
end
puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..."
storage_migrator.rollback(project)
else
hashed_projects_count = if helper.using_ranges?
Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count
else
Project.with_storage_feature(:repository).count
end
next
end
if hashed_projects_count == 0
abort 'There are no projects that can have storage rolledback. Nothing to do!'
end
hashed_projects_count = Project.with_storage_feature(:repository).count
print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}"
if hashed_projects_count == 0
warn 'There are no projects that can have storage rolledback. Nothing to do!'
helper.project_id_batches_rollback do |start, finish|
storage_migrator.bulk_schedule_rollback(start: start, finish: finish)
next
end
print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}"
helper.project_id_batches_rollback do |start, finish|
storage_migrator.bulk_schedule_rollback(start: start, finish: finish)
print '.'
print '.'
end
end
puts ' Done!'

46
spec/support/matchers/abort_matcher.rb Normal file
View file

@ -0,0 +1,46 @@
RSpec::Matchers.define :abort_execution do
match do |code_block|
@captured_stderr = StringIO.new
original_stderr = $stderr
$stderr = @captured_stderr
code_block.call
false
rescue SystemExit => e
captured = @captured_stderr.string.chomp
@actual_exit_code = e.status
break false unless e.status == 1
if @message
if @message.is_a? String
@message == captured
elsif @message.is_a? Regexp
@message.match?(captured)
else
raise ArgumentError, 'with_message must be either a String or a Regular Expression'
end
end
ensure
$stderr = original_stderr
end
chain :with_message do |message|
@message = message
end
failure_message do |block|
unless @actual_exit_code
break "expected #{block} to abort with '#{@message}' but didnt call abort."
end
if @actual_exit_code != 1
break "expected #{block} to abort with: '#{@message}' but exited with success instead."
end
"expected #{block} to abort with: '#{@message}' \n but received: '#{@captured_stderr.string.chomp}' instead."
end
supports_block_expectations
end

34
spec/tasks/gitlab/storage_rake_spec.rb
View file

@ -50,7 +50,7 @@ describe 'rake gitlab:storage:*', :sidekiq do
expect(Project).not_to receive(:with_unmigrated_storage)
expect { run_rake_task(task) }.to output(/This task requires database write access. Exiting./).to_stderr
expect { run_rake_task(task) }.to abort_execution.with_message(/This task requires database write access. Exiting./)
end
end
end
@ -96,7 +96,7 @@ describe 'rake gitlab:storage:*', :sidekiq do
expect(Project).not_to receive(:with_unmigrated_storage)
expect { run_rake_task(task) }.to output(/There is already a rollback operation in progress/).to_stderr
expect { run_rake_task(task) }.to abort_execution.with_message(/There is already a rollback operation in progress/)
end
end
end
@ -105,14 +105,23 @@ describe 'rake gitlab:storage:*', :sidekiq do
it 'does nothing' do
expect(::HashedStorage::MigratorWorker).not_to receive(:perform_async)
run_rake_task(task)
expect { run_rake_task(task) }.to abort_execution.with_message('There are no projects requiring storage migration. Nothing to do!')
end
end
context 'with 3 legacy projects' do
let(:projects) { create_list(:project, 3, :legacy_storage) }
it_behaves_like "handles custom BATCH env var", ::HashedStorage::MigratorWorker
it 'enqueues migrations and count projects correctly' do
projects.map(&:id).sort.tap do |ids|
stub_env('ID_FROM', ids[0])
stub_env('ID_TO', ids[1])
end
expect { run_rake_task(task) }.to output(/Enqueuing migration of 2 projects in batches/).to_stdout
end
it_behaves_like 'handles custom BATCH env var', ::HashedStorage::MigratorWorker
end
context 'with same id in range' do
@ -120,7 +129,7 @@ describe 'rake gitlab:storage:*', :sidekiq do
stub_env('ID_FROM', 99999)
stub_env('ID_TO', 99999)
expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stderr
expect { run_rake_task(task) }.to abort_execution.with_message(/There are no projects requiring storage migration with ID=99999/)
end
it 'displays a message when project exists but its already migrated' do
@ -128,7 +137,7 @@ describe 'rake gitlab:storage:*', :sidekiq do
stub_env('ID_FROM', project.id)
stub_env('ID_TO', project.id)
expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stderr
expect { run_rake_task(task) }.to abort_execution.with_message(/There are no projects requiring storage migration with ID=#{project.id}/)
end
it 'enqueues migration when project can be found' do
@ -153,7 +162,7 @@ describe 'rake gitlab:storage:*', :sidekiq do
expect(Project).not_to receive(:with_unmigrated_storage)
expect { run_rake_task(task) }.to output(/There is already a migration operation in progress/).to_stderr
expect { run_rake_task(task) }.to abort_execution.with_message(/There is already a migration operation in progress/)
end
end
end
@ -162,13 +171,22 @@ describe 'rake gitlab:storage:*', :sidekiq do
it 'does nothing' do
expect(::HashedStorage::RollbackerWorker).not_to receive(:perform_async)
run_rake_task(task)
expect { run_rake_task(task) }.to abort_execution.with_message('There are no projects that can have storage rolledback. Nothing to do!')
end
end
context 'with 3 hashed projects' do
let(:projects) { create_list(:project, 3) }
it 'enqueues migrations and count projects correctly' do
projects.map(&:id).sort.tap do |ids|
stub_env('ID_FROM', ids[0])
stub_env('ID_TO', ids[1])
end
expect { run_rake_task(task) }.to output(/Enqueuing rollback of 2 projects in batches/).to_stdout
end
it_behaves_like "handles custom BATCH env var", ::HashedStorage::RollbackerWorker
end
end