Prepare rake task for storage rollback
We are keeping compatibility with existing scheduled jobs.
This commit is contained in:
parent
07a196be3d
commit
c2c34eba62
|
@ -3,8 +3,14 @@
|
|||
class StorageMigratorWorker
|
||||
include ApplicationWorker
|
||||
|
||||
def perform(start, finish)
|
||||
# @param [Integer] start initial ID of the batch
|
||||
# @param [Integer] finish last ID of the batch
|
||||
# @param [String] operation the operation to be performed: ['migrate', 'rollback']
|
||||
def perform(start, finish, operation = :migrate)
|
||||
# when scheduling a job symbols are converted to string, we need to convert back
|
||||
operation = operation.to_sym if operation
|
||||
|
||||
migrator = Gitlab::HashedStorage::Migrator.new
|
||||
migrator.bulk_migrate(start, finish)
|
||||
migrator.bulk_migrate(start: start, finish: finish, operation: operation)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -11,10 +11,11 @@ module Gitlab
|
|||
|
||||
# Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously
|
||||
#
|
||||
# @param [Object] start first project id for the range
|
||||
# @param [Object] finish last project id for the range
|
||||
def bulk_schedule(start, finish)
|
||||
StorageMigratorWorker.perform_async(start, finish)
|
||||
# @param [Integer] start first project id for the range
|
||||
# @param [Integer] finish last project id for the range
|
||||
# @param [Symbol] operation [:migrate, :rollback]
|
||||
def bulk_schedule(start:, finish:, operation: :migrate)
|
||||
StorageMigratorWorker.perform_async(start, finish, operation)
|
||||
end
|
||||
|
||||
# Start migration of projects from specified range
|
||||
|
@ -22,21 +23,27 @@ module Gitlab
|
|||
# Flagging a project to be migrated is a synchronous action,
|
||||
# but the migration runs through async jobs
|
||||
#
|
||||
# @param [Object] start first project id for the range
|
||||
# @param [Object] finish last project id for the range
|
||||
# @param [Integer] start first project id for the range
|
||||
# @param [Integer] finish last project id for the range
|
||||
# @param [Symbol] operation [:migrate, :rollback]
|
||||
# rubocop: disable CodeReuse/ActiveRecord
|
||||
def bulk_migrate(start, finish)
|
||||
def bulk_migrate(start:, finish:, operation: :migrate)
|
||||
projects = build_relation(start, finish)
|
||||
|
||||
projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
|
||||
migrate(project)
|
||||
case operation
|
||||
when :migrate
|
||||
migrate(project)
|
||||
when :rollback
|
||||
rollback(project)
|
||||
end
|
||||
end
|
||||
end
|
||||
# rubocop: enable CodeReuse/ActiveRecord
|
||||
|
||||
# Flag a project to be migrated
|
||||
#
|
||||
# @param [Object] project that will be migrated
|
||||
# @param [Project] project that will be migrated
|
||||
def migrate(project)
|
||||
Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
|
||||
|
||||
|
@ -45,6 +52,10 @@ module Gitlab
|
|||
Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
|
||||
end
|
||||
|
||||
def rollback(project)
|
||||
# TODO: implement rollback strategy
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# rubocop: disable CodeReuse/ActiveRecord
|
||||
|
|
|
@ -37,7 +37,7 @@ namespace :gitlab do
|
|||
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}"
|
||||
|
||||
helper.project_id_batches do |start, finish|
|
||||
storage_migrator.bulk_schedule(start, finish)
|
||||
storage_migrator.bulk_schedule(start: start, finish: finish, operation: :migrate)
|
||||
|
||||
print '.'
|
||||
end
|
||||
|
|
|
@ -4,7 +4,7 @@ describe Gitlab::HashedStorage::Migrator do
|
|||
describe '#bulk_schedule' do
|
||||
it 'schedules job to StorageMigratorWorker' do
|
||||
Sidekiq::Testing.fake! do
|
||||
expect { subject.bulk_schedule(1, 5) }.to change(StorageMigratorWorker.jobs, :size).by(1)
|
||||
expect { subject.bulk_schedule(start: 1, finish: 5) }.to change(StorageMigratorWorker.jobs, :size).by(1)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -15,13 +15,13 @@ describe Gitlab::HashedStorage::Migrator do
|
|||
|
||||
it 'enqueue jobs to ProjectMigrateHashedStorageWorker' do
|
||||
Sidekiq::Testing.fake! do
|
||||
expect { subject.bulk_migrate(ids.min, ids.max) }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(2)
|
||||
expect { subject.bulk_migrate(start: ids.min, finish: ids.max) }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(2)
|
||||
end
|
||||
end
|
||||
|
||||
it 'rescues and log exceptions' do
|
||||
allow_any_instance_of(Project).to receive(:migrate_to_hashed_storage!).and_raise(StandardError)
|
||||
expect { subject.bulk_migrate(ids.min, ids.max) }.not_to raise_error
|
||||
expect { subject.bulk_migrate(start: ids.min, finish: ids.max) }.not_to raise_error
|
||||
end
|
||||
|
||||
it 'delegates each project in specified range to #migrate' do
|
||||
|
@ -29,12 +29,12 @@ describe Gitlab::HashedStorage::Migrator do
|
|||
expect(subject).to receive(:migrate).with(project)
|
||||
end
|
||||
|
||||
subject.bulk_migrate(ids.min, ids.max)
|
||||
subject.bulk_migrate(start: ids.min, finish: ids.max)
|
||||
end
|
||||
|
||||
it 'has migrated projects set as writable' do
|
||||
perform_enqueued_jobs do
|
||||
subject.bulk_migrate(ids.min, ids.max)
|
||||
subject.bulk_migrate(start: ids.min, finish: ids.max)
|
||||
end
|
||||
|
||||
projects.each do |project|
|
||||
|
|
|
@ -74,7 +74,7 @@ describe 'rake gitlab:storage:*' do
|
|||
|
||||
it 'enqueues one StorageMigratorWorker per project' do
|
||||
projects.each do |project|
|
||||
expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id)
|
||||
expect(StorageMigratorWorker).to receive(:perform_async).with(project.id, project.id, :migrate)
|
||||
end
|
||||
|
||||
run_rake_task(task)
|
||||
|
@ -89,7 +89,7 @@ describe 'rake gitlab:storage:*' do
|
|||
it 'enqueues one StorageMigratorWorker per 2 projects' do
|
||||
projects.map(&:id).sort.each_slice(2) do |first, last|
|
||||
last ||= first
|
||||
expect(StorageMigratorWorker).to receive(:perform_async).with(first, last)
|
||||
expect(StorageMigratorWorker).to receive(:perform_async).with(first, last, :migrate)
|
||||
end
|
||||
|
||||
run_rake_task(task)
|
||||
|
|
|
@ -7,7 +7,7 @@ describe StorageMigratorWorker do
|
|||
|
||||
describe '#perform' do
|
||||
it 'delegates to MigratorService' do
|
||||
expect_any_instance_of(Gitlab::HashedStorage::Migrator).to receive(:bulk_migrate).with(5, 10)
|
||||
expect_any_instance_of(Gitlab::HashedStorage::Migrator).to receive(:bulk_migrate).with(start: 5, finish: 10, operation: :migrate)
|
||||
|
||||
worker.perform(5, 10)
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue