Extracted auxiliary methods from storage.rake into specific RakeHelper
This commit is contained in:
parent
6e354cb642
commit
78d25079f5
|
@ -0,0 +1,81 @@
|
|||
module Gitlab
|
||||
module HashedStorage
|
||||
module RakeHelper
|
||||
def self.batch_size
|
||||
ENV.fetch('BATCH', 200).to_i
|
||||
end
|
||||
|
||||
def self.project_id_batches(&block)
|
||||
Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
|
||||
ids = relation.pluck(:id)
|
||||
|
||||
yield ids.min, ids.max
|
||||
end
|
||||
end
|
||||
|
||||
def self.legacy_attachments_relation
|
||||
Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
|
||||
JOIN projects
|
||||
ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
|
||||
SQL
|
||||
end
|
||||
|
||||
def self.hashed_attachments_relation
|
||||
Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
|
||||
JOIN projects
|
||||
ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
|
||||
SQL
|
||||
end
|
||||
|
||||
def self.relation_summary(relation_name, relation)
|
||||
relation_count = relation.count
|
||||
$stdout.puts "* Found #{relation_count} #{relation_name}".color(:green)
|
||||
|
||||
relation_count
|
||||
end
|
||||
|
||||
def self.projects_list(relation_name, relation)
|
||||
relation_count = relation_summary(relation_name, relation)
|
||||
|
||||
projects = relation.with_route
|
||||
limit = ENV.fetch('LIMIT', 500).to_i
|
||||
|
||||
return unless relation_count > 0
|
||||
|
||||
$stdout.puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
|
||||
|
||||
counter = 0
|
||||
projects.find_in_batches(batch_size: batch_size) do |batch|
|
||||
batch.each do |project|
|
||||
counter += 1
|
||||
|
||||
$stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red)
|
||||
|
||||
return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.attachments_list(relation_name, relation)
|
||||
relation_count = relation_summary(relation_name, relation)
|
||||
|
||||
limit = ENV.fetch('LIMIT', 500).to_i
|
||||
|
||||
return unless relation_count > 0
|
||||
|
||||
$stdout.puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
|
||||
|
||||
counter = 0
|
||||
relation.find_in_batches(batch_size: batch_size) do |batch|
|
||||
batch.each do |upload|
|
||||
counter += 1
|
||||
|
||||
$stdout.puts " - #{upload.path} (id: #{upload.id})".color(:red)
|
||||
|
||||
return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -10,9 +10,9 @@ namespace :gitlab do
|
|||
next
|
||||
end
|
||||
|
||||
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
|
||||
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{Gitlab::HashedStorage::RakeHelper.batch_size}"
|
||||
|
||||
project_id_batches do |start, finish|
|
||||
Gitlab::HashedStorage::RakeHelper.project_id_batches do |start, finish|
|
||||
StorageMigratorWorker.perform_async(start, finish)
|
||||
|
||||
print '.'
|
||||
|
@ -23,118 +23,42 @@ namespace :gitlab do
|
|||
|
||||
desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
|
||||
task legacy_projects: :environment do
|
||||
relation_summary('projects', Project.without_storage_feature(:repository))
|
||||
Gitlab::HashedStorage::RakeHelper.relation_summary('projects', Project.without_storage_feature(:repository))
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | List existing projects using Legacy Storage'
|
||||
task list_legacy_projects: :environment do
|
||||
projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
|
||||
Gitlab::HashedStorage::RakeHelper.projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
|
||||
task hashed_projects: :environment do
|
||||
relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
|
||||
Gitlab::HashedStorage::RakeHelper.relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | List existing projects using Hashed Storage'
|
||||
task list_hashed_projects: :environment do
|
||||
projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
|
||||
Gitlab::HashedStorage::RakeHelper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
|
||||
task legacy_attachments: :environment do
|
||||
relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
|
||||
Gitlab::HashedStorage::RakeHelper.relation_summary('attachments using Legacy Storage', Gitlab::HashedStorage::RakeHelper.legacy_attachments_relation)
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
|
||||
task list_legacy_attachments: :environment do
|
||||
attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
|
||||
Gitlab::HashedStorage::RakeHelper.attachments_list('attachments using Legacy Storage', Gitlab::HashedStorage::RakeHelper.legacy_attachments_relation)
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
|
||||
task hashed_attachments: :environment do
|
||||
relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
|
||||
Gitlab::HashedStorage::RakeHelper.relation_summary('attachments using Hashed Storage', Gitlab::HashedStorage::RakeHelper.hashed_attachments_relation)
|
||||
end
|
||||
|
||||
desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
|
||||
task list_hashed_attachments: :environment do
|
||||
attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
|
||||
end
|
||||
|
||||
def batch_size
|
||||
ENV.fetch('BATCH', 200).to_i
|
||||
end
|
||||
|
||||
def project_id_batches(&block)
|
||||
Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
|
||||
ids = relation.pluck(:id)
|
||||
|
||||
yield ids.min, ids.max
|
||||
end
|
||||
end
|
||||
|
||||
def legacy_attachments_relation
|
||||
Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
|
||||
JOIN projects
|
||||
ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
|
||||
SQL
|
||||
end
|
||||
|
||||
def hashed_attachments_relation
|
||||
Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
|
||||
JOIN projects
|
||||
ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
|
||||
SQL
|
||||
end
|
||||
|
||||
def relation_summary(relation_name, relation)
|
||||
relation_count = relation.count
|
||||
puts "* Found #{relation_count} #{relation_name}".color(:green)
|
||||
|
||||
relation_count
|
||||
end
|
||||
|
||||
def projects_list(relation_name, relation)
|
||||
relation_count = relation_summary(relation_name, relation)
|
||||
|
||||
projects = relation.with_route
|
||||
limit = ENV.fetch('LIMIT', 500).to_i
|
||||
|
||||
return unless relation_count > 0
|
||||
|
||||
puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
|
||||
|
||||
counter = 0
|
||||
projects.find_in_batches(batch_size: batch_size) do |batch|
|
||||
batch.each do |project|
|
||||
counter += 1
|
||||
|
||||
puts " - #{project.full_path} (id: #{project.id})".color(:red)
|
||||
|
||||
return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def attachments_list(relation_name, relation)
|
||||
relation_count = relation_summary(relation_name, relation)
|
||||
|
||||
limit = ENV.fetch('LIMIT', 500).to_i
|
||||
|
||||
return unless relation_count > 0
|
||||
|
||||
puts " ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
|
||||
|
||||
counter = 0
|
||||
relation.find_in_batches(batch_size: batch_size) do |batch|
|
||||
batch.each do |upload|
|
||||
counter += 1
|
||||
|
||||
puts " - #{upload.path} (id: #{upload.id})".color(:red)
|
||||
|
||||
return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator, Cop/AvoidReturnFromBlocks
|
||||
end
|
||||
end
|
||||
Gitlab::HashedStorage::RakeHelper.attachments_list('attachments using Hashed Storage', Gitlab::HashedStorage::RakeHelper.hashed_attachments_relation)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue