2017-11-27 06:57:21 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2017-11-06 21:44:30 +00:00
|
|
|
module Gitlab
|
|
|
|
module BackgroundMigration
|
2017-11-27 06:57:21 +00:00
|
|
|
# This class finds all non-hashed uploaded file paths and saves them to a
|
|
|
|
# `untracked_files_for_uploads` table.
|
|
|
|
class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength
|
2017-11-07 20:53:24 +00:00
|
|
|
# For bulk_queue_background_migration_jobs_by_range
|
|
|
|
include Database::MigrationHelpers
|
2018-01-09 17:30:04 +00:00
|
|
|
include ::Gitlab::Utils::StrongMemoize
|
2017-11-07 20:53:24 +00:00
|
|
|
|
2017-11-27 06:57:21 +00:00
|
|
|
FIND_BATCH_SIZE = 500
|
2017-11-15 00:11:53 +00:00
|
|
|
RELATIVE_UPLOAD_DIR = "uploads".freeze
|
2018-01-29 17:57:34 +00:00
|
|
|
ABSOLUTE_UPLOAD_DIR = File.join(
|
|
|
|
Gitlab.config.uploads.storage_path,
|
|
|
|
RELATIVE_UPLOAD_DIR
|
|
|
|
)
|
2017-11-08 04:54:54 +00:00
|
|
|
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
|
2018-01-29 17:57:34 +00:00
|
|
|
START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}
|
2017-11-15 00:11:53 +00:00
|
|
|
EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
|
|
|
|
EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
|
2017-11-07 01:07:35 +00:00
|
|
|
|
2017-11-27 06:57:21 +00:00
|
|
|
# This class is used to iterate over batches of
|
|
|
|
# `untracked_files_for_uploads` rows.
|
2017-11-10 01:17:56 +00:00
|
|
|
class UntrackedFile < ActiveRecord::Base
|
2017-11-07 20:53:24 +00:00
|
|
|
include EachBatch
|
|
|
|
|
2017-11-10 01:17:56 +00:00
|
|
|
self.table_name = 'untracked_files_for_uploads'
|
2017-11-06 21:44:30 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def perform
|
2017-11-15 13:19:07 +00:00
|
|
|
ensure_temporary_tracking_table_exists
|
2017-11-21 00:27:24 +00:00
|
|
|
|
|
|
|
# Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since
|
|
|
|
# doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be
|
|
|
|
# slow, start with an empty table for Postgres < 9.5.
|
|
|
|
# That way we can do bulk inserts at ~30x the speed of individual
|
|
|
|
# inserts (~20 minutes worth of inserts at GitLab.com scale instead of
|
|
|
|
# ~10 hours).
|
|
|
|
# In all other cases, installations will get both bulk inserts and the
|
|
|
|
# ability for these jobs to retry without having to clear and reinsert.
|
|
|
|
clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates?
|
|
|
|
|
2017-11-10 01:17:56 +00:00
|
|
|
store_untracked_file_paths
|
2017-11-21 00:27:24 +00:00
|
|
|
|
2018-02-08 18:17:00 +00:00
|
|
|
if UntrackedFile.all.empty?
|
|
|
|
drop_temp_table
|
|
|
|
else
|
|
|
|
schedule_populate_untracked_uploads_jobs
|
|
|
|
end
|
2017-11-06 21:44:30 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
2017-11-15 13:19:07 +00:00
|
|
|
def ensure_temporary_tracking_table_exists
|
2017-11-27 06:57:21 +00:00
|
|
|
table_name = :untracked_files_for_uploads
|
|
|
|
unless UntrackedFile.connection.table_exists?(table_name)
|
|
|
|
UntrackedFile.connection.create_table table_name do |t|
|
2017-11-15 13:19:07 +00:00
|
|
|
t.string :path, limit: 600, null: false
|
2017-11-22 18:23:24 +00:00
|
|
|
t.index :path, unique: true
|
2017-11-15 13:19:07 +00:00
|
|
|
end
|
|
|
|
end
|
2017-11-06 21:44:30 +00:00
|
|
|
end
|
|
|
|
|
2017-11-21 00:27:24 +00:00
|
|
|
def clear_untracked_file_paths
|
|
|
|
UntrackedFile.delete_all
|
|
|
|
end
|
|
|
|
|
2017-11-10 01:17:56 +00:00
|
|
|
def store_untracked_file_paths
|
2017-11-15 00:11:53 +00:00
|
|
|
return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR)
|
2017-11-07 01:07:35 +00:00
|
|
|
|
2017-11-27 06:57:21 +00:00
|
|
|
each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths|
|
2017-11-08 23:05:08 +00:00
|
|
|
insert_file_paths(file_paths)
|
2017-11-07 01:07:35 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-11-08 23:05:08 +00:00
|
|
|
def each_file_batch(search_dir, batch_size, &block)
|
2017-11-07 01:07:35 +00:00
|
|
|
cmd = build_find_command(search_dir)
|
2017-11-08 23:05:08 +00:00
|
|
|
|
2017-11-07 01:07:35 +00:00
|
|
|
Open3.popen2(*cmd) do |stdin, stdout, status_thread|
|
2017-11-08 23:05:08 +00:00
|
|
|
yield_paths_in_batches(stdout, batch_size, &block)
|
|
|
|
|
2017-11-07 01:07:35 +00:00
|
|
|
raise "Find command failed" unless status_thread.value.success?
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-11-08 23:05:08 +00:00
|
|
|
def yield_paths_in_batches(stdout, batch_size, &block)
|
|
|
|
paths = []
|
|
|
|
|
|
|
|
stdout.each_line("\0") do |line|
|
2018-01-29 17:57:34 +00:00
|
|
|
paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '')
|
2017-11-08 23:05:08 +00:00
|
|
|
|
|
|
|
if paths.size >= batch_size
|
|
|
|
yield(paths)
|
|
|
|
paths = []
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-02-08 18:08:55 +00:00
|
|
|
yield(paths) if paths.any?
|
2017-11-08 23:05:08 +00:00
|
|
|
end
|
|
|
|
|
2017-11-07 01:07:35 +00:00
|
|
|
def build_find_command(search_dir)
|
2017-12-04 20:57:44 +00:00
|
|
|
cmd = %W[find -L #{search_dir}
|
2017-11-27 06:57:21 +00:00
|
|
|
-type f
|
|
|
|
! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune )
|
|
|
|
! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune )
|
|
|
|
-print0]
|
2017-11-07 01:07:35 +00:00
|
|
|
|
2017-11-17 21:49:25 +00:00
|
|
|
ionice = which_ionice
|
|
|
|
cmd = %W[#{ionice} -c Idle] + cmd if ionice
|
2017-11-07 01:07:35 +00:00
|
|
|
|
2017-11-27 06:57:21 +00:00
|
|
|
log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\""
|
|
|
|
Rails.logger.info log_msg
|
2017-11-17 00:24:42 +00:00
|
|
|
|
2017-11-07 01:07:35 +00:00
|
|
|
cmd
|
|
|
|
end
|
|
|
|
|
2017-11-17 21:49:25 +00:00
|
|
|
def which_ionice
|
2017-11-07 01:07:35 +00:00
|
|
|
Gitlab::Utils.which('ionice')
|
|
|
|
rescue StandardError
|
2017-11-27 06:57:21 +00:00
|
|
|
# In this case, returning false is relatively safe,
|
|
|
|
# even though it isn't very nice
|
2017-11-07 01:07:35 +00:00
|
|
|
false
|
|
|
|
end
|
|
|
|
|
|
|
|
def insert_file_paths(file_paths)
|
2017-11-27 06:57:21 +00:00
|
|
|
sql = insert_sql(file_paths)
|
2017-11-06 21:44:30 +00:00
|
|
|
|
2017-11-21 00:27:24 +00:00
|
|
|
ActiveRecord::Base.connection.execute(sql)
|
|
|
|
end
|
2017-11-15 12:51:28 +00:00
|
|
|
|
2017-11-27 06:57:21 +00:00
|
|
|
def insert_sql(file_paths)
|
|
|
|
if postgresql_pre_9_5?
|
|
|
|
"INSERT INTO #{table_columns_and_values_for_insert(file_paths)};"
|
|
|
|
elsif postgresql?
|
|
|
|
"INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\
|
|
|
|
" ON CONFLICT DO NOTHING;"
|
|
|
|
else # MySQL
|
|
|
|
"INSERT IGNORE INTO"\
|
|
|
|
" #{table_columns_and_values_for_insert(file_paths)};"
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-11-21 00:27:24 +00:00
|
|
|
def table_columns_and_values_for_insert(file_paths)
|
|
|
|
values = file_paths.map do |file_path|
|
2017-11-27 06:57:21 +00:00
|
|
|
ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength
|
2017-11-21 00:27:24 +00:00
|
|
|
end.join(', ')
|
2017-11-08 20:31:51 +00:00
|
|
|
|
2017-11-24 07:51:29 +00:00
|
|
|
"#{UntrackedFile.table_name} (path) VALUES #{values}"
|
2017-11-08 20:31:51 +00:00
|
|
|
end
|
|
|
|
|
2017-11-15 12:51:28 +00:00
|
|
|
def postgresql?
|
2018-01-09 17:30:04 +00:00
|
|
|
strong_memoize(:postgresql) do
|
|
|
|
Gitlab::Database.postgresql?
|
|
|
|
end
|
2017-11-15 12:51:28 +00:00
|
|
|
end
|
|
|
|
|
2017-11-21 00:27:24 +00:00
|
|
|
def can_bulk_insert_and_ignore_duplicates?
|
|
|
|
!postgresql_pre_9_5?
|
|
|
|
end
|
|
|
|
|
2017-11-15 12:51:28 +00:00
|
|
|
def postgresql_pre_9_5?
|
2018-01-09 17:30:04 +00:00
|
|
|
strong_memoize(:postgresql_pre_9_5) do
|
|
|
|
postgresql? && Gitlab::Database.version.to_f < 9.5
|
|
|
|
end
|
2017-11-15 12:51:28 +00:00
|
|
|
end
|
|
|
|
|
2017-11-06 21:44:30 +00:00
|
|
|
def schedule_populate_untracked_uploads_jobs
|
2017-11-27 06:57:21 +00:00
|
|
|
bulk_queue_background_migration_jobs_by_range(
|
|
|
|
UntrackedFile, FOLLOW_UP_MIGRATION)
|
2017-11-06 21:44:30 +00:00
|
|
|
end
|
2018-02-08 18:17:00 +00:00
|
|
|
|
|
|
|
def drop_temp_table
|
2018-02-13 20:31:30 +00:00
|
|
|
unless Rails.env.test? # Dropping a table intermittently breaks test cleanup
|
|
|
|
UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
|
|
|
|
if_exists: true)
|
|
|
|
end
|
2018-02-08 18:17:00 +00:00
|
|
|
end
|
2017-11-06 21:44:30 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|