2017-11-23 12:56:51 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
# rubocop:disable Metrics/MethodLength
|
|
|
|
# rubocop:disable Metrics/LineLength
|
|
|
|
# rubocop:disable Metrics/AbcSize
|
|
|
|
# rubocop:disable Style/Documentation
|
|
|
|
|
2017-07-03 14:48:59 +00:00
|
|
|
module Gitlab
|
|
|
|
module BackgroundMigration
|
|
|
|
class DeserializeMergeRequestDiffsAndCommits
|
|
|
|
attr_reader :diff_ids, :commit_rows, :file_rows
|
|
|
|
|
Make MR diff background migration less likely to time out
This version does not use transactions, but individual statements. As we have
unique constraints on the target tables for the inserts, we can just ignore
uniqueness violations there (as long as we always insert the same batch size, in
the same order).
This means the spec now must use truncation, not a transaction, as the
uniqueness violation means that the whole transaction for that spec would be
invalid, which isn't what we'd want. In real-world use, this isn't run in a
transaction anyway.
This commit also wraps unhandled exceptions, for easier finding in Sentry, and
logs with a consistent format, for easier searching.
2017-09-26 14:09:37 +00:00
|
|
|
class Error < StandardError
|
|
|
|
def backtrace
|
|
|
|
cause.backtrace
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-07-03 14:48:59 +00:00
|
|
|
class MergeRequestDiff < ActiveRecord::Base
|
|
|
|
self.table_name = 'merge_request_diffs'
|
|
|
|
end
|
|
|
|
|
|
|
|
BUFFER_ROWS = 1000
|
2017-08-18 11:33:29 +00:00
|
|
|
DIFF_FILE_BUFFER_ROWS = 100
|
2017-07-03 14:48:59 +00:00
|
|
|
|
|
|
|
def perform(start_id, stop_id)
|
|
|
|
merge_request_diffs = MergeRequestDiff
|
|
|
|
.select(:id, :st_commits, :st_diffs)
|
|
|
|
.where('st_commits IS NOT NULL OR st_diffs IS NOT NULL')
|
|
|
|
.where(id: start_id..stop_id)
|
|
|
|
|
|
|
|
reset_buffers!
|
|
|
|
|
|
|
|
merge_request_diffs.each do |merge_request_diff|
|
|
|
|
commits, files = single_diff_rows(merge_request_diff)
|
|
|
|
|
|
|
|
diff_ids << merge_request_diff.id
|
|
|
|
commit_rows.concat(commits)
|
|
|
|
file_rows.concat(files)
|
|
|
|
|
|
|
|
if diff_ids.length > BUFFER_ROWS ||
|
|
|
|
commit_rows.length > BUFFER_ROWS ||
|
2017-08-18 11:33:29 +00:00
|
|
|
file_rows.length > DIFF_FILE_BUFFER_ROWS
|
2017-07-03 14:48:59 +00:00
|
|
|
|
|
|
|
flush_buffers!
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
flush_buffers!
|
Make MR diff background migration less likely to time out
This version does not use transactions, but individual statements. As we have
unique constraints on the target tables for the inserts, we can just ignore
uniqueness violations there (as long as we always insert the same batch size, in
the same order).
This means the spec now must use truncation, not a transaction, as the
uniqueness violation means that the whole transaction for that spec would be
invalid, which isn't what we'd want. In real-world use, this isn't run in a
transaction anyway.
This commit also wraps unhandled exceptions, for easier finding in Sentry, and
logs with a consistent format, for easier searching.
2017-09-26 14:09:37 +00:00
|
|
|
rescue => e
|
|
|
|
Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}")
|
|
|
|
|
|
|
|
raise Error.new(e.inspect)
|
2017-07-03 14:48:59 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
private
|
|
|
|
|
|
|
|
def reset_buffers!
|
|
|
|
@diff_ids = []
|
|
|
|
@commit_rows = []
|
|
|
|
@file_rows = []
|
|
|
|
end
|
|
|
|
|
|
|
|
def flush_buffers!
|
|
|
|
if diff_ids.any?
|
Make MR diff background migration less likely to time out
This version does not use transactions, but individual statements. As we have
unique constraints on the target tables for the inserts, we can just ignore
uniqueness violations there (as long as we always insert the same batch size, in
the same order).
This means the spec now must use truncation, not a transaction, as the
uniqueness violation means that the whole transaction for that spec would be
invalid, which isn't what we'd want. In real-world use, this isn't run in a
transaction anyway.
This commit also wraps unhandled exceptions, for easier finding in Sentry, and
logs with a consistent format, for easier searching.
2017-09-26 14:09:37 +00:00
|
|
|
commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice|
|
|
|
|
bulk_insert('merge_request_diff_commits', commit_rows_slice)
|
|
|
|
end
|
2017-07-03 14:48:59 +00:00
|
|
|
|
Make MR diff background migration less likely to time out
This version does not use transactions, but individual statements. As we have
unique constraints on the target tables for the inserts, we can just ignore
uniqueness violations there (as long as we always insert the same batch size, in
the same order).
This means the spec now must use truncation, not a transaction, as the
uniqueness violation means that the whole transaction for that spec would be
invalid, which isn't what we'd want. In real-world use, this isn't run in a
transaction anyway.
This commit also wraps unhandled exceptions, for easier finding in Sentry, and
logs with a consistent format, for easier searching.
2017-09-26 14:09:37 +00:00
|
|
|
file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice|
|
|
|
|
bulk_insert('merge_request_diff_files', file_rows_slice)
|
2017-07-03 14:48:59 +00:00
|
|
|
end
|
Make MR diff background migration less likely to time out
This version does not use transactions, but individual statements. As we have
unique constraints on the target tables for the inserts, we can just ignore
uniqueness violations there (as long as we always insert the same batch size, in
the same order).
This means the spec now must use truncation, not a transaction, as the
uniqueness violation means that the whole transaction for that spec would be
invalid, which isn't what we'd want. In real-world use, this isn't run in a
transaction anyway.
This commit also wraps unhandled exceptions, for easier finding in Sentry, and
logs with a consistent format, for easier searching.
2017-09-26 14:09:37 +00:00
|
|
|
|
|
|
|
MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil)
|
2017-07-03 14:48:59 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
reset_buffers!
|
|
|
|
end
|
|
|
|
|
Make MR diff background migration less likely to time out
This version does not use transactions, but individual statements. As we have
unique constraints on the target tables for the inserts, we can just ignore
uniqueness violations there (as long as we always insert the same batch size, in
the same order).
This means the spec now must use truncation, not a transaction, as the
uniqueness violation means that the whole transaction for that spec would be
invalid, which isn't what we'd want. In real-world use, this isn't run in a
transaction anyway.
This commit also wraps unhandled exceptions, for easier finding in Sentry, and
logs with a consistent format, for easier searching.
2017-09-26 14:09:37 +00:00
|
|
|
def bulk_insert(table, rows)
|
|
|
|
Gitlab::Database.bulk_insert(table, rows)
|
|
|
|
rescue ActiveRecord::RecordNotUnique
|
|
|
|
ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort
|
|
|
|
|
|
|
|
Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}")
|
|
|
|
end
|
|
|
|
|
2017-07-03 14:48:59 +00:00
|
|
|
def single_diff_rows(merge_request_diff)
|
|
|
|
sha_attribute = Gitlab::Database::ShaAttribute.new
|
|
|
|
commits = YAML.load(merge_request_diff.st_commits) rescue []
|
2017-10-17 11:15:40 +00:00
|
|
|
commits ||= []
|
2017-07-03 14:48:59 +00:00
|
|
|
|
|
|
|
commit_rows = commits.map.with_index do |commit, index|
|
|
|
|
commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids)
|
|
|
|
sha = commit_hash.delete(:id)
|
|
|
|
|
|
|
|
commit_hash.merge(
|
|
|
|
merge_request_diff_id: merge_request_diff.id,
|
|
|
|
relative_order: index,
|
|
|
|
sha: sha_attribute.type_cast_for_database(sha)
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
diffs = YAML.load(merge_request_diff.st_diffs) rescue []
|
|
|
|
diffs = [] unless valid_raw_diffs?(diffs)
|
|
|
|
|
|
|
|
file_rows = diffs.map.with_index do |diff, index|
|
|
|
|
diff_hash = diff.to_hash.with_indifferent_access.merge(
|
|
|
|
binary: false,
|
|
|
|
merge_request_diff_id: merge_request_diff.id,
|
|
|
|
relative_order: index
|
|
|
|
)
|
|
|
|
|
|
|
|
diff_hash.tap do |hash|
|
|
|
|
diff_text = hash[:diff]
|
|
|
|
|
2017-08-11 10:58:21 +00:00
|
|
|
hash[:too_large] = !!hash[:too_large]
|
|
|
|
|
2017-08-18 12:02:30 +00:00
|
|
|
hash[:a_mode] ||= guess_mode(hash[:new_file], hash[:diff])
|
|
|
|
hash[:b_mode] ||= guess_mode(hash[:deleted_file], hash[:diff])
|
|
|
|
|
|
|
|
# Compatibility with old diffs created with Psych.
|
2017-07-03 14:48:59 +00:00
|
|
|
if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?
|
|
|
|
hash[:binary] = true
|
|
|
|
hash[:diff] = [diff_text].pack('m0')
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
[commit_rows, file_rows]
|
|
|
|
end
|
|
|
|
|
2017-08-18 12:02:30 +00:00
|
|
|
# This doesn't have to be 100% accurate, because it's only used for
|
|
|
|
# display - it won't change file modes in the repository. Submodules are
|
|
|
|
# created as 600, regular files as 644.
|
|
|
|
def guess_mode(file_missing, diff)
|
|
|
|
return '0' if file_missing
|
|
|
|
|
|
|
|
diff.include?('Subproject commit') ? '160000' : '100644'
|
|
|
|
end
|
|
|
|
|
2017-07-03 14:48:59 +00:00
|
|
|
# Unlike MergeRequestDiff#valid_raw_diff?, don't count Rugged objects as
|
|
|
|
# valid, because we don't render them usefully anyway.
|
|
|
|
def valid_raw_diffs?(diffs)
|
|
|
|
return false unless diffs.respond_to?(:each)
|
|
|
|
|
|
|
|
diffs.all? { |diff| diff.is_a?(Hash) }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|