Insert at most 1,000 rows at once in MR diff background migration
We were hitting the statement timeout for very large MR diffs. Now we insert at most 1,000 rows to `merge_request_diff_commits` in a single statement, or 100 rows to `merge_request_diff_files`.
This commit is contained in:
parent
ccfe686007
commit
917194153f
2 changed files with 39 additions and 5 deletions
|
@ -8,6 +8,7 @@ module Gitlab
|
|||
end
|
||||
|
||||
BUFFER_ROWS = 1000
|
||||
DIFF_FILE_BUFFER_ROWS = 100
|
||||
|
||||
def perform(start_id, stop_id)
|
||||
merge_request_diffs = MergeRequestDiff
|
||||
|
@ -26,7 +27,7 @@ module Gitlab
|
|||
|
||||
if diff_ids.length > BUFFER_ROWS ||
|
||||
commit_rows.length > BUFFER_ROWS ||
|
||||
file_rows.length > BUFFER_ROWS
|
||||
file_rows.length > DIFF_FILE_BUFFER_ROWS
|
||||
|
||||
flush_buffers!
|
||||
end
|
||||
|
@ -46,8 +47,13 @@ module Gitlab
|
|||
def flush_buffers!
|
||||
if diff_ids.any?
|
||||
MergeRequestDiff.transaction do
|
||||
Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows)
|
||||
Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows)
|
||||
commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice|
|
||||
Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows_slice)
|
||||
end
|
||||
|
||||
file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice|
|
||||
Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows_slice)
|
||||
end
|
||||
|
||||
MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil)
|
||||
end
|
||||
|
|
|
@ -70,8 +70,8 @@ describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do
|
|||
before do
|
||||
merge_request.reload_diff(true)
|
||||
|
||||
convert_to_yaml(start_id, merge_request_diff.commits, merge_request_diff.diffs)
|
||||
convert_to_yaml(stop_id, updated_merge_request_diff.commits, updated_merge_request_diff.diffs)
|
||||
convert_to_yaml(start_id, merge_request_diff.commits, diffs_to_hashes(merge_request_diff.merge_request_diff_files))
|
||||
convert_to_yaml(stop_id, updated_merge_request_diff.commits, diffs_to_hashes(updated_merge_request_diff.merge_request_diff_files))
|
||||
|
||||
MergeRequestDiffCommit.delete_all
|
||||
MergeRequestDiffFile.delete_all
|
||||
|
@ -80,6 +80,8 @@ describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do
|
|||
context 'when BUFFER_ROWS is exceeded' do
|
||||
before do
|
||||
stub_const("#{described_class}::BUFFER_ROWS", 1)
|
||||
|
||||
allow(Gitlab::Database).to receive(:bulk_insert).and_call_original
|
||||
end
|
||||
|
||||
it 'updates and continues' do
|
||||
|
@ -87,6 +89,32 @@ describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do
|
|||
|
||||
subject.perform(start_id, stop_id)
|
||||
end
|
||||
|
||||
it 'inserts commit rows in chunks of BUFFER_ROWS' do
|
||||
# There are 29 commits in each diff, so we should have slices of 20 + 9 + 20 + 9.
|
||||
stub_const("#{described_class}::BUFFER_ROWS", 20)
|
||||
|
||||
expect(Gitlab::Database).to receive(:bulk_insert)
|
||||
.with('merge_request_diff_commits', anything)
|
||||
.exactly(4)
|
||||
.times
|
||||
.and_call_original
|
||||
|
||||
subject.perform(start_id, stop_id)
|
||||
end
|
||||
|
||||
it 'inserts diff rows in chunks of DIFF_FILE_BUFFER_ROWS' do
|
||||
# There are 20 files in each diff, so we should have slices of 20 + 20.
|
||||
stub_const("#{described_class}::DIFF_FILE_BUFFER_ROWS", 20)
|
||||
|
||||
expect(Gitlab::Database).to receive(:bulk_insert)
|
||||
.with('merge_request_diff_files', anything)
|
||||
.exactly(2)
|
||||
.times
|
||||
.and_call_original
|
||||
|
||||
subject.perform(start_id, stop_id)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when BUFFER_ROWS is not exceeded' do
|
||||
|
|
Loading…
Reference in a new issue