diff --git a/app/models/concerns/diff_file.rb b/app/models/concerns/diff_file.rb new file mode 100644 index 00000000000..72332072012 --- /dev/null +++ b/app/models/concerns/diff_file.rb @@ -0,0 +1,9 @@ +module DiffFile + extend ActiveSupport::Concern + + def to_hash + keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] + + as_json(only: keys).merge(diff: diff).with_indifferent_access + end +end diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 616a626419b..d752d5bcdee 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -3,6 +3,7 @@ # A note of this type can be resolvable. class DiffNote < Note include NoteOnDiff + include Gitlab::Utils::StrongMemoize NOTEABLE_TYPES = %w(MergeRequest Commit).freeze @@ -12,7 +13,6 @@ class DiffNote < Note validates :original_position, presence: true validates :position, presence: true - validates :diff_line, presence: true, if: :on_text? validates :line_code, presence: true, line_code: true, if: :on_text? validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete @@ -23,6 +23,7 @@ class DiffNote < Note before_validation :update_position, on: :create, if: :on_text? before_validation :set_line_code, if: :on_text? after_save :keep_around_commits + after_commit :create_diff_file, on: :create def discussion_class(*) DiffDiscussion @@ -53,21 +54,25 @@ class DiffNote < Note position.position_type == "image" end + def create_diff_file + return unless should_create_diff_file? + + diff_file = fetch_diff_file + diff_line = diff_file.line_for_position(self.original_position) + + creation_params = diff_file.diff.to_hash + .except(:too_large) + .merge(diff: diff_file.diff_hunk(diff_line)) + + create_note_diff_file(creation_params) + end + def diff_file - @diff_file ||= - begin - if created_at_diff?(noteable.diff_refs) - # We're able to use the already persisted diffs (Postgres) if we're - # presenting a "current version" of the MR discussion diff. - # So no need to make an extra Gitaly diff request for it. - # As an extra benefit, the returned `diff_file` already - # has `highlighted_diff_lines` data set from Redis on - # `Diff::FileCollection::MergeRequestDiff`. - noteable.diffs(paths: original_position.paths, expanded: true).diff_files.first - else - original_position.diff_file(self.project.repository) - end - end + strong_memoize(:diff_file) do + enqueue_diff_file_creation_job if should_create_diff_file? + + fetch_diff_file + end end def diff_line @@ -98,6 +103,38 @@ class DiffNote < Note private + def enqueue_diff_file_creation_job + # Avoid enqueuing multiple file creation jobs at once for a note (i.e. + # parallel calls to `DiffNote#diff_file`). + lease = Gitlab::ExclusiveLease.new("note_diff_file_creation:#{id}", timeout: 1.hour.to_i) + return unless lease.try_obtain + + CreateNoteDiffFileWorker.perform_async(id) + end + + def should_create_diff_file? + on_text? && note_diff_file.nil? && self == discussion.first_note + end + + def fetch_diff_file + if note_diff_file + diff = Gitlab::Git::Diff.new(note_diff_file.to_hash) + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: original_position.diff_refs) + elsif created_at_diff?(noteable.diff_refs) + # We're able to use the already persisted diffs (Postgres) if we're + # presenting a "current version" of the MR discussion diff. + # So no need to make an extra Gitaly diff request for it. + # As an extra benefit, the returned `diff_file` already + # has `highlighted_diff_lines` data set from Redis on + # `Diff::FileCollection::MergeRequestDiff`. + noteable.diffs(paths: original_position.paths, expanded: true).diff_files.first + else + original_position.diff_file(self.project.repository) + end + end + def supported? for_commit? || self.noteable.has_complete_diff_refs? end diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index 1199ff5af22..cd8ba6b904d 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -1,5 +1,6 @@ class MergeRequestDiffFile < ActiveRecord::Base include Gitlab::EncodingHelper + include DiffFile belongs_to :merge_request_diff @@ -12,10 +13,4 @@ class MergeRequestDiffFile < ActiveRecord::Base def diff binary? ? super.unpack('m0').first : super end - - def to_hash - keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] - - as_json(only: keys).merge(diff: diff).with_indifferent_access - end end diff --git a/app/models/note.rb b/app/models/note.rb index 109405d3f17..02f7a9b1e4f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -63,6 +63,7 @@ class Note < ActiveRecord::Base has_many :todos has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :system_note_metadata + has_one :note_diff_file, inverse_of: :diff_note, foreign_key: :diff_note_id delegate :gfm_reference, :local_reference, to: :noteable delegate :name, to: :project, prefix: true @@ -100,7 +101,8 @@ class Note < ActiveRecord::Base scope :inc_author_project, -> { includes(:project, :author) } scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> do - includes(:project, :author, :updated_by, :resolved_by, :award_emoji, :system_note_metadata) + includes(:project, :author, :updated_by, :resolved_by, :award_emoji, + :system_note_metadata, :note_diff_file) end scope :diff_notes, -> { where(type: %w(LegacyDiffNote DiffNote)) } diff --git a/app/models/note_diff_file.rb b/app/models/note_diff_file.rb new file mode 100644 index 00000000000..e688018a6d9 --- /dev/null +++ b/app/models/note_diff_file.rb @@ -0,0 +1,7 @@ +class NoteDiffFile < ActiveRecord::Base + include DiffFile + + belongs_to :diff_note, inverse_of: :note_diff_file + + validates :diff_note, presence: true +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index b6433eb3eff..d07865a4043 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -115,3 +115,4 @@ - upload_checksum - web_hook - repository_update_remote_mirror +- create_note_diff_file diff --git a/app/workers/create_note_diff_file_worker.rb b/app/workers/create_note_diff_file_worker.rb new file mode 100644 index 00000000000..624b638a24e --- /dev/null +++ b/app/workers/create_note_diff_file_worker.rb @@ -0,0 +1,9 @@ +class CreateNoteDiffFileWorker + include ApplicationWorker + + def perform(diff_note_id) + diff_note = DiffNote.find(diff_note_id) + + diff_note.create_diff_file + end +end diff --git a/changelogs/unreleased/45190-create-notes-diff-files.yml b/changelogs/unreleased/45190-create-notes-diff-files.yml new file mode 100644 index 00000000000..efe322b682d --- /dev/null +++ b/changelogs/unreleased/45190-create-notes-diff-files.yml @@ -0,0 +1,5 @@ +--- +title: Persist truncated note diffs on a new table +merge_request: +author: +type: performance diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index e1e8f36b663..d16060e8f45 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -75,4 +75,5 @@ - [pipeline_background, 1] - [repository_update_remote_mirror, 1] - [repository_remove_remote, 1] + - [create_note_diff_file, 1] diff --git a/db/migrate/20180515121227_create_notes_diff_files.rb b/db/migrate/20180515121227_create_notes_diff_files.rb new file mode 100644 index 00000000000..7108bc1a64b --- /dev/null +++ b/db/migrate/20180515121227_create_notes_diff_files.rb @@ -0,0 +1,21 @@ +class CreateNotesDiffFiles < ActiveRecord::Migration + DOWNTIME = false + + disable_ddl_transaction! + + def change + create_table :note_diff_files do |t| + t.references :diff_note, references: :notes, null: false, index: { unique: true } + t.text :diff, null: false + t.boolean :new_file, null: false + t.boolean :renamed_file, null: false + t.boolean :deleted_file, null: false + t.string :a_mode, null: false + t.string :b_mode, null: false + t.text :new_path, null: false + t.text :old_path, null: false + end + + add_foreign_key :note_diff_files, :notes, column: :diff_note_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index 37d336b9928..884e333874c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1302,6 +1302,20 @@ ActiveRecord::Schema.define(version: 20180521171529) do add_index "namespaces", ["runners_token"], name: "index_namespaces_on_runners_token", unique: true, using: :btree add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree + create_table "note_diff_files", force: :cascade do |t| + t.integer "diff_note_id", null: false + t.text "diff", null: false + t.boolean "new_file", null: false + t.boolean "renamed_file", null: false + t.boolean "deleted_file", null: false + t.string "a_mode", null: false + t.string "b_mode", null: false + t.text "new_path", null: false + t.text "old_path", null: false + end + + add_index "note_diff_files", ["diff_note_id"], name: "index_note_diff_files_on_diff_note_id", unique: true, using: :btree + create_table "notes", force: :cascade do |t| t.text "note" t.string "noteable_type" @@ -2243,6 +2257,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do add_foreign_key "merge_requests_closing_issues", "merge_requests", on_delete: :cascade add_foreign_key "milestones", "namespaces", column: "group_id", name: "fk_95650a40d4", on_delete: :cascade add_foreign_key "milestones", "projects", name: "fk_9bd0a0c791", on_delete: :cascade + add_foreign_key "note_diff_files", "notes", column: "diff_note_id", on_delete: :cascade add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id" add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 014854da55c..765fb0289a8 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -76,6 +76,13 @@ module Gitlab line_code(line) if line end + # Returns the raw diff content up to the given line index + def diff_hunk(diff_line) + # Adding 2 because of the @@ diff header and Enum#take should consider + # an extra line, because we're passing an index. + raw_diff.each_line.take(diff_line.index + 2).join + end + def old_sha diff_refs&.base_sha end diff --git a/spec/lib/gitlab/diff/file_spec.rb b/spec/lib/gitlab/diff/file_spec.rb index 0c2e18c268a..0588fe935c3 100644 --- a/spec/lib/gitlab/diff/file_spec.rb +++ b/spec/lib/gitlab/diff/file_spec.rb @@ -468,4 +468,58 @@ describe Gitlab::Diff::File do end end end + + describe '#diff_hunk' do + let(:raw_diff) do + < path } +- options = { chdir: path } ++ ++ vars = { ++ "PWD" => path ++ } ++ ++ options = { ++ chdir: path ++ } + + unless File.directory?(path) + FileUtils.mkdir_p(path) +@@ -19,6 +25,7 @@ module Popen + + @cmd_output = "" + @cmd_status = 0 ++ + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + @cmd_output << stdout.read + @cmd_output << stderr.read +EOS + end + + it 'returns raw diff up to given line index' do + allow(diff_file).to receive(:raw_diff) { raw_diff } + diff_line = instance_double(Gitlab::Diff::Line, index: 5) + + diff_hunk = <