Merge branch 'banzai-issue-filter-queries' into 'master'
Reduce SQL query counts in IssueReferenceFilter ## What does this MR do? This MR adds a preparation phase for reference filters that allows them to prepare/create data structures used while iterating over HTML nodes. In this particular case the preparation phase is used for issue references to greatly cut down the amount of queries executed to get projects/issues for Markdown references. ## Are there points in the code the reviewer needs to double check? No. ## Why was this MR needed? Rendering Markdown containing issue references would run at most two queries for every issue reference: one to get the project and one to get the issue from said project. When rendering Markdown with lots of issue references this would result in _a lot_ of queries being executed. ## What are the relevant issue numbers? #18042 See merge request !4410
This commit is contained in:
commit
84632f0aff
|
@ -103,7 +103,7 @@ module Banzai
|
|||
ref_pattern = object_class.reference_pattern
|
||||
link_pattern = object_class.link_reference_pattern
|
||||
|
||||
each_node do |node|
|
||||
nodes.each do |node|
|
||||
if text_node?(node) && ref_pattern
|
||||
replace_text_when_pattern_matches(node, ref_pattern) do |content|
|
||||
object_link_filter(content, ref_pattern)
|
||||
|
@ -206,6 +206,55 @@ module Banzai
|
|||
text
|
||||
end
|
||||
|
||||
# Returns a Hash containing all object references (e.g. issue IDs) per the
|
||||
# project they belong to.
|
||||
def references_per_project
|
||||
@references_per_project ||= begin
|
||||
refs = Hash.new { |hash, key| hash[key] = Set.new }
|
||||
|
||||
regex = Regexp.union(object_class.reference_pattern,
|
||||
object_class.link_reference_pattern)
|
||||
|
||||
nodes.each do |node|
|
||||
node.to_html.scan(regex) do
|
||||
project = $~[:project] || current_project_path
|
||||
|
||||
refs[project] << $~[object_sym]
|
||||
end
|
||||
end
|
||||
|
||||
refs
|
||||
end
|
||||
end
|
||||
|
||||
# Returns a Hash containing referenced projects grouped per their full
|
||||
# path.
|
||||
def projects_per_reference
|
||||
@projects_per_reference ||= begin
|
||||
hash = {}
|
||||
refs = Set.new
|
||||
|
||||
references_per_project.each do |project_ref, _|
|
||||
refs << project_ref
|
||||
end
|
||||
|
||||
find_projects_for_paths(refs.to_a).each do |project|
|
||||
hash[project.path_with_namespace] = project
|
||||
end
|
||||
|
||||
hash
|
||||
end
|
||||
end
|
||||
|
||||
# Returns the projects for the given paths.
|
||||
def find_projects_for_paths(paths)
|
||||
Project.where_paths_in(paths).includes(:namespace)
|
||||
end
|
||||
|
||||
def current_project_path
|
||||
@current_project_path ||= project.path_with_namespace
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def project_refs_cache
|
||||
|
|
|
@ -11,13 +11,40 @@ module Banzai
|
|||
Issue
|
||||
end
|
||||
|
||||
def find_object(project, id)
|
||||
project.get_issue(id)
|
||||
def find_object(project, iid)
|
||||
issues_per_project[project][iid]
|
||||
end
|
||||
|
||||
def url_for_object(issue, project)
|
||||
IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path])
|
||||
end
|
||||
|
||||
def project_from_ref(ref)
|
||||
projects_per_reference[ref || current_project_path]
|
||||
end
|
||||
|
||||
# Returns a Hash containing the issues per Project instance.
|
||||
def issues_per_project
|
||||
@issues_per_project ||= begin
|
||||
hash = Hash.new { |h, k| h[k] = {} }
|
||||
|
||||
projects_per_reference.each do |path, project|
|
||||
issue_ids = references_per_project[path]
|
||||
|
||||
next unless project.default_issues_tracker?
|
||||
|
||||
project.issues.where(iid: issue_ids.to_a).each do |issue|
|
||||
hash[project][issue.iid] = issue
|
||||
end
|
||||
end
|
||||
|
||||
hash
|
||||
end
|
||||
end
|
||||
|
||||
def find_projects_for_paths(paths)
|
||||
super(paths).includes(:gitlab_issue_tracker_service)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe Banzai::Filter::AbstractReferenceFilter do
|
||||
let(:project) { create(:empty_project) }
|
||||
|
||||
describe '#references_per_project' do
|
||||
it 'returns a Hash containing references grouped per project paths' do
|
||||
doc = Nokogiri::HTML.fragment("#1 #{project.to_reference}#2")
|
||||
filter = described_class.new(doc, project: project)
|
||||
|
||||
expect(filter).to receive(:object_class).twice.and_return(Issue)
|
||||
expect(filter).to receive(:object_sym).twice.and_return(:issue)
|
||||
|
||||
refs = filter.references_per_project
|
||||
|
||||
expect(refs).to be_an_instance_of(Hash)
|
||||
expect(refs[project.to_reference]).to eq(Set.new(%w[1 2]))
|
||||
end
|
||||
end
|
||||
|
||||
describe '#projects_per_reference' do
|
||||
it 'returns a Hash containing projects grouped per project paths' do
|
||||
doc = Nokogiri::HTML.fragment('')
|
||||
filter = described_class.new(doc, project: project)
|
||||
|
||||
expect(filter).to receive(:references_per_project).
|
||||
and_return({ project.path_with_namespace => Set.new(%w[1]) })
|
||||
|
||||
expect(filter.projects_per_reference).
|
||||
to eq({ project.path_with_namespace => project })
|
||||
end
|
||||
end
|
||||
|
||||
describe '#find_projects_for_paths' do
|
||||
it 'returns a list of Projects for a list of paths' do
|
||||
doc = Nokogiri::HTML.fragment('')
|
||||
filter = described_class.new(doc, project: project)
|
||||
|
||||
expect(filter.find_projects_for_paths([project.path_with_namespace])).
|
||||
to eq([project])
|
||||
end
|
||||
end
|
||||
|
||||
describe '#current_project_path' do
|
||||
it 'returns the path of the current project' do
|
||||
doc = Nokogiri::HTML.fragment('')
|
||||
filter = described_class.new(doc, project: project)
|
||||
|
||||
expect(filter.current_project_path).to eq(project.path_with_namespace)
|
||||
end
|
||||
end
|
||||
end
|
|
@ -25,7 +25,9 @@ describe Banzai::Filter::IssueReferenceFilter, lib: true do
|
|||
let(:reference) { issue.to_reference }
|
||||
|
||||
it 'ignores valid references when using non-default tracker' do
|
||||
expect(project).to receive(:get_issue).with(issue.iid).and_return(nil)
|
||||
expect_any_instance_of(described_class).to receive(:find_object).
|
||||
with(project, issue.iid).
|
||||
and_return(nil)
|
||||
|
||||
exp = act = "Issue #{reference}"
|
||||
expect(reference_filter(act).to_html).to eq exp
|
||||
|
@ -107,8 +109,9 @@ describe Banzai::Filter::IssueReferenceFilter, lib: true do
|
|||
let(:reference) { issue.to_reference(project) }
|
||||
|
||||
it 'ignores valid references when cross-reference project uses external tracker' do
|
||||
expect_any_instance_of(Project).to receive(:get_issue).
|
||||
with(issue.iid).and_return(nil)
|
||||
expect_any_instance_of(described_class).to receive(:find_object).
|
||||
with(project2, issue.iid).
|
||||
and_return(nil)
|
||||
|
||||
exp = act = "Issue #{reference}"
|
||||
expect(reference_filter(act).to_html).to eq exp
|
||||
|
|
|
@ -312,7 +312,8 @@ describe GitPushService, services: true do
|
|||
end
|
||||
|
||||
it "doesn't close issues when external issue tracker is in use" do
|
||||
allow(project).to receive(:default_issues_tracker?).and_return(false)
|
||||
allow_any_instance_of(Project).to receive(:default_issues_tracker?).
|
||||
and_return(false)
|
||||
|
||||
# The push still shouldn't create cross-reference notes.
|
||||
expect do
|
||||
|
|
Loading…
Reference in New Issue