2019-07-25 05:27:42 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
2017-01-23 20:40:25 +00:00
|
|
|
shared_examples 'issuables list meta-data' do |issuable_type, action = nil|
|
2018-04-24 11:06:05 +00:00
|
|
|
include ProjectForksHelper
|
2017-01-23 20:40:25 +00:00
|
|
|
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
def get_action(action, project, extra_params = {})
|
2017-01-23 20:40:25 +00:00
|
|
|
if action
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
get action, params: { author_id: project.creator.id }.merge(extra_params)
|
2017-01-23 20:40:25 +00:00
|
|
|
else
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
get :index, params: { namespace_id: project.namespace, project_id: project }.merge(extra_params)
|
2017-01-23 20:40:25 +00:00
|
|
|
end
|
2018-04-24 11:06:05 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def create_issuable(issuable_type, project, source_branch:)
|
|
|
|
|
if issuable_type == :issue
|
|
|
|
|
create(issuable_type, project: project, author: project.creator)
|
|
|
|
|
else
|
|
|
|
|
create(issuable_type, source_project: project, source_branch: source_branch, author: project.creator)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
let!(:issuables) do
|
|
|
|
|
%w[fix improve/awesome].map do |source_branch|
|
|
|
|
|
create_issuable(issuable_type, project, source_branch: source_branch)
|
2018-04-24 11:06:05 +00:00
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
let(:issuable_ids) { issuables.map(&:id) }
|
|
|
|
|
|
2018-04-24 11:06:05 +00:00
|
|
|
it "creates indexed meta-data object for issuable notes and votes count" do
|
|
|
|
|
get_action(action, project)
|
2017-01-23 20:40:25 +00:00
|
|
|
|
|
|
|
|
meta_data = assigns(:issuable_meta_data)
|
|
|
|
|
|
2017-08-01 16:55:57 +00:00
|
|
|
aggregate_failures do
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
expect(meta_data.keys).to match_array(issuables.map(&:id))
|
2017-08-01 16:55:57 +00:00
|
|
|
expect(meta_data.values).to all(be_kind_of(Issuable::IssuableMeta))
|
2017-01-23 20:40:25 +00:00
|
|
|
end
|
|
|
|
|
end
|
2017-04-04 10:54:58 +00:00
|
|
|
|
Extend CTE search optimisation to projects
When we use the `search` param on an `IssuableFinder`, we can run into
issues. We have trigram indexes to support these searches. On
GitLab.com, we often see Postgres's optimiser prioritise the (global)
trigram indexes over the index on `project_id`. For group and project
searches, we know that it will be quicker to filter by `project_id`
first, as it returns fewer rows in most cases.
For group issues search, we ran into this issue previously, and went
through the following iterations:
1. Use a CTE on the project IDs as an optimisation fence. This prevents
the planner from disregarding the index on `project_id`.
Unfortunately it breaks some types of sorting, like priority and
popularity, as they sort on a joined table.
2. Use a subquery for listing issues, and a CTE for counts. The subquery
- in the case of group lists - didn't help as much as the CTE, but
was faster than not including it. We can safely use a CTE for counts
as they don't have sorting.
Now, however, we're seeing the same issue in a project context. The
subquery doesn't help at all there (it would only return one row, after
all). In an attempt to keep total code complexity under control, this
commit removes the subquery optimisation and applies the CTE
optimisation only for sorts we know that are safe.
This means that for more complicated sorts (like priority and
popularity), the search will continue to be very slow. If this is a
high-priority issue, we can consider introducing further optimisations,
but this finder is already very complicated and additional complexity
has a cost.
The group CTE optimisation is controlled by the same feature flag as
before, `attempt_group_search_optimizations`, which is enabled by
default. The new project CTE optimisation is controlled by a new feature
flag, `attempt_project_search_optimizations`, which is disabled by
default.
2019-04-03 09:46:13 +00:00
|
|
|
context 'searching' do
|
|
|
|
|
let(:result_issuable) { issuables.first }
|
|
|
|
|
let(:search) { result_issuable.title }
|
|
|
|
|
|
|
|
|
|
before do
|
|
|
|
|
stub_feature_flags(attempt_project_search_optimizations: true)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# .simple_sorts is the same across all Sortable classes
|
|
|
|
|
sorts = ::Issue.simple_sorts.keys + %w[popularity priority label_priority]
|
|
|
|
|
sorts.each do |sort|
|
|
|
|
|
it "works when sorting by #{sort}" do
|
|
|
|
|
get_action(action, project, search: search, sort: sort)
|
|
|
|
|
|
|
|
|
|
expect(assigns(:issuable_meta_data).keys).to include(result_issuable.id)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
2018-04-24 11:06:05 +00:00
|
|
|
it "avoids N+1 queries" do
|
|
|
|
|
control = ActiveRecord::QueryRecorder.new { get_action(action, project) }
|
|
|
|
|
issuable = create_issuable(issuable_type, project, source_branch: 'csv')
|
|
|
|
|
|
|
|
|
|
if issuable_type == :merge_request
|
|
|
|
|
issuable.update!(source_project: fork_project(project))
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
expect { get_action(action, project) }.not_to exceed_query_limit(control.count)
|
|
|
|
|
end
|
|
|
|
|
|
2017-04-04 10:54:58 +00:00
|
|
|
describe "when given empty collection" do
|
2017-08-02 19:55:11 +00:00
|
|
|
let(:project2) { create(:project, :public) }
|
2017-04-04 10:54:58 +00:00
|
|
|
|
|
|
|
|
it "doesn't execute any queries with false conditions" do
|
2018-04-24 11:06:05 +00:00
|
|
|
get_empty =
|
2017-04-04 10:54:58 +00:00
|
|
|
if action
|
2018-12-17 22:52:17 +00:00
|
|
|
proc { get action, params: { author_id: project.creator.id } }
|
2017-04-04 10:54:58 +00:00
|
|
|
else
|
2018-12-17 22:52:17 +00:00
|
|
|
proc { get :index, params: { namespace_id: project2.namespace, project_id: project2 } }
|
2017-04-04 10:54:58 +00:00
|
|
|
end
|
|
|
|
|
|
2018-04-24 11:06:05 +00:00
|
|
|
expect(&get_empty).not_to make_queries_matching(/WHERE (?:1=0|0=1)/)
|
2017-04-04 10:54:58 +00:00
|
|
|
end
|
|
|
|
|
end
|
2017-01-23 20:40:25 +00:00
|
|
|
end
|