Move cycle analytics calculations to SQL.
1. Use Arel for composable queries. 2. For a project with ~10k issues, the page loads in around 600ms. Previously, a project with ~5k issues would have a ~20s page load time.
This commit is contained in:
parent
161804bf40
commit
7d69ff3ddf
|
@ -0,0 +1,54 @@
|
||||||
|
module DatabaseMedian
|
||||||
|
extend ActiveSupport::Concern
|
||||||
|
|
||||||
|
def median_datetime(arel_table, query_so_far, column_sym)
|
||||||
|
# TODO: MySQL
|
||||||
|
pg_median_datetime(arel_table, query_so_far, column_sym)
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
# https://www.periscopedata.com/blog/medians-in-sql.html
|
||||||
|
def pg_median_datetime(arel_table, query_so_far, column_sym)
|
||||||
|
# Create a CTE with the column we're operating on, row number (after sorting by the column
|
||||||
|
# we're operating on), and count of the table we're operating on (duplicated across) all rows
|
||||||
|
# of the CTE. For example, if we're looking to find the median of the `projects.star_count`
|
||||||
|
# column, the CTE might look like this:
|
||||||
|
#
|
||||||
|
# star_count | row_id | ct
|
||||||
|
# ------------+--------+----
|
||||||
|
# 5 | 1 | 3
|
||||||
|
# 9 | 2 | 3
|
||||||
|
# 15 | 3 | 3
|
||||||
|
cte_table = Arel::Table.new(("ordered_records"))
|
||||||
|
cte = Arel::Nodes::As.new(cte_table,
|
||||||
|
arel_table.
|
||||||
|
project(arel_table[column_sym].as(column_sym.to_s),
|
||||||
|
Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []),
|
||||||
|
Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'),
|
||||||
|
arel_table.project("COUNT(1)").as('ct')).
|
||||||
|
# Disallow negative values
|
||||||
|
where(arel_table[column_sym].gteq(zero_interval)))
|
||||||
|
|
||||||
|
# From the CTE, select either the middle row or the middle two rows (this is accomplished
|
||||||
|
# by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the
|
||||||
|
# selected rows, and this is the median value.
|
||||||
|
cte_table.project(Arel::Nodes::NamedFunction.new("AVG",
|
||||||
|
[extract_epoch(cte_table[column_sym])],
|
||||||
|
"median")).
|
||||||
|
where(Arel::Nodes::Between.new(cte_table[:row_id],
|
||||||
|
Arel::Nodes::And.new([(cte_table[:ct] / Arel::Nodes::SqlLiteral.new('2.0')),
|
||||||
|
(cte_table[:ct] / Arel::Nodes::SqlLiteral.new('2.0') + 1)]))).
|
||||||
|
with(query_so_far, cte)
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def extract_epoch(arel_attribute)
|
||||||
|
Arel::Nodes::SqlLiteral.new("EXTRACT(EPOCH FROM \"#{arel_attribute.relation.name}\".\"#{arel_attribute.name}\")")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Need to cast '0' to an INTERVAL before we can check if the interval is positive
|
||||||
|
def zero_interval
|
||||||
|
Arel::Nodes::NamedFunction.new("CAST", [Arel::Nodes::SqlLiteral.new("'0' AS INTERVAL")])
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,9 +1,9 @@
|
||||||
class CycleAnalytics
|
class CycleAnalytics
|
||||||
attr_reader :from
|
include DatabaseMedian
|
||||||
|
|
||||||
def initialize(project, from:)
|
def initialize(project, from:)
|
||||||
|
@project = project
|
||||||
@from = from
|
@from = from
|
||||||
@queries = Queries.new(project)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def as_json(options = {})
|
def as_json(options = {})
|
||||||
|
@ -14,65 +14,94 @@ class CycleAnalytics
|
||||||
end
|
end
|
||||||
|
|
||||||
def issue
|
def issue
|
||||||
calculate_metric(@queries.issues(created_after: @from),
|
calculate_metric!(:issue,
|
||||||
-> (data_point) { data_point[:issue].created_at },
|
TableReferences.issues[:created_at],
|
||||||
[@queries.issue_first_associated_with_milestone_at, @queries.issue_first_added_to_list_label_at])
|
[TableReferences.issue_metrics[:first_associated_with_milestone_at],
|
||||||
|
TableReferences.issue_metrics[:first_added_to_board_at]],
|
||||||
|
load_merge_requests: false)
|
||||||
end
|
end
|
||||||
|
|
||||||
def plan
|
def plan
|
||||||
calculate_metric(@queries.issues(created_after: @from),
|
calculate_metric!(:plan,
|
||||||
[@queries.issue_first_associated_with_milestone_at, @queries.issue_first_added_to_list_label_at],
|
[TableReferences.issue_metrics[:first_associated_with_milestone_at],
|
||||||
@queries.issue_first_mentioned_in_commit_at)
|
TableReferences.issue_metrics[:first_added_to_board_at]],
|
||||||
|
TableReferences.issue_metrics[:first_mentioned_in_commit_at],
|
||||||
|
load_merge_requests: false)
|
||||||
end
|
end
|
||||||
|
|
||||||
def code
|
def code
|
||||||
calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
|
calculate_metric!(:code,
|
||||||
@queries.issue_first_mentioned_in_commit_at,
|
TableReferences.issue_metrics[:first_mentioned_in_commit_at],
|
||||||
-> (data_point) { data_point[:merge_request].created_at })
|
TableReferences.merge_requests[:created_at],
|
||||||
|
load_merge_requests: true)
|
||||||
end
|
end
|
||||||
|
|
||||||
def test
|
def test
|
||||||
calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
|
calculate_metric!(:test,
|
||||||
@queries.merge_request_build_started_at,
|
TableReferences.merge_request_metrics[:latest_build_started_at],
|
||||||
@queries.merge_request_build_finished_at)
|
TableReferences.merge_request_metrics[:latest_build_finished_at],
|
||||||
|
load_merge_requests: true)
|
||||||
end
|
end
|
||||||
|
|
||||||
def review
|
def review
|
||||||
calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
|
calculate_metric!(:review,
|
||||||
-> (data_point) { data_point[:merge_request].created_at },
|
TableReferences.merge_requests[:created_at],
|
||||||
@queries.merge_request_merged_at)
|
TableReferences.merge_request_metrics[:merged_at],
|
||||||
|
load_merge_requests: true)
|
||||||
end
|
end
|
||||||
|
|
||||||
def staging
|
def staging
|
||||||
calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
|
calculate_metric!(:staging,
|
||||||
@queries.merge_request_merged_at,
|
TableReferences.merge_request_metrics[:merged_at],
|
||||||
@queries.merge_request_deployed_to_production_at)
|
TableReferences.merge_request_metrics[:first_deployed_to_production_at],
|
||||||
|
load_merge_requests: true)
|
||||||
end
|
end
|
||||||
|
|
||||||
def production
|
def production
|
||||||
calculate_metric(@queries.merge_requests_closing_issues(created_after: @from),
|
calculate_metric!(:production,
|
||||||
-> (data_point) { data_point[:issue].created_at },
|
TableReferences.issues[:created_at],
|
||||||
@queries.merge_request_deployed_to_production_at)
|
TableReferences.merge_request_metrics[:first_deployed_to_production_at],
|
||||||
|
load_merge_requests: true)
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def calculate_metric(data, start_time_fns, end_time_fns)
|
def calculate_metric!(name, start_time_attrs, end_time_attrs, load_merge_requests: false)
|
||||||
times = data.map do |data_point|
|
cte_table = Arel::Table.new("cte_table_for_#{name}")
|
||||||
start_time = Array.wrap(start_time_fns).map { |fn| fn[data_point] }.compact.first
|
|
||||||
end_time = Array.wrap(end_time_fns).map { |fn| fn[data_point] }.compact.first
|
|
||||||
|
|
||||||
if start_time.present? && end_time.present? && end_time >= start_time
|
# Add a `SELECT` for (end_time - start-time), and add an alias for it.
|
||||||
end_time - start_time
|
# Note: We use COALESCE to pick up the first non-null column for end_time / start_time.
|
||||||
end
|
query = Arel::Nodes::As.new(
|
||||||
|
cte_table,
|
||||||
|
base_query(load_merge_requests: load_merge_requests).project(
|
||||||
|
Arel::Nodes::Subtraction.new(
|
||||||
|
Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)),
|
||||||
|
Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))
|
||||||
|
).as(name.to_s)))
|
||||||
|
|
||||||
|
query = median_datetime(cte_table, query, name)
|
||||||
|
median = ActiveRecord::Base.connection.execute(query.to_sql).first['median']
|
||||||
|
median.to_f if median.present?
|
||||||
end
|
end
|
||||||
|
|
||||||
median(times.compact.sort)
|
# Join table with a row for every <issue,merge_request> pair (where the merge request
|
||||||
end
|
# closes the given issue) with issue and merge request metrics included. The metrics
|
||||||
|
# are loaded with an inner join, so issues / merge requests without metrics are
|
||||||
|
# automatically excluded.
|
||||||
|
def base_query(load_merge_requests:)
|
||||||
|
arel_table = TableReferences.merge_requests_closing_issues
|
||||||
|
|
||||||
def median(coll)
|
query = arel_table.join(TableReferences.issues).on(TableReferences.issues[:id].eq(arel_table[:issue_id])).
|
||||||
return if coll.empty?
|
join(TableReferences.issue_metrics).on(TableReferences.issues[:id].eq(TableReferences.issue_metrics[:issue_id])).
|
||||||
size = coll.length
|
where(TableReferences.issues[:project_id].eq(@project.id)).
|
||||||
(coll[size / 2] + coll[(size - 1) / 2]) / 2.0
|
where(TableReferences.issues[:deleted_at].eq(nil)).
|
||||||
|
where(TableReferences.issues[:created_at].gteq(@from))
|
||||||
|
|
||||||
|
if load_merge_requests
|
||||||
|
query.join(TableReferences.merge_requests, Arel::Nodes::OuterJoin).on(TableReferences.merge_requests[:id].eq(arel_table[:merge_request_id])).
|
||||||
|
join(TableReferences.merge_request_metrics).on(TableReferences.merge_requests[:id].eq(TableReferences.merge_request_metrics[:merge_request_id]))
|
||||||
|
else
|
||||||
|
query
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,25 @@
|
||||||
|
class CycleAnalytics
|
||||||
|
module TableReferences
|
||||||
|
class << self
|
||||||
|
def issues
|
||||||
|
Issue.arel_table
|
||||||
|
end
|
||||||
|
|
||||||
|
def issue_metrics
|
||||||
|
Issue::Metrics.arel_table
|
||||||
|
end
|
||||||
|
|
||||||
|
def merge_requests
|
||||||
|
MergeRequest.arel_table
|
||||||
|
end
|
||||||
|
|
||||||
|
def merge_request_metrics
|
||||||
|
MergeRequest::Metrics.arel_table
|
||||||
|
end
|
||||||
|
|
||||||
|
def merge_requests_closing_issues
|
||||||
|
MergeRequestsClosingIssues.arel_table
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue