276 lines
11 KiB
Ruby
276 lines
11 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Gitlab
|
|
module Pagination
|
|
module Keyset
|
|
# This class is a special ORDER BY clause which is compatible with ActiveRecord. It helps
|
|
# building keyset paginated queries.
|
|
#
|
|
# In ActiveRecord we use the `order()` method which will generate the `ORDER BY X` SQL clause
|
|
#
|
|
# Project.where(active: true).order(id: :asc)
|
|
#
|
|
# # Or
|
|
#
|
|
# Project.where(active: true).order(created_at: :asc, id: desc)
|
|
#
|
|
# Gitlab::Pagination::Keyset::Order class encapsulates more information about the order columns
|
|
# in order to implement keyset pagination in a generic way
|
|
#
|
|
# - Extract values from a record (usually the last item of the previous query)
|
|
# - Build query conditions based on the column configuration
|
|
#
|
|
# Example 1: Order by primary key
|
|
#
|
|
# # Simple order definition for the primary key as an ActiveRecord scope
|
|
# scope :id_asc_ordered, -> {
|
|
# keyset_order = Gitlab::Pagination::Keyset::Order.build([
|
|
# Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
|
|
# attribute: :id,
|
|
# order_expression: Project.arel_table[:id].asc
|
|
# )
|
|
# ])
|
|
#
|
|
# reorder(keyset_order)
|
|
# }
|
|
#
|
|
# # ... Later in the application code:
|
|
#
|
|
# # Compatible with ActiveRecord's `order()` method
|
|
# page1 = Project.where(active: true).id_asc_ordered.limit(5)
|
|
# keyset_order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(page1)
|
|
#
|
|
# last_record = page1.last
|
|
# cursor_values = keyset_order.cursor_attributes_for_node(last_record) # { id: x }
|
|
#
|
|
# page2 = keyset_order.apply_cursor_conditions(Project.where(active: true).id_asc_ordered, cursor_values).limit(5)
|
|
#
|
|
# last_record = page2.last
|
|
# cursor_values = keyset_order.cursor_attributes_for_node(last_record)
|
|
#
|
|
# page3 = keyset_order.apply_cursor_conditions(Project.where(active: true).id_asc_ordered, cursor_values).limit(5)
|
|
#
|
|
# Example 2: Order by creation time and primary key (primary key is the tie breaker)
|
|
#
|
|
# scope :created_at_ordered, -> {
|
|
# keyset_order = Gitlab::Pagination::Keyset::Order.build([
|
|
# Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
|
|
# attribute_name: :created_at,
|
|
# column_expression: Project.arel_table[:created_at],
|
|
# order_expression: Project.arel_table[:created_at].asc,
|
|
# distinct: false, # values in the column are not unique
|
|
# nullable: :nulls_last # we might see NULL values (bottom)
|
|
# ),
|
|
# Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
|
|
# attribute_name: :id,
|
|
# order_expression: Project.arel_table[:id].asc
|
|
# )
|
|
# ])
|
|
#
|
|
# reorder(keyset_order)
|
|
# }
|
|
#
|
|
class Order < Arel::Nodes::SqlLiteral
|
|
attr_reader :column_definitions
|
|
|
|
def initialize(column_definitions:)
|
|
@column_definitions = column_definitions
|
|
|
|
super(to_sql_literal(@column_definitions))
|
|
end
|
|
|
|
# Tells whether the given ActiveRecord::Relation has keyset ordering
|
|
def self.keyset_aware?(scope)
|
|
scope.order_values.first.is_a?(self) && scope.order_values.one?
|
|
end
|
|
|
|
def self.extract_keyset_order_object(scope)
|
|
scope.order_values.first
|
|
end
|
|
|
|
def self.build(column_definitions)
|
|
new(column_definitions: column_definitions)
|
|
end
|
|
|
|
def cursor_attributes_for_node(node)
|
|
column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash|
|
|
field_value = node[column_definition.attribute_name]
|
|
hash[column_definition.attribute_name] = if field_value.is_a?(Time)
|
|
field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z')
|
|
elsif field_value.nil?
|
|
nil
|
|
elsif lower_named_function?(column_definition)
|
|
field_value.downcase
|
|
else
|
|
field_value.to_s
|
|
end
|
|
end
|
|
end
|
|
|
|
# This methods builds the conditions for the keyset pagination
|
|
#
|
|
# Example:
|
|
#
|
|
# |created_at|id|
|
|
# |----------|--|
|
|
# |2020-01-01| 1|
|
|
# | null| 2|
|
|
# | null| 3|
|
|
# |2020-02-01| 4|
|
|
#
|
|
# Note: created_at is not distinct and nullable
|
|
# Order `ORDER BY created_at DESC, id DESC`
|
|
#
|
|
# We get the following cursor values from the previous page:
|
|
# { id: 4, created_at: '2020-02-01' }
|
|
#
|
|
# To get the next rows, we need to build the following conditions:
|
|
#
|
|
# (created_at = '2020-02-01' AND id < 4) OR (created_at < '2020-01-01')
|
|
#
|
|
# DESC ordering ensures that NULL values are on top so we don't need conditions for NULL values
|
|
#
|
|
# Another cursor example:
|
|
# { id: 3, created_at: nil }
|
|
#
|
|
# To get the next rows, we need to build the following conditions:
|
|
#
|
|
# (id < 3 AND created_at IS NULL) OR (created_at IS NOT NULL)
|
|
def build_where_values(values)
|
|
return [] if values.blank?
|
|
|
|
verify_incoming_values!(values)
|
|
|
|
return use_composite_row_comparison(values) if composite_row_comparison_possible?
|
|
|
|
column_definitions
|
|
.map { ColumnConditionBuilder.new(_1, values[_1.attribute_name]) }
|
|
.reverse
|
|
.reduce([]) { |where_conditions, column| column.where_conditions(where_conditions) }
|
|
end
|
|
|
|
def where_values_with_or_query(values)
|
|
build_or_query(build_where_values(values.with_indifferent_access))
|
|
end
|
|
|
|
# rubocop: disable CodeReuse/ActiveRecord
|
|
def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false, in_operator_optimization_options: nil })
|
|
values ||= {}
|
|
transformed_values = values.with_indifferent_access
|
|
scope = apply_custom_projections(scope)
|
|
|
|
where_values = build_where_values(transformed_values)
|
|
|
|
if options[:use_union_optimization] && where_values.size > 1
|
|
build_union_query(scope, where_values).reorder(self)
|
|
elsif options[:in_operator_optimization_options]
|
|
opts = options[:in_operator_optimization_options]
|
|
|
|
Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new(
|
|
**{
|
|
scope: scope.reorder(self),
|
|
values: values
|
|
}.merge(opts)
|
|
).execute
|
|
else
|
|
scope.where(build_or_query(where_values)) # rubocop: disable CodeReuse/ActiveRecord
|
|
end
|
|
end
|
|
# rubocop: enable CodeReuse/ActiveRecord
|
|
|
|
def reversed_order
|
|
self.class.build(column_definitions.map(&:reverse))
|
|
end
|
|
|
|
alias_method :to_sql, :to_s
|
|
|
|
private
|
|
|
|
def lower_named_function?(column_definition)
|
|
column_definition.column_expression.is_a?(Arel::Nodes::NamedFunction) && column_definition.column_expression.name&.downcase == 'lower'
|
|
end
|
|
|
|
def composite_row_comparison_possible?
|
|
!column_definitions.one? &&
|
|
column_definitions.all?(&:not_nullable?) &&
|
|
column_definitions.map(&:order_direction).uniq.one? # all columns uses the same order direction
|
|
end
|
|
|
|
# composite row comparison works with NOT NULL columns and may use only one index scan given a proper index setup
|
|
# Example: (created_at, id) > ('2012-09-18 01:40:01+00', 15)
|
|
def use_composite_row_comparison(values)
|
|
columns = Arel::Nodes::Grouping.new(column_definitions.map(&:column_expression))
|
|
values = Arel::Nodes::Grouping.new(column_definitions.map do |column_definition|
|
|
value = values[column_definition.attribute_name]
|
|
build_quoted(value, column_definition.column_expression)
|
|
end)
|
|
|
|
if column_definitions.first.ascending_order?
|
|
[columns.gt(values)]
|
|
else
|
|
[columns.lt(values)]
|
|
end
|
|
end
|
|
|
|
def build_quoted(value, column_expression)
|
|
return value if value.instance_of?(Arel::Nodes::SqlLiteral)
|
|
|
|
Arel::Nodes.build_quoted(value, column_expression)
|
|
end
|
|
|
|
# Adds extra columns to the SELECT clause
|
|
def apply_custom_projections(scope)
|
|
additional_projections = column_definitions.select(&:add_to_projections).map do |column_definition|
|
|
# avoid mutating the original column_expression
|
|
column_definition.column_expression.dup.as(column_definition.attribute_name).to_sql
|
|
end
|
|
|
|
scope = scope.reselect(*scope.arel.projections, *additional_projections) unless additional_projections.blank?
|
|
scope
|
|
end
|
|
|
|
def build_or_query(expressions)
|
|
return [] if expressions.blank?
|
|
|
|
or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) }
|
|
Arel::Nodes::Grouping.new(or_expression)
|
|
end
|
|
|
|
def build_union_query(scope, where_values)
|
|
scopes = where_values.map do |where_value|
|
|
scope.dup.where(where_value).reorder(self) # rubocop: disable CodeReuse/ActiveRecord
|
|
end
|
|
scope.model.from_union(scopes, remove_duplicates: false, remove_order: false)
|
|
end
|
|
|
|
def to_sql_literal(column_definitions)
|
|
column_definitions.map do |column_definition|
|
|
if column_definition.order_expression.respond_to?(:to_sql)
|
|
column_definition.order_expression.to_sql
|
|
else
|
|
column_definition.order_expression.to_s
|
|
end
|
|
end.join(', ')
|
|
end
|
|
|
|
def verify_incoming_values!(values)
|
|
value_keys = values.keys.map(&:to_s)
|
|
order_attrbute_names = column_definitions.map(&:attribute_name).map(&:to_s)
|
|
missing_items = order_attrbute_names - value_keys
|
|
extra_items = value_keys - order_attrbute_names
|
|
|
|
if missing_items.any? || extra_items.any?
|
|
error_text = ['Incorrect cursor values were given']
|
|
|
|
error_text << "Extra items: #{extra_items.join(', ')}" if extra_items.any?
|
|
error_text << "Missing items: #{missing_items.join(', ')}" if missing_items.any?
|
|
|
|
error_text.compact
|
|
|
|
raise error_text.join('. ')
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|