From f8aee5b0866df2a58522162cb348824d7e1fb3f0 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Wed, 25 Apr 2018 16:42:49 +0200 Subject: [PATCH] Add keyset pagination for API calls. Closes #45756. --- lib/api/helpers/pagination.rb | 170 ++++++++++++++++++- spec/lib/api/helpers/pagination_spec.rb | 212 +++++++++++++++++++++++- 2 files changed, 369 insertions(+), 13 deletions(-) diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index cf9501c31fe..7f5f46f66be 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -2,15 +2,175 @@ module API module Helpers module Pagination def paginate(relation) - DefaultPaginationStrategy.new(self).paginate(relation) + strategy = if params[:pagination] == 'keyset' + KeysetPaginationStrategy + else + DefaultPaginationStrategy + end + + strategy.new(self).paginate(relation) + end + + class KeysetPaginationInfo + attr_reader :relation, :request_context + + def initialize(relation, request_context) + # This is because it's rather complex to support multiple values with possibly different sort directions + # (and we don't need this in the API) + if relation.order_values.size > 1 + raise "Pagination only supports ordering by a single column." \ + "The following columns were given: #{relation.order_values.map { |v| v.expr.name }}" + end + + @relation = relation + @request_context = request_context + end + + def fields + keys.zip(values).reject { |_, v| v.nil? }.to_h + end + + def column_for_order_by(relation) + relation.order_values.first&.expr&.name + end + + # Sort direction (`:asc` or `:desc`) + def sort + @sort ||= if order_by_primary_key? + # Default order is by id DESC + :desc + else + # API defaults to DESC order if param `sort` not present + request_context.params[:sort]&.to_sym || :desc + end + end + + # Do we only sort by primary key? + def order_by_primary_key? + keys.size == 1 && keys.first == primary_key + end + + def primary_key + relation.model.primary_key.to_sym + end + + def sort_ascending? + sort == :asc + end + + # Build hash of request parameters for a given record (relevant to pagination) + def params_for(record) + return {} unless record + + keys.each_with_object({}) do |key, h| + h["ks_prev_#{key}".to_sym] = record.attributes[key.to_s] + end + end + + private + + # All values present in request parameters that correspond to #keys. + def values + @values ||= keys.map do |key| + request_context.params["ks_prev_#{key}".to_sym] + end + end + + # All keys relevant to pagination. + # This always includes the primary key. Optionally, the `order_by` key is prepended. + def keys + @keys ||= [column_for_order_by(relation), primary_key].compact.uniq + end + end + + class KeysetPaginationStrategy + attr_reader :request_context + delegate :params, :header, :request, to: :request_context + + def initialize(request_context) + @request_context = request_context + end + + def paginate(relation) + pagination = KeysetPaginationInfo.new(relation, request_context) + + paged_relation = relation.limit(per_page) + + if conds = conditions(pagination) + paged_relation = paged_relation.where(*conds) + end + + # In all cases: sort by primary key (possibly in addition to another sort column) + paged_relation = paged_relation.order(pagination.primary_key => pagination.sort) + + add_default_pagination_headers + + if last_record = paged_relation.last + next_page_params = pagination.params_for(last_record) + add_navigation_links(next_page_params) + end + + paged_relation + end + + private + + def conditions(pagination) + fields = pagination.fields + + return nil if fields.empty? + + placeholder = fields.map { '?' } + + comp = if pagination.sort_ascending? + '>' + else + '<' + end + + [ + # Row value comparison: + # (A, B) < (a, b) <=> (A < a) OR (A = a AND B < b) + # <=> A <= a AND ((A < a) OR (A = a AND B < b)) + "(#{fields.keys.join(',')}) #{comp} (#{placeholder.join(',')})", + *fields.values + ] + end + + def per_page + params[:per_page] + end + + def add_default_pagination_headers + header 'X-Per-Page', per_page.to_s + end + + def add_navigation_links(next_page_params) + header 'X-Next-Page', page_href(next_page_params) + header 'Link', link_for('next', next_page_params) + end + + def page_href(next_page_params) + request_url = request.url.split('?').first + request_params = params.dup + request_params[:per_page] = per_page + + request_params.merge!(next_page_params) if next_page_params + + "#{request_url}?#{request_params.to_query}" + end + + def link_for(rel, next_page_params) + %(<#{page_href(next_page_params)}>; rel="#{rel}") + end end class DefaultPaginationStrategy - attr_reader :ctx - delegate :params, :header, :request, to: :ctx + attr_reader :request_context + delegate :params, :header, :request, to: :request_context - def initialize(ctx) - @ctx = ctx + def initialize(request_context) + @request_context = request_context end def paginate(relation) diff --git a/spec/lib/api/helpers/pagination_spec.rb b/spec/lib/api/helpers/pagination_spec.rb index a547988d631..c73c6023b60 100644 --- a/spec/lib/api/helpers/pagination_spec.rb +++ b/spec/lib/api/helpers/pagination_spec.rb @@ -7,7 +7,203 @@ describe API::Helpers::Pagination do Class.new.include(described_class).new end - describe '#paginate' do + describe '#paginate (keyset pagination)' do + let(:value) { spy('return value') } + + before do + allow(value).to receive(:to_query).and_return(value) + + allow(subject).to receive(:header).and_return(value) + allow(subject).to receive(:params).and_return(value) + allow(subject).to receive(:request).and_return(value) + end + + context 'when resource can be paginated' do + let!(:projects) do + [ + create(:project, name: 'One'), + create(:project, name: 'Two'), + create(:project, name: 'Three') + ].sort_by { |e| -e.id } # sort by id desc (this is the default sort order for the API) + end + + describe 'first page' do + before do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', per_page: 2 }) + end + + it 'returns appropriate amount of resources' do + expect(subject.paginate(resource).count).to eq 2 + end + + it 'returns the first two records (by id desc)' do + expect(subject.paginate(resource)).to eq(projects[0..1]) + end + + it 'adds appropriate headers' do + expect_header('X-Per-Page', '2') + expect_header('X-Next-Page', "#{value}?ks_prev_id=#{projects[1].id}&pagination=keyset&per_page=2") + + expect_header('Link', anything) do |_key, val| + expect(val).to include('rel="next"') + end + + subject.paginate(resource) + end + end + + describe 'second page' do + before do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', per_page: 2, ks_prev_id: projects[1].id }) + end + + it 'returns appropriate amount of resources' do + expect(subject.paginate(resource).count).to eq 1 + end + + it 'returns the third record' do + expect(subject.paginate(resource)).to eq(projects[2..2]) + end + + it 'adds appropriate headers' do + expect_header('X-Per-Page', '2') + expect_header('X-Next-Page', "#{value}?ks_prev_id=#{projects[2].id}&pagination=keyset&per_page=2") + + expect_header('Link', anything) do |_key, val| + expect(val).to include('rel="next"') + end + + subject.paginate(resource) + end + end + + describe 'third page' do + before do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', per_page: 2, ks_prev_id: projects[2].id }) + end + + it 'returns appropriate amount of resources' do + expect(subject.paginate(resource).count).to eq 0 + end + + it 'adds appropriate headers' do + expect_header('X-Per-Page', '2') + expect(subject).not_to receive(:header).with('Link') + + subject.paginate(resource) + end + end + + context 'if order' do + context 'is not present' do + before do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', per_page: 2 }) + end + + it 'is not present it adds default order(:id) desc' do + resource.order_values = [] + + paginated_relation = subject.paginate(resource) + + expect(resource.order_values).to be_empty + expect(paginated_relation.order_values).to be_present + expect(paginated_relation.order_values.size).to eq(1) + expect(paginated_relation.order_values.first).to be_descending + expect(paginated_relation.order_values.first.expr.name).to eq :id + end + end + + context 'is present' do + let(:resource) { Project.all.order(name: :desc) } + let!(:projects) do + [ + create(:project, name: 'One'), + create(:project, name: 'Two'), + create(:project, name: 'Three'), + create(:project, name: 'Three'), # Note the duplicate name + create(:project, name: 'Four'), + create(:project, name: 'Five'), + create(:project, name: 'Six') + ] + + # if we sort this by name descending, id descending, this yields: + # { + # 2 => "Two", + # 4 => "Three", + # 3 => "Three", + # 7 => "Six", + # 1 => "One", + # 5 => "Four", + # 6 => "Five" + # } + # + # (key is the id) + end + + it 'it also orders by primary key' do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', per_page: 2 }) + paginated_relation = subject.paginate(resource) + + expect(paginated_relation.order_values).to be_present + expect(paginated_relation.order_values.size).to eq(2) + expect(paginated_relation.order_values.first).to be_descending + expect(paginated_relation.order_values.first.expr.name).to eq :name + expect(paginated_relation.order_values.second).to be_descending + expect(paginated_relation.order_values.second.expr.name).to eq :id + end + + it 'it returns the right records (first page)' do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', per_page: 2 }) + result = subject.paginate(resource) + + expect(result.first).to eq(projects[1]) + expect(result.second).to eq(projects[3]) + end + + it 'it returns the right records (second page)' do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', ks_prev_id: projects[3].id, ks_prev_name: projects[3].name, per_page: 2 }) + result = subject.paginate(resource) + + expect(result.first).to eq(projects[2]) + expect(result.second).to eq(projects[6]) + end + + it 'it returns the right records (third page), note increased per_page' do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', ks_prev_id: projects[6].id, ks_prev_name: projects[6].name, per_page: 5 }) + result = subject.paginate(resource) + + expect(result.size).to eq(3) + expect(result.first).to eq(projects[0]) + expect(result.second).to eq(projects[4]) + expect(result.last).to eq(projects[5]) + end + + it 'it returns the right link to the next page' do + allow(subject).to receive(:params) + .and_return({ pagination: 'keyset', ks_prev_id: projects[3].id, ks_prev_name: projects[3].name, per_page: 2 }) + expect_header('X-Per-Page', '2') + expect_header('X-Next-Page', "#{value}?ks_prev_id=#{projects[6].id}&ks_prev_name=#{projects[6].name}&pagination=keyset&per_page=2") + + expect_header('Link', anything) do |_key, val| + expect(val).to include('rel="next"') + end + + subject.paginate(resource) + end + end + end + end + end + + describe '#paginate (default offset-based pagination)' do let(:value) { spy('return value') } before do @@ -146,14 +342,14 @@ describe API::Helpers::Pagination do end end end + end - def expect_header(*args, &block) - expect(subject).to receive(:header).with(*args, &block) - end + def expect_header(*args, &block) + expect(subject).to receive(:header).with(*args, &block) + end - def expect_message(method) - expect(subject).to receive(method) - .at_least(:once).and_return(value) - end + def expect_message(method) + expect(subject).to receive(method) + .at_least(:once).and_return(value) end end