Skip to content

G
gitlab-ce
Commit 66d6be2c authored by Heinrich Lee Yu's avatar Heinrich Lee Yu
Browse files
Options

Merge branch 'ab/refactor-pagination' into 'master' 

Refactor pagination helper (extract classes, remove keyset pagination)

See merge request gitlab-org/gitlab!18840
parents b53fd47f 957d97d0
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 337 additions and 640 deletions
lib/api/helpers/pagination.rb
View file @ 66d6be2c
...@@ -4,254 +4,7 @@ module API ...@@ -4,254 +4,7 @@ module API
module Helpers module Helpers
module Pagination module Pagination
def paginate(relation) def paginate(relation)
strategy = if params[:pagination] == 'keyset' && Feature.enabled?('api_keyset_pagination') ::Gitlab::Pagination::OffsetPagination.new(self).paginate(relation)
KeysetPaginationStrategy
else
DefaultPaginationStrategy
end
strategy.new(self).paginate(relation)
end
class Base
private
def per_page
@per_page ||= params[:per_page]
end
def base_request_uri
@base_request_uri ||= URI.parse(request.url).tap do |uri|
uri.host = Gitlab.config.gitlab.host
uri.port = Gitlab.config.gitlab.port
end
end
def build_page_url(query_params:)
base_request_uri.tap do |uri|
uri.query = query_params
end.to_s
end
def page_href(next_page_params = {})
query_params = params.merge(**next_page_params, per_page: per_page).to_query
build_page_url(query_params: query_params)
end
end
class KeysetPaginationInfo
attr_reader :relation, :request_context
def initialize(relation, request_context)
# This is because it's rather complex to support multiple values with possibly different sort directions
# (and we don't need this in the API)
if relation.order_values.size > 1
raise "Pagination only supports ordering by a single column." \
"The following columns were given: #{relation.order_values.map { |v| v.expr.name }}"
end
@relation = relation
@request_context = request_context
end
def fields
keys.zip(values).reject { |_, v| v.nil? }.to_h
end
def column_for_order_by(relation)
relation.order_values.first&.expr&.name
end
# Sort direction (`:asc` or `:desc`)
def sort
@sort ||= if order_by_primary_key?
# Default order is by id DESC
:desc
else
# API defaults to DESC order if param `sort` not present
request_context.params[:sort]&.to_sym || :desc
end
end
# Do we only sort by primary key?
def order_by_primary_key?
keys.size == 1 && keys.first == primary_key
end
def primary_key
relation.model.primary_key.to_sym
end
def sort_ascending?
sort == :asc
end
# Build hash of request parameters for a given record (relevant to pagination)
def params_for(record)
return {} unless record
keys.each_with_object({}) do |key, h|
h["ks_prev_#{key}".to_sym] = record.attributes[key.to_s]
end
end
private
# All values present in request parameters that correspond to #keys.
def values
@values ||= keys.map do |key|
request_context.params["ks_prev_#{key}".to_sym]
end
end
# All keys relevant to pagination.
# This always includes the primary key. Optionally, the `order_by` key is prepended.
def keys
@keys ||= [column_for_order_by(relation), primary_key].compact.uniq
end
end
class KeysetPaginationStrategy < Base
attr_reader :request_context
delegate :params, :header, :request, to: :request_context
def initialize(request_context)
@request_context = request_context
end
# rubocop: disable CodeReuse/ActiveRecord
def paginate(relation)
pagination = KeysetPaginationInfo.new(relation, request_context)
paged_relation = relation.limit(per_page)
if conds = conditions(pagination)
paged_relation = paged_relation.where(*conds)
end
# In all cases: sort by primary key (possibly in addition to another sort column)
paged_relation = paged_relation.order(pagination.primary_key => pagination.sort)
add_default_pagination_headers
if last_record = paged_relation.last
next_page_params = pagination.params_for(last_record)
add_navigation_links(next_page_params)
end
paged_relation
end
# rubocop: enable CodeReuse/ActiveRecord
private
def conditions(pagination)
fields = pagination.fields
return if fields.empty?
placeholder = fields.map { '?' }
comp = if pagination.sort_ascending?
'>'
else
'<'
end
[
# Row value comparison:
# (A, B) < (a, b) <=> (A < a) OR (A = a AND B < b)
# <=> A <= a AND ((A < a) OR (A = a AND B < b))
"(#{fields.keys.join(',')}) #{comp} (#{placeholder.join(',')})",
*fields.values
]
end
def add_default_pagination_headers
header 'X-Per-Page', per_page.to_s
end
def add_navigation_links(next_page_params)
header 'X-Next-Page', page_href(next_page_params)
header 'Link', link_for('next', next_page_params)
end
def link_for(rel, next_page_params)
%(<#{page_href(next_page_params)}>; rel="#{rel}")
end
end
class DefaultPaginationStrategy < Base
attr_reader :request_context
delegate :params, :header, :request, to: :request_context
def initialize(request_context)
@request_context = request_context
end
def paginate(relation)
paginate_with_limit_optimization(add_default_order(relation)).tap do |data|
add_pagination_headers(data)
end
end
private
def paginate_with_limit_optimization(relation)
pagination_data = relation.page(params[:page]).per(params[:per_page])
return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation)
return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit)
limited_total_count = pagination_data.total_count_with_limit
if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT
# The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?`
# We need to call `reset` because `without_count` relies on `@arel` being unmemoized
pagination_data.reset.without_count
else
pagination_data
end
end
def add_default_order(relation)
if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord
end
relation
end
def add_pagination_headers(paginated_data)
header 'X-Per-Page', paginated_data.limit_value.to_s
header 'X-Page', paginated_data.current_page.to_s
header 'X-Next-Page', paginated_data.next_page.to_s
header 'X-Prev-Page', paginated_data.prev_page.to_s
header 'Link', pagination_links(paginated_data)
return if data_without_counts?(paginated_data)
header 'X-Total', paginated_data.total_count.to_s
header 'X-Total-Pages', total_pages(paginated_data).to_s
end
def pagination_links(paginated_data)
[].tap do |links|
links << %(<#{page_href(page: paginated_data.prev_page)}>; rel="prev") if paginated_data.prev_page
links << %(<#{page_href(page: paginated_data.next_page)}>; rel="next") if paginated_data.next_page
links << %(<#{page_href(page: 1)}>; rel="first")
links << %(<#{page_href(page: total_pages(paginated_data))}>; rel="last") unless data_without_counts?(paginated_data)
end.join(', ')
end
def total_pages(paginated_data)
# Ensure there is in total at least 1 page
[paginated_data.total_pages, 1].max
end
def data_without_counts?(paginated_data)
paginated_data.is_a?(Kaminari::PaginatableWithoutCount)
end
end end
end end
end end
......
lib/gitlab/pagination/base.rb 0 → 100644
View file @ 66d6be2c
# frozen_string_literal: true
module Gitlab
module Pagination
class Base
private
def per_page
@per_page ||= params[:per_page]
end
def base_request_uri
@base_request_uri ||= URI.parse(request.url).tap do |uri|
uri.host = Gitlab.config.gitlab.host
uri.port = Gitlab.config.gitlab.port
end
end
def build_page_url(query_params:)
base_request_uri.tap do |uri|
uri.query = query_params
end.to_s
end
def page_href(next_page_params = {})
query_params = params.merge(**next_page_params, per_page: per_page).to_query
build_page_url(query_params: query_params)
end
end
end
end
lib/gitlab/pagination/offset_pagination.rb 0 → 100644
View file @ 66d6be2c
# frozen_string_literal: true
module Gitlab
module Pagination
class OffsetPagination < Base
attr_reader :request_context
delegate :params, :header, :request, to: :request_context
def initialize(request_context)
@request_context = request_context
end
def paginate(relation)
paginate_with_limit_optimization(add_default_order(relation)).tap do |data|
add_pagination_headers(data)
end
end
private
def paginate_with_limit_optimization(relation)
pagination_data = relation.page(params[:page]).per(params[:per_page])
return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation)
return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit)
limited_total_count = pagination_data.total_count_with_limit
if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT
# The call to `total_count_with_limit` memoizes `@arel` because of a call to `references_eager_loaded_tables?`
# We need to call `reset` because `without_count` relies on `@arel` being unmemoized
pagination_data.reset.without_count
else
pagination_data
end
end
def add_default_order(relation)
if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord
end
relation
end
def add_pagination_headers(paginated_data)
header 'X-Per-Page', paginated_data.limit_value.to_s
header 'X-Page', paginated_data.current_page.to_s
header 'X-Next-Page', paginated_data.next_page.to_s
header 'X-Prev-Page', paginated_data.prev_page.to_s
header 'Link', pagination_links(paginated_data)
return if data_without_counts?(paginated_data)
header 'X-Total', paginated_data.total_count.to_s
header 'X-Total-Pages', total_pages(paginated_data).to_s
end
def pagination_links(paginated_data)
[].tap do |links|
links << %(<#{page_href(page: paginated_data.prev_page)}>; rel="prev") if paginated_data.prev_page
links << %(<#{page_href(page: paginated_data.next_page)}>; rel="next") if paginated_data.next_page
links << %(<#{page_href(page: 1)}>; rel="first")
links << %(<#{page_href(page: total_pages(paginated_data))}>; rel="last") unless data_without_counts?(paginated_data)
end.join(', ')
end
def total_pages(paginated_data)
# Ensure there is in total at least 1 page
[paginated_data.total_pages, 1].max
end
def data_without_counts?(paginated_data)
paginated_data.is_a?(Kaminari::PaginatableWithoutCount)
end
end
end
end
lib/gitlab/serializer/pagination.rb
View file @ 66d6be2c
...@@ -4,7 +4,6 @@ module Gitlab ...@@ -4,7 +4,6 @@ module Gitlab
module Serializer module Serializer
class Pagination class Pagination
InvalidResourceError = Class.new(StandardError) InvalidResourceError = Class.new(StandardError)
include ::API::Helpers::Pagination
def initialize(request, response) def initialize(request, response)
@request = request @request = request
...@@ -13,13 +12,13 @@ module Gitlab ...@@ -13,13 +12,13 @@ module Gitlab
def paginate(resource) def paginate(resource)
if resource.respond_to?(:page) if resource.respond_to?(:page)
super(resource) ::Gitlab::Pagination::OffsetPagination.new(self).paginate(resource)
else else
raise InvalidResourceError raise InvalidResourceError
end end
end end
# Methods needed by `API::Helpers::Pagination` # Methods needed by `Gitlab::Pagination::OffsetPagination`
# #
attr_reader :request attr_reader :request
......
spec/lib/api/helpers/pagination_spec.rb
View file @ 66d6be2c
This diff is collapsed.
spec/lib/gitlab/pagination/offset_pagination_spec.rb 0 → 100644
View file @ 66d6be2c
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Pagination::OffsetPagination do
let(:resource) { Project.all }
let(:custom_port) { 8080 }
let(:incoming_api_projects_url) { "#{Gitlab.config.gitlab.url}:#{custom_port}/api/v4/projects" }
before do
stub_config_setting(port: custom_port)
end
let(:request_context) { double("request_context") }
subject do
described_class.new(request_context)
end
describe '#paginate' do
let(:value) { spy('return value') }
let(:base_query) { { foo: 'bar', bar: 'baz' } }
let(:query) { base_query }
before do
allow(request_context).to receive(:header).and_return(value)
allow(request_context).to receive(:params).and_return(query)
allow(request_context).to receive(:request).and_return(double(url: "#{incoming_api_projects_url}?#{query.to_query}"))
end
context 'when resource can be paginated' do
before do
create_list(:project, 3)
end
describe 'first page' do
shared_examples 'response with pagination headers' do
it 'adds appropriate headers' do
expect_header('X-Total', '3')
expect_header('X-Total-Pages', '2')
expect_header('X-Per-Page', '2')
expect_header('X-Page', '1')
expect_header('X-Next-Page', '2')
expect_header('X-Prev-Page', '')
expect_header('Link', anything) do |_key, val|
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="last"))
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next"))
expect(val).not_to include('rel="prev"')
end
subject.paginate(resource)
end
end
shared_examples 'paginated response' do
it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 2
end
it 'executes only one SELECT COUNT query' do
expect { subject.paginate(resource) }.to make_queries_matching(/SELECT COUNT/, 1)
end
end
let(:query) { base_query.merge(page: 1, per_page: 2) }
context 'when the api_kaminari_count_with_limit feature flag is unset' do
it_behaves_like 'paginated response'
it_behaves_like 'response with pagination headers'
end
context 'when the api_kaminari_count_with_limit feature flag is disabled' do
before do
stub_feature_flags(api_kaminari_count_with_limit: false)
end
it_behaves_like 'paginated response'
it_behaves_like 'response with pagination headers'
end
context 'when the api_kaminari_count_with_limit feature flag is enabled' do
before do
stub_feature_flags(api_kaminari_count_with_limit: true)
end
context 'when resources count is less than MAX_COUNT_LIMIT' do
before do
stub_const("::Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT", 4)
end
it_behaves_like 'paginated response'
it_behaves_like 'response with pagination headers'
end
context 'when resources count is more than MAX_COUNT_LIMIT' do
before do
stub_const("::Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT", 2)
end
it_behaves_like 'paginated response'
it 'does not return the X-Total and X-Total-Pages headers' do
expect_no_header('X-Total')
expect_no_header('X-Total-Pages')
expect_header('X-Per-Page', '2')
expect_header('X-Page', '1')
expect_header('X-Next-Page', '2')
expect_header('X-Prev-Page', '')
expect_header('Link', anything) do |_key, val|
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next"))
expect(val).not_to include('rel="last"')
expect(val).not_to include('rel="prev"')
end
subject.paginate(resource)
end
end
end
end
describe 'second page' do
let(:query) { base_query.merge(page: 2, per_page: 2) }
it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 1
end
it 'adds appropriate headers' do
expect_header('X-Total', '3')
expect_header('X-Total-Pages', '2')
expect_header('X-Per-Page', '2')
expect_header('X-Page', '2')
expect_header('X-Next-Page', '')
expect_header('X-Prev-Page', '1')
expect_header('Link', anything) do |_key, val|
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="last"))
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="prev"))
expect(val).not_to include('rel="next"')
end
subject.paginate(resource)
end
end
context 'if order' do
it 'is not present it adds default order(:id) if no order is present' do
resource.order_values = []
paginated_relation = subject.paginate(resource)
expect(resource.order_values).to be_empty
expect(paginated_relation.order_values).to be_present
expect(paginated_relation.order_values.first).to be_ascending
expect(paginated_relation.order_values.first.expr.name).to eq 'id'
end
it 'is present it does not add anything' do
paginated_relation = subject.paginate(resource.order(created_at: :desc))
expect(paginated_relation.order_values).to be_present
expect(paginated_relation.order_values.first).to be_descending
expect(paginated_relation.order_values.first.expr.name).to eq 'created_at'
end
end
end
context 'when resource empty' do
describe 'first page' do
let(:query) { base_query.merge(page: 1, per_page: 2) }
it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 0
end
it 'adds appropriate headers' do
expect_header('X-Total', '0')
expect_header('X-Total-Pages', '1')
expect_header('X-Per-Page', '2')
expect_header('X-Page', '1')
expect_header('X-Next-Page', '')
expect_header('X-Prev-Page', '')
expect_header('Link', anything) do |_key, val|
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include(%Q(<#{incoming_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="last"))
expect(val).not_to include('rel="prev"')
expect(val).not_to include('rel="next"')
expect(val).not_to include('page=0')
end
subject.paginate(resource)
end
end
end
end
def expect_header(*args, &block)
expect(subject).to receive(:header).with(*args, &block)
end
def expect_no_header(*args, &block)
expect(subject).not_to receive(:header).with(*args)
end
def expect_message(method)
expect(subject).to receive(method)
.at_least(:once).and_return(value)
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7