Commit abd49d55 authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'documentation-for-keyset-pagination' into 'master'

Documentation for keyset pagination

See merge request gitlab-org/gitlab!62659
parents 9340d10f 305da642
......@@ -2,8 +2,8 @@
module PaginatorExtension
# This method loads the records for the requested page and returns a keyset paginator object.
def keyset_paginate(cursor: nil, per_page: 20)
Gitlab::Pagination::Keyset::Paginator.new(scope: self.dup, cursor: cursor, per_page: per_page)
def keyset_paginate(cursor: nil, per_page: 20, keyset_order_options: {})
Gitlab::Pagination::Keyset::Paginator.new(scope: self.dup, cursor: cursor, per_page: per_page, keyset_order_options: keyset_order_options)
end
end
......
This diff is collapsed.
......@@ -58,9 +58,7 @@ It's not possible to make all filter and sort combinations performant, so we sho
### Prepare for scaling
Offset-based pagination is the easiest way to paginate over records, however, it does not scale well for large tables. As a long-term solution, keyset pagination is preferred. The tooling around keyset pagination is not as mature as for offset pagination so currently, it's easier to start with offset pagination and then switch to keyset pagination.
To avoid losing functionality and maintaining backward compatibility when switching pagination methods, it's advised to consider the following approach in the design phase:
Offset-based pagination is the easiest way to paginate over records, however, it does not scale well for large database tables. As a long-term solution, [keyset pagination](keyset_pagination.md) is preferred. Switching between offset and keyset pagination is generally straightforward and can be done without affecting the end-user if the following conditions are met:
- Avoid presenting total counts, prefer limit counts.
- Example: count maximum 1001 records, and then on the UI show 1000+ if the count is 1001, show the actual number otherwise.
......@@ -304,7 +302,22 @@ LIMIT 20
##### Tooling
Using keyset pagination outside of GraphQL is not straightforward. We have the low-level blocks for building keyset pagination database queries, however, the usage in application code is still not streamlined yet.
A generic keyset pagination library is available within the GitLab project which can most of the cases easly replace the existing, kaminari based pagination with significant performance improvements when dealing with large datasets.
Example:
```ruby
# first page
paginator = Project.order(:created_at, :id).keyset_paginate(per_page: 20)
puts paginator.to_a # records
# next page
cursor = paginator.cursor_for_next_page
paginator = Project.order(:created_at, :id).keyset_paginate(cursor: cursor, per_page: 20)
puts paginator.to_a # records
```
For a comprehensive overview, take a look at the [keyset pagination guide](keyset_pagination.md) page.
#### Performance
......
......@@ -26,7 +26,7 @@ module Gitlab
# per_page - Number of items per page.
# cursor_converter - Object that serializes and de-serializes the cursor attributes. Implements dump and parse methods.
# direction_key - Symbol that will be the hash key of the direction within the cursor. (default: _kd => keyset direction)
def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd)
def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd, keyset_order_options: {})
@keyset_scope = build_scope(scope)
@order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@keyset_scope)
@per_page = per_page
......@@ -36,6 +36,7 @@ module Gitlab
@at_last_page = false
@at_first_page = false
@cursor_attributes = decode_cursor_attributes(cursor)
@keyset_order_options = keyset_order_options
set_pagination_helper_flags!
end
......@@ -45,13 +46,13 @@ module Gitlab
@records ||= begin
items = if paginate_backward?
reversed_order
.apply_cursor_conditions(keyset_scope, cursor_attributes)
.apply_cursor_conditions(keyset_scope, cursor_attributes, keyset_order_options)
.reorder(reversed_order)
.limit(per_page_plus_one)
.to_a
else
order
.apply_cursor_conditions(keyset_scope, cursor_attributes)
.apply_cursor_conditions(keyset_scope, cursor_attributes, keyset_order_options)
.limit(per_page_plus_one)
.to_a
end
......@@ -120,7 +121,7 @@ module Gitlab
private
attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes
attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes, :keyset_order_options
delegate :reversed_order, to: :order
......
......@@ -117,4 +117,27 @@ RSpec.describe Gitlab::Pagination::Keyset::Paginator do
expect { scope.keyset_paginate }.to raise_error(/does not support keyset pagination/)
end
end
context 'when use_union_optimization option is true and ordering by two columns' do
let(:scope) { Project.order(name: :asc, id: :desc) }
it 'uses UNION queries' do
paginator_first_page = scope.keyset_paginate(
per_page: 2,
keyset_order_options: { use_union_optimization: true }
)
paginator_second_page = scope.keyset_paginate(
per_page: 2,
cursor: paginator_first_page.cursor_for_next_page,
keyset_order_options: { use_union_optimization: true }
)
expect_next_instances_of(Gitlab::SQL::Union, 1) do |instance|
expect(instance.to_sql).to include(paginator_first_page.records.last.name)
end
paginator_second_page.records.to_a
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment