Merge branch 'documentation-for-keyset-pagination' into 'master'

Documentation for keyset pagination

See merge request gitlab-org/gitlab!62659

config/initializers/active_record_keyset_pagination.rb

@@ -2,8 +2,8 @@
 
 module PaginatorExtension
   # This method loads the records for the requested page and returns a keyset paginator object.
-  def keyset_paginate(cursor: nil, per_page: 20)
-    Gitlab::Pagination::Keyset::Paginator.new(scope: self.dup, cursor: cursor, per_page: per_page)
+  def keyset_paginate(cursor: nil, per_page: 20, keyset_order_options: {})
+    Gitlab::Pagination::Keyset::Paginator.new(scope: self.dup, cursor: cursor, per_page: per_page, keyset_order_options: keyset_order_options)
   end
 end
 

doc/development/database/pagination_guidelines.md

@@ -58,9 +58,7 @@ It's not possible to make all filter and sort combinations performant, so we sho
 
 ### Prepare for scaling
 
-Offset-based pagination is the easiest way to paginate over records, however, it does not scale well for large tables. As a long-term solution, keyset pagination is preferred. The tooling around keyset pagination is not as mature as for offset pagination so currently, it's easier to start with offset pagination and then switch to keyset pagination.
-
-To avoid losing functionality and maintaining backward compatibility when switching pagination methods, it's advised to consider the following approach in the design phase:
+Offset-based pagination is the easiest way to paginate over records, however, it does not scale well for large database tables. As a long-term solution, [keyset pagination](keyset_pagination.md) is preferred. Switching between offset and keyset pagination is generally straightforward and can be done without affecting the end-user if the following conditions are met:
 
 - Avoid presenting total counts, prefer limit counts.
   - Example: count maximum 1001 records, and then on the UI show 1000+ if the count is 1001, show the actual number otherwise.
@@ -304,7 +302,22 @@ LIMIT 20
 
 ##### Tooling
 
-Using keyset pagination outside of GraphQL is not straightforward. We have the low-level blocks for building keyset pagination database queries, however, the usage in application code is still not streamlined yet.
+A generic keyset pagination library is available within the GitLab project which can most of the cases easly replace the existing, kaminari based pagination with significant performance improvements when dealing with large datasets.
+
+Example:
+
+```ruby
+# first page
+paginator = Project.order(:created_at, :id).keyset_paginate(per_page: 20)
+puts paginator.to_a # records
+
+# next page
+cursor = paginator.cursor_for_next_page
+paginator = Project.order(:created_at, :id).keyset_paginate(cursor: cursor, per_page: 20)
+puts paginator.to_a # records
+```
+
+For a comprehensive overview, take a look at the [keyset pagination guide](keyset_pagination.md) page.
 
 #### Performance
 

lib/gitlab/pagination/keyset/paginator.rb

@@ -26,7 +26,7 @@ module Gitlab
         # per_page               - Number of items per page.
         # cursor_converter       - Object that serializes and de-serializes the cursor attributes. Implements dump and parse methods.
         # direction_key          - Symbol that will be the hash key of the direction within the cursor. (default: _kd => keyset direction)
-        def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd)
+        def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd, keyset_order_options: {})
           @keyset_scope = build_scope(scope)
           @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@keyset_scope)
           @per_page = per_page
@@ -36,6 +36,7 @@ module Gitlab
           @at_last_page = false
           @at_first_page = false
           @cursor_attributes = decode_cursor_attributes(cursor)
+          @keyset_order_options = keyset_order_options
 
           set_pagination_helper_flags!
         end
@@ -45,13 +46,13 @@ module Gitlab
           @records ||= begin
             items = if paginate_backward?
                       reversed_order
-                        .apply_cursor_conditions(keyset_scope, cursor_attributes)
+                        .apply_cursor_conditions(keyset_scope, cursor_attributes, keyset_order_options)
                         .reorder(reversed_order)
                         .limit(per_page_plus_one)
                         .to_a
                     else
                       order
-                        .apply_cursor_conditions(keyset_scope, cursor_attributes)
+                        .apply_cursor_conditions(keyset_scope, cursor_attributes, keyset_order_options)
                         .limit(per_page_plus_one)
                         .to_a
                     end
@@ -120,7 +121,7 @@ module Gitlab
 
         private
 
-        attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes
+        attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes, :keyset_order_options
 
         delegate :reversed_order, to: :order
 

spec/lib/gitlab/pagination/keyset/paginator_spec.rb

@@ -117,4 +117,27 @@ RSpec.describe Gitlab::Pagination::Keyset::Paginator do
       expect { scope.keyset_paginate }.to raise_error(/does not support keyset pagination/)
     end
   end
+
+  context 'when use_union_optimization option is true and ordering by two columns' do
+    let(:scope) { Project.order(name: :asc, id: :desc) }
+
+    it 'uses UNION queries' do
+      paginator_first_page = scope.keyset_paginate(
+        per_page: 2,
+        keyset_order_options: { use_union_optimization: true }
+      )
+
+      paginator_second_page = scope.keyset_paginate(
+        per_page: 2,
+        cursor: paginator_first_page.cursor_for_next_page,
+        keyset_order_options: { use_union_optimization: true }
+      )
+
+      expect_next_instances_of(Gitlab::SQL::Union, 1) do |instance|
+        expect(instance.to_sql).to include(paginator_first_page.records.last.name)
+      end
+
+      paginator_second_page.records.to_a
+    end
+  end
 end