Merge branch 'id-fetch-blobs-in-batches' into 'master'

Limit number of blobs fetched in a single call

See merge request gitlab-org/gitlab!24247

lib/gitlab/git/blob.rb

@@ -13,6 +13,11 @@ module Gitlab
       # use load_all_data!.
       MAX_DATA_DISPLAY_SIZE = 10.megabytes
 
+      # The number of blobs loaded in a single Gitaly call
+      # When a large number of blobs requested, we'd want to fetch them in
+      # multiple Gitaly calls
+      BATCH_SIZE = 250
+
       # These limits are used as a heuristic to ignore files which can't be LFS
       # pointers. The format of these is described in
       # https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer
@@ -67,7 +72,13 @@ module Gitlab
         # to the caller to limit the number of blobs and blob_size_limit.
         #
         def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE)
-          repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a
+          if Feature.enabled?(:blobs_fetch_in_batches, default_enabled: true)
+            blob_references.each_slice(BATCH_SIZE).flat_map do |refs|
+              repository.gitaly_blob_client.get_blobs(refs, blob_size_limit).to_a
+            end
+          else
+            repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a
+          end
         end
 
         # Returns an array of Blob instances just with the metadata, that means

spec/lib/gitlab/git/blob_spec.rb

@@ -244,6 +244,61 @@ describe Gitlab::Git::Blob, :seed_helper do
         end
       end
     end
+
+    context 'when large number of blobs requested' do
+      let(:first_batch) do
+        [
+          [SeedRepo::Commit::ID, 'files/ruby/popen.rb'],
+          [SeedRepo::Commit::ID, 'six']
+        ]
+      end
+
+      let(:second_batch) do
+        [
+          [SeedRepo::Commit::ID, 'some'],
+          [SeedRepo::Commit::ID, 'other']
+        ]
+      end
+
+      let(:third_batch) do
+        [
+          [SeedRepo::Commit::ID, 'files']
+        ]
+      end
+
+      let(:blob_references) do
+        first_batch + second_batch + third_batch
+      end
+
+      let(:client) { repository.gitaly_blob_client }
+      let(:limit) { 10.megabytes }
+
+      before do
+        stub_const('Gitlab::Git::Blob::BATCH_SIZE', 2)
+      end
+
+      context 'blobs_fetch_in_batches is enabled' do
+        it 'fetches the blobs in batches' do
+          expect(client).to receive(:get_blobs).with(first_batch, limit).ordered
+          expect(client).to receive(:get_blobs).with(second_batch, limit).ordered
+          expect(client).to receive(:get_blobs).with(third_batch, limit).ordered
+
+          subject
+        end
+      end
+
+      context 'blobs_fetch_in_batches is disabled' do
+        before do
+          stub_feature_flags(blobs_fetch_in_batches: false)
+        end
+
+        it 'fetches the blobs in a single batch' do
+          expect(client).to receive(:get_blobs).with(blob_references, limit)
+
+          subject
+        end
+      end
+    end
   end
 
   describe '.batch_metadata' do