Merge branch 'georgekoltsov/reduce-project-export-relation-batch-size' into 'master'

Lower project export batch_size

See merge request gitlab-org/gitlab!34057

lib/gitlab/import_export/json/streaming_serializer.rb

@@ -7,6 +7,15 @@ module Gitlab
         include Gitlab::ImportExport::CommandLineUtil
 
         BATCH_SIZE = 100
+        SMALLER_BATCH_SIZE = 20
+
+        def self.batch_size(exportable)
+          if Feature.enabled?(:export_reduce_relation_batch_size, exportable)
+            SMALLER_BATCH_SIZE
+          else
+            BATCH_SIZE
+          end
+        end
 
         class Raw < String
           def to_json(*_args)
@@ -60,7 +69,7 @@ module Gitlab
             key_preloads = preloads&.dig(key)
             records = records.preload(key_preloads) if key_preloads
 
-            records.find_each(batch_size: BATCH_SIZE) do |record|
+            records.find_each(batch_size: batch_size) do |record|
               items << Raw.new(record.to_json(options))
             end
           end
@@ -91,6 +100,10 @@ module Gitlab
         def preloads
           relations_schema[:preload]
         end
+
+        def batch_size
+          @batch_size ||= self.class.batch_size(@exportable)
+        end
       end
     end
   end

lib/gitlab/import_export/legacy_relation_tree_saver.rb

@@ -7,7 +7,7 @@ module Gitlab
 
       def serialize(exportable, relations_tree)
         Gitlab::ImportExport::FastHashSerializer
-          .new(exportable, relations_tree)
+          .new(exportable, relations_tree, batch_size: batch_size(exportable))
           .execute
       end
 
@@ -18,6 +18,12 @@ module Gitlab
 
         File.write(File.join(dir_path, filename), tree_json)
       end
+
+      private
+
+      def batch_size(exportable)
+        Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable)
+      end
     end
   end
 end

spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb

@@ -95,4 +95,26 @@ describe Gitlab::ImportExport::JSON::StreamingSerializer do
       end
     end
   end
+
+  describe '.batch_size' do
+    context 'when export_reduce_relation_batch_size feature flag is enabled' do
+      before do
+        stub_feature_flags(export_reduce_relation_batch_size: true)
+      end
+
+      it 'returns 20' do
+        expect(described_class.batch_size(exportable)).to eq(described_class::SMALLER_BATCH_SIZE)
+      end
+    end
+
+    context 'when export_reduce_relation_batch_size feature flag is disabled' do
+      before do
+        stub_feature_flags(export_reduce_relation_batch_size: false)
+      end
+
+      it 'returns default batch size' do
+        expect(described_class.batch_size(exportable)).to eq(described_class::BATCH_SIZE)
+      end
+    end
+  end
 end

spec/lib/gitlab/import_export/legacy_relation_tree_saver_spec.rb

@@ -8,17 +8,35 @@ describe Gitlab::ImportExport::LegacyRelationTreeSaver do
   let(:tree) { {} }
 
   describe '#serialize' do
-    let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) }
+    shared_examples 'FastHashSerializer with batch size' do |batch_size|
+      let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) }
 
-    it 'uses FastHashSerializer' do
-      expect(Gitlab::ImportExport::FastHashSerializer)
-        .to receive(:new)
-        .with(exportable, tree)
-        .and_return(serializer)
+      it 'uses FastHashSerializer' do
+        expect(Gitlab::ImportExport::FastHashSerializer)
+          .to receive(:new)
+          .with(exportable, tree, batch_size: batch_size)
+          .and_return(serializer)
 
-      expect(serializer).to receive(:execute)
+        expect(serializer).to receive(:execute)
 
-      relation_tree_saver.serialize(exportable, tree)
+        relation_tree_saver.serialize(exportable, tree)
+      end
+    end
+
+    context 'when export_reduce_relation_batch_size feature flag is enabled' do
+      before do
+        stub_feature_flags(export_reduce_relation_batch_size: true)
+      end
+
+      include_examples 'FastHashSerializer with batch size', Gitlab::ImportExport::JSON::StreamingSerializer::SMALLER_BATCH_SIZE
+    end
+
+    context 'when export_reduce_relation_batch_size feature flag is disabled' do
+      before do
+        stub_feature_flags(export_reduce_relation_batch_size: false)
+      end
+
+      include_examples 'FastHashSerializer with batch size', Gitlab::ImportExport::JSON::StreamingSerializer::BATCH_SIZE
     end
   end
 end