Commit 49a59f32 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'georgekoltsov/reduce-project-export-relation-batch-size' into 'master'

Lower project export batch_size

See merge request gitlab-org/gitlab!34057
parents 73704d11 1cd63f48
......@@ -7,6 +7,15 @@ module Gitlab
include Gitlab::ImportExport::CommandLineUtil
BATCH_SIZE = 100
SMALLER_BATCH_SIZE = 20
def self.batch_size(exportable)
if Feature.enabled?(:export_reduce_relation_batch_size, exportable)
SMALLER_BATCH_SIZE
else
BATCH_SIZE
end
end
class Raw < String
def to_json(*_args)
......@@ -60,7 +69,7 @@ module Gitlab
key_preloads = preloads&.dig(key)
records = records.preload(key_preloads) if key_preloads
records.find_each(batch_size: BATCH_SIZE) do |record|
records.find_each(batch_size: batch_size) do |record|
items << Raw.new(record.to_json(options))
end
end
......@@ -91,6 +100,10 @@ module Gitlab
def preloads
relations_schema[:preload]
end
def batch_size
@batch_size ||= self.class.batch_size(@exportable)
end
end
end
end
......
......@@ -7,7 +7,7 @@ module Gitlab
def serialize(exportable, relations_tree)
Gitlab::ImportExport::FastHashSerializer
.new(exportable, relations_tree)
.new(exportable, relations_tree, batch_size: batch_size(exportable))
.execute
end
......@@ -18,6 +18,12 @@ module Gitlab
File.write(File.join(dir_path, filename), tree_json)
end
private
def batch_size(exportable)
Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable)
end
end
end
end
......@@ -95,4 +95,26 @@ describe Gitlab::ImportExport::JSON::StreamingSerializer do
end
end
end
describe '.batch_size' do
context 'when export_reduce_relation_batch_size feature flag is enabled' do
before do
stub_feature_flags(export_reduce_relation_batch_size: true)
end
it 'returns 20' do
expect(described_class.batch_size(exportable)).to eq(described_class::SMALLER_BATCH_SIZE)
end
end
context 'when export_reduce_relation_batch_size feature flag is disabled' do
before do
stub_feature_flags(export_reduce_relation_batch_size: false)
end
it 'returns default batch size' do
expect(described_class.batch_size(exportable)).to eq(described_class::BATCH_SIZE)
end
end
end
end
......@@ -8,17 +8,35 @@ describe Gitlab::ImportExport::LegacyRelationTreeSaver do
let(:tree) { {} }
describe '#serialize' do
let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) }
shared_examples 'FastHashSerializer with batch size' do |batch_size|
let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) }
it 'uses FastHashSerializer' do
expect(Gitlab::ImportExport::FastHashSerializer)
.to receive(:new)
.with(exportable, tree)
.and_return(serializer)
it 'uses FastHashSerializer' do
expect(Gitlab::ImportExport::FastHashSerializer)
.to receive(:new)
.with(exportable, tree, batch_size: batch_size)
.and_return(serializer)
expect(serializer).to receive(:execute)
expect(serializer).to receive(:execute)
relation_tree_saver.serialize(exportable, tree)
relation_tree_saver.serialize(exportable, tree)
end
end
context 'when export_reduce_relation_batch_size feature flag is enabled' do
before do
stub_feature_flags(export_reduce_relation_batch_size: true)
end
include_examples 'FastHashSerializer with batch size', Gitlab::ImportExport::JSON::StreamingSerializer::SMALLER_BATCH_SIZE
end
context 'when export_reduce_relation_batch_size feature flag is disabled' do
before do
stub_feature_flags(export_reduce_relation_batch_size: false)
end
include_examples 'FastHashSerializer with batch size', Gitlab::ImportExport::JSON::StreamingSerializer::BATCH_SIZE
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment