Insert at most 1,000 rows at once in MR diff background migration

We were hitting the statement timeout for very large MR diffs. Now we insert at
most 1,000 rows to `merge_request_diff_commits` in a single statement, or 100
rows to `merge_request_diff_files`.

lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb

@@ -8,6 +8,7 @@ module Gitlab
       end
 
       BUFFER_ROWS = 1000
+      DIFF_FILE_BUFFER_ROWS = 100
 
       def perform(start_id, stop_id)
         merge_request_diffs = MergeRequestDiff
@@ -26,7 +27,7 @@ module Gitlab
 
           if diff_ids.length > BUFFER_ROWS ||
               commit_rows.length > BUFFER_ROWS ||
-              file_rows.length > BUFFER_ROWS
+              file_rows.length > DIFF_FILE_BUFFER_ROWS
 
             flush_buffers!
           end
@@ -46,8 +47,13 @@ module Gitlab
       def flush_buffers!
         if diff_ids.any?
           MergeRequestDiff.transaction do
-            Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows)
-            Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows)
+            commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice|
+              Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows_slice)
+            end
+
+            file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice|
+              Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows_slice)
+            end
 
             MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil)
           end

spec/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits_spec.rb

@@ -70,8 +70,8 @@ describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do
       before do
         merge_request.reload_diff(true)
 
-        convert_to_yaml(start_id, merge_request_diff.commits, merge_request_diff.diffs)
-        convert_to_yaml(stop_id, updated_merge_request_diff.commits, updated_merge_request_diff.diffs)
+        convert_to_yaml(start_id, merge_request_diff.commits, diffs_to_hashes(merge_request_diff.merge_request_diff_files))
+        convert_to_yaml(stop_id, updated_merge_request_diff.commits, diffs_to_hashes(updated_merge_request_diff.merge_request_diff_files))
 
         MergeRequestDiffCommit.delete_all
         MergeRequestDiffFile.delete_all
@@ -80,6 +80,8 @@ describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do
       context 'when BUFFER_ROWS is exceeded' do
         before do
           stub_const("#{described_class}::BUFFER_ROWS", 1)
+
+          allow(Gitlab::Database).to receive(:bulk_insert).and_call_original
         end
 
         it 'updates and continues' do
@@ -87,6 +89,32 @@ describe Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits do
 
           subject.perform(start_id, stop_id)
         end
+
+        it 'inserts commit rows in chunks of BUFFER_ROWS' do
+          # There are 29 commits in each diff, so we should have slices of 20 + 9 + 20 + 9.
+          stub_const("#{described_class}::BUFFER_ROWS", 20)
+
+          expect(Gitlab::Database).to receive(:bulk_insert)
+                                        .with('merge_request_diff_commits', anything)
+                                        .exactly(4)
+                                        .times
+                                        .and_call_original
+
+          subject.perform(start_id, stop_id)
+        end
+
+        it 'inserts diff rows in chunks of DIFF_FILE_BUFFER_ROWS' do
+          # There are 20 files in each diff, so we should have slices of 20 + 20.
+          stub_const("#{described_class}::DIFF_FILE_BUFFER_ROWS", 20)
+
+          expect(Gitlab::Database).to receive(:bulk_insert)
+                                        .with('merge_request_diff_files', anything)
+                                        .exactly(2)
+                                        .times
+                                        .and_call_original
+
+          subject.perform(start_id, stop_id)
+        end
       end
 
       context 'when BUFFER_ROWS is not exceeded' do