Merge branch 'feature/handle-big-diffs' into 'master'

Improve handling of large diffs

Diffs with a large number of changed lines time out (504 HTTP error) or
generate a HTML page that's so heavy web browsers struggle with it.

https://github.com/gitlabhq/gitlabhq/pull/5014 introduced limits on
commit line count so that only a safe portion is rendered. This was
later undone by code refactoring in be5b6db8, e0eb4803 and c741fcab.
This patch re-introduces a safe limit on number of lines.

See merge request !539

CHANGELOG

@@ -22,6 +22,8 @@ v 7.11.0 (unreleased)
   - Fix bug causing `@whatever` inside an issue's first code block to be picked up as a user mention.
   - Fix bug causing `@whatever` inside an inline code snippet (backtick-style) to be picked up as a user mention.
   - When use change branches link at MR form - save source branch selection instead of target one
+  - Improve handling of large diffs
+  -
   - Show Atom feed buttons everywhere where applicable.
   - Add project activity atom feed.
   - Don't crash when an MR from a fork has a cross-reference comment from the target project on one of its commits.

app/helpers/diff_helper.rb

@@ -7,14 +7,23 @@ module DiffHelper
     end
   end
 
-  def safe_diff_files(diffs)
-    diffs.first(allowed_diff_size).map do |diff|
-      Gitlab::Diff::File.new(diff)
+  def allowed_diff_lines
+    if diff_hard_limit_enabled?
+      Commit::DIFF_HARD_LIMIT_LINES
+    else
+      Commit::DIFF_SAFE_LINES
     end
   end
 
-  def show_diff_size_warning?(diffs)
-    diffs.size > allowed_diff_size
+  def safe_diff_files(diffs)
+    lines = 0
+    safe_files = []
+    diffs.first(allowed_diff_size).each do |diff|
+      lines += diff.diff.lines.count
+      break if lines > allowed_diff_lines
+      safe_files << Gitlab::Diff::File.new(diff)
+    end
+    safe_files
   end
 
   def diff_hard_limit_enabled?

app/views/projects/diffs/_diffs.html.haml

@@ -5,11 +5,13 @@
       = parallel_diff_btn
   = render 'projects/diffs/stats', diffs: diffs
 
-- if show_diff_size_warning?(diffs)
-  = render 'projects/diffs/warning', diffs: diffs
+- diff_files = safe_diff_files(diffs)
+
+- if diff_files.count < diffs.size
+  = render 'projects/diffs/warning', diffs: diffs, shown_files_count: diff_files.count
 
 .files
-  - safe_diff_files(diffs).each_with_index do |diff_file, index|
+  - diff_files.each_with_index do |diff_file, index|
     = render 'projects/diffs/file', diff_file: diff_file, i: index, project: project
 
 - if @diff_timeout

app/views/projects/diffs/_warning.html.haml

@@ -14,6 +14,6 @@
           = link_to "Email patch", merge_request_path(@merge_request, format: :patch), class: "btn btn-warning btn-sm"
   %p
     To preserve performance only
-    %strong #{allowed_diff_size} of #{diffs.size}
+    %strong #{shown_files_count} of #{diffs.size}
     files are displayed.
 

spec/helpers/diff_helper_spec.rb

@@ -5,7 +5,8 @@ describe DiffHelper do
 
   let(:project) { create(:project) }
   let(:commit) { project.commit(sample_commit.id) }
-  let(:diff) { commit.diffs.first }
+  let(:diffs) { commit.diffs }
+  let(:diff) { diffs.first }
   let(:diff_file) { Gitlab::Diff::File.new(diff) }
 
   describe 'diff_hard_limit_enabled?' do
@@ -30,6 +31,57 @@ describe DiffHelper do
     end
   end
 
+  describe 'allowed_diff_lines' do
+    it 'should return hard limit for number of lines in a diff if force diff is true' do
+      allow(controller).to receive(:params) { { force_show_diff: true } }
+      expect(allowed_diff_lines).to eq(50000)
+    end
+
+    it 'should return safe limit for numbers of lines a diff if force diff is false' do
+      expect(allowed_diff_lines).to eq(5000)
+    end
+  end
+
+  describe 'safe_diff_files' do
+    it 'should return all files from a commit that is smaller than safe limits' do
+      expect(safe_diff_files(diffs).length).to eq(2)
+    end
+
+    it 'should return only the first file if the diff line count in the 2nd file takes the total beyond safe limits' do
+      diffs[1].diff.stub(lines: [""] * 4999) #simulate 4999 lines
+      expect(safe_diff_files(diffs).length).to eq(1)
+    end
+
+    it 'should return all files from a commit that is beyond safe limit for numbers of lines if force diff is true' do
+      allow(controller).to receive(:params) { { force_show_diff: true } }
+      diffs[1].diff.stub(lines: [""] * 4999) #simulate 4999 lines
+      expect(safe_diff_files(diffs).length).to eq(2)
+    end
+
+    it 'should return only the first file if the diff line count in the 2nd file takes the total beyond hard limits' do
+      allow(controller).to receive(:params) { { force_show_diff: true } }
+      diffs[1].diff.stub(lines: [""] * 49999) #simulate 49999 lines
+      expect(safe_diff_files(diffs).length).to eq(1)
+    end
+
+    it 'should return only a safe number of file diffs if a commit touches more files than the safe limits' do
+      large_diffs = diffs * 100 #simulate 200 diffs
+      expect(safe_diff_files(large_diffs).length).to eq(100)
+    end
+
+    it 'should return all file diffs if a commit touches more files than the safe limits but force diff is true' do
+      allow(controller).to receive(:params) { { force_show_diff: true } }
+      large_diffs = diffs * 100 #simulate 200 diffs
+      expect(safe_diff_files(large_diffs).length).to eq(200)
+    end
+
+    it 'should return a limited file diffs if a commit touches more files than the hard limits and force diff is true' do
+      allow(controller).to receive(:params) { { force_show_diff: true } }
+      very_large_diffs = diffs * 1000 #simulate 2000 diffs
+      expect(safe_diff_files(very_large_diffs).length).to eq(1000)
+    end
+  end
+
   describe 'parallel_diff' do
     it 'should return an array of arrays containing the parsed diff' do
       expect(parallel_diff(diff_file, 0)).