Add errored test case cases to test comparer specs

Test error_count, new_errors, existing_errors, and resolved_errors
Make sure *_errors methods obey limits on number of returned tests

spec/lib/gitlab/ci/reports/test_reports_comparer_spec.rb

@@ -57,6 +57,17 @@ describe Gitlab::Ci::Reports::TestReportsComparer do
         is_expected.to eq(Gitlab::Ci::Reports::TestCase::STATUS_FAILED)
       end
     end
+
+    context 'when there is an error test case in head suites' do
+      before do
+        head_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+        head_reports.get_suite('junit').add_test_case(create_test_case_java_error)
+      end
+
+      it 'returns the total status in head suite' do
+        is_expected.to eq(Gitlab::Ci::Reports::TestCase::STATUS_FAILED)
+      end
+    end
   end
 
   describe '#total_count' do
@@ -75,7 +86,7 @@ describe Gitlab::Ci::Reports::TestReportsComparer do
   describe '#resolved_count' do
     subject { comparer.resolved_count }
 
-    context 'when there is a resolved test case in head suites' do
+    context 'when there is a resolved failure test case in head suites' do
       before do
         base_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
         base_reports.get_suite('junit').add_test_case(create_test_case_java_failed)
@@ -88,6 +99,19 @@ describe Gitlab::Ci::Reports::TestReportsComparer do
       end
     end
 
+    context 'when there is a resolved error test case in head suites' do
+      before do
+        base_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+        base_reports.get_suite('junit').add_test_case(create_test_case_java_error)
+        head_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+        head_reports.get_suite('junit').add_test_case(create_test_case_java_success)
+      end
+
+      it 'returns the correct count' do
+        is_expected.to eq(1)
+      end
+    end
+
     context 'when there are no resolved test cases in head suites' do
       before do
         base_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
@@ -127,4 +151,30 @@ describe Gitlab::Ci::Reports::TestReportsComparer do
       end
     end
   end
+
+  describe '#error_count' do
+    subject { comparer.error_count }
+
+    context 'when there is an error test case in head suites' do
+      before do
+        head_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+        head_reports.get_suite('junit').add_test_case(create_test_case_java_error)
+      end
+
+      it 'returns the correct count' do
+        is_expected.to eq(1)
+      end
+    end
+
+    context 'when there are no error test cases in head suites' do
+      before do
+        head_reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+        head_reports.get_suite('junit').add_test_case(create_test_case_rspec_success)
+      end
+
+      it 'returns the correct count' do
+        is_expected.to eq(0)
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/reports/test_suite_comparer_spec.rb

@@ -9,8 +9,9 @@ describe Gitlab::Ci::Reports::TestSuiteComparer do
   let(:name) { 'rpsec' }
   let(:base_suite) { Gitlab::Ci::Reports::TestSuite.new(name) }
   let(:head_suite) { Gitlab::Ci::Reports::TestSuite.new(name) }
-  let(:test_case_success) { create_test_case_rspec_success }
-  let(:test_case_failed) { create_test_case_rspec_failed }
+  let(:test_case_success) { create_test_case_java_success }
+  let(:test_case_failed) { create_test_case_java_failed }
+  let(:test_case_error) { create_test_case_java_error }
 
   describe '#new_failures' do
     subject { comparer.new_failures }
@@ -135,6 +136,129 @@ describe Gitlab::Ci::Reports::TestSuiteComparer do
     end
   end
 
+  describe '#new_errors' do
+    subject { comparer.new_errors }
+
+    context 'when head suite has a new error test case which does not exist in base' do
+      before do
+        base_suite.add_test_case(test_case_success)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'returns the error test case' do
+        is_expected.to eq([test_case_error])
+      end
+    end
+
+    context 'when head suite still has an error test case which errored in base' do
+      before do
+        base_suite.add_test_case(test_case_error)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'does not return the error test case' do
+        is_expected.to be_empty
+      end
+    end
+
+    context 'when head suite has a success test case which errored in base' do
+      before do
+        base_suite.add_test_case(test_case_error)
+        head_suite.add_test_case(test_case_success)
+      end
+
+      it 'does not return the error test case' do
+        is_expected.to be_empty
+      end
+    end
+  end
+
+  describe '#existing_errors' do
+    subject { comparer.existing_errors }
+
+    context 'when head suite has a new error test case which does not exist in base' do
+      before do
+        base_suite.add_test_case(test_case_success)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'does not return the error test case' do
+        is_expected.to be_empty
+      end
+    end
+
+    context 'when head suite still has an error test case which errored in base' do
+      before do
+        base_suite.add_test_case(test_case_error)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'returns the error test case' do
+        is_expected.to eq([test_case_error])
+      end
+    end
+
+    context 'when head suite has a success test case which errored in base' do
+      before do
+        base_suite.add_test_case(test_case_error)
+        head_suite.add_test_case(test_case_success)
+      end
+
+      it 'does not return the error test case' do
+        is_expected.to be_empty
+      end
+    end
+  end
+
+  describe '#resolved_errors' do
+    subject { comparer.resolved_errors }
+
+    context 'when head suite has a new error test case which does not exist in base' do
+      before do
+        base_suite.add_test_case(test_case_success)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'does not return the error test case' do
+        is_expected.to be_empty
+      end
+
+      it 'returns the correct resolved count' do
+        expect(comparer.resolved_count).to eq(0)
+      end
+    end
+
+    context 'when head suite still has an error test case which errored in base' do
+      before do
+        base_suite.add_test_case(test_case_error)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'does not return the error test case' do
+        is_expected.to be_empty
+      end
+
+      it 'returns the correct resolved count' do
+        expect(comparer.resolved_count).to eq(0)
+      end
+    end
+
+    context 'when head suite has a success test case which errored in base' do
+      before do
+        base_suite.add_test_case(test_case_error)
+        head_suite.add_test_case(test_case_success)
+      end
+
+      it 'returns the resolved test case' do
+        is_expected.to eq([test_case_success])
+      end
+
+      it 'returns the correct resolved count' do
+        expect(comparer.resolved_count).to eq(1)
+      end
+    end
+  end
+
   describe '#total_count' do
     subject { comparer.total_count }
 
@@ -208,7 +332,17 @@ describe Gitlab::Ci::Reports::TestSuiteComparer do
         head_suite.add_test_case(test_case_failed)
       end
 
-      it 'returns the total status in head suite' do
+      it 'returns the total status in head suite as failed' do
+        is_expected.to eq(Gitlab::Ci::Reports::TestCase::STATUS_FAILED)
+      end
+    end
+
+    context 'when there is an error test case in head suite' do
+      before do
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'returns the total status in head suite as failed' do
         is_expected.to eq(Gitlab::Ci::Reports::TestCase::STATUS_FAILED)
       end
     end

spec/lib/gitlab/ci/reports/test_suite_spec.rb

@@ -74,6 +74,15 @@ describe Gitlab::Ci::Reports::TestSuite do
 
       it { is_expected.to eq(Gitlab::Ci::Reports::TestCase::STATUS_FAILED) }
     end
+
+    context 'when a test case errored' do
+      before do
+        test_suite.add_test_case(test_case_success)
+        test_suite.add_test_case(test_case_error)
+      end
+
+      it { is_expected.to eq(Gitlab::Ci::Reports::TestCase::STATUS_FAILED) }
+    end
   end
 
   Gitlab::Ci::Reports::TestCase::STATUS_TYPES.each do |status_type|

spec/serializers/test_reports_comparer_entity_spec.rb

@@ -24,7 +24,7 @@ describe TestReportsComparerEntity do
 
         it 'contains correct compared test reports details' do
           expect(subject[:status]).to eq('success')
-          expect(subject[:summary]).to include(total: 2, resolved: 0, failed: 0)
+          expect(subject[:summary]).to include(total: 2, resolved: 0, failed: 0, errored: 0)
           expect(subject[:suites].first[:name]).to eq('rspec')
           expect(subject[:suites].first[:status]).to eq('success')
           expect(subject[:suites].second[:name]).to eq('junit')
@@ -42,7 +42,7 @@ describe TestReportsComparerEntity do
 
         it 'contains correct compared test reports details' do
           expect(subject[:status]).to eq('failed')
-          expect(subject[:summary]).to include(total: 2, resolved: 0, failed: 1)
+          expect(subject[:summary]).to include(total: 2, resolved: 0, failed: 1, errored: 0)
           expect(subject[:suites].first[:name]).to eq('rspec')
           expect(subject[:suites].first[:status]).to eq('success')
           expect(subject[:suites].second[:name]).to eq('junit')
@@ -60,7 +60,7 @@ describe TestReportsComparerEntity do
 
         it 'contains correct compared test reports details' do
           expect(subject[:status]).to eq('success')
-          expect(subject[:summary]).to include(total: 2, resolved: 1, failed: 0)
+          expect(subject[:summary]).to include(total: 2, resolved: 1, failed: 0, errored: 0)
           expect(subject[:suites].first[:name]).to eq('rspec')
           expect(subject[:suites].first[:status]).to eq('success')
           expect(subject[:suites].second[:name]).to eq('junit')

spec/serializers/test_suite_comparer_entity_spec.rb

@@ -12,6 +12,7 @@ describe TestSuiteComparerEntity do
   let(:head_suite) { Gitlab::Ci::Reports::TestSuite.new(name) }
   let(:test_case_success) { create_test_case_rspec_success }
   let(:test_case_failed) { create_test_case_rspec_failed }
+  let(:test_case_error) { create_test_case_rspec_error }
 
   describe '#as_json' do
     subject { entity.as_json }
@@ -25,7 +26,7 @@ describe TestSuiteComparerEntity do
       it 'contains correct compared test suite details' do
         expect(subject[:name]).to eq(name)
         expect(subject[:status]).to eq('failed')
-        expect(subject[:summary]).to include(total: 1, resolved: 0, failed: 1)
+        expect(subject[:summary]).to include(total: 1, resolved: 0, failed: 1, errored: 0)
         subject[:new_failures].first.tap do |new_failure|
           expect(new_failure[:status]).to eq(test_case_failed.status)
           expect(new_failure[:name]).to eq(test_case_failed.name)
@@ -37,6 +38,27 @@ describe TestSuiteComparerEntity do
       end
     end
 
+    context 'when head suite has a new error test case which does not exist in base' do
+      before do
+        base_suite.add_test_case(test_case_success)
+        head_suite.add_test_case(test_case_error)
+      end
+
+      it 'contains correct compared test suite details' do
+        expect(subject[:name]).to eq(name)
+        expect(subject[:status]).to eq('failed')
+        expect(subject[:summary]).to include(total: 1, resolved: 0, failed: 0, errored: 1)
+        subject[:new_errors].first.tap do |new_error|
+          expect(new_error[:status]).to eq(test_case_error.status)
+          expect(new_error[:name]).to eq(test_case_error.name)
+          expect(new_error[:execution_time]).to eq(test_case_error.execution_time)
+          expect(new_error[:system_output]).to eq(test_case_error.system_output)
+        end
+        expect(subject[:resolved_failures]).to be_empty
+        expect(subject[:existing_failures]).to be_empty
+      end
+    end
+
     context 'when head suite still has a failed test case which failed in base' do
       before do
         base_suite.add_test_case(test_case_failed)
@@ -46,7 +68,7 @@ describe TestSuiteComparerEntity do
       it 'contains correct compared test suite details' do
         expect(subject[:name]).to eq(name)
         expect(subject[:status]).to eq('failed')
-        expect(subject[:summary]).to include(total: 1, resolved: 0, failed: 1)
+        expect(subject[:summary]).to include(total: 1, resolved: 0, failed: 1, errored: 0)
         expect(subject[:new_failures]).to be_empty
         expect(subject[:resolved_failures]).to be_empty
         subject[:existing_failures].first.tap do |existing_failure|
@@ -67,7 +89,7 @@ describe TestSuiteComparerEntity do
       it 'contains correct compared test suite details' do
         expect(subject[:name]).to eq(name)
         expect(subject[:status]).to eq('success')
-        expect(subject[:summary]).to include(total: 1, resolved: 1, failed: 0)
+        expect(subject[:summary]).to include(total: 1, resolved: 1, failed: 0, errored: 0)
         expect(subject[:new_failures]).to be_empty
         subject[:resolved_failures].first.tap do |resolved_failure|
           expect(resolved_failure[:status]).to eq(test_case_success.status)
@@ -88,42 +110,57 @@ describe TestSuiteComparerEntity do
       context 'prefers new over existing and resolved' do
         before do
           3.times { add_new_failure }
+          3.times { add_new_error }
           3.times { add_existing_failure }
+          3.times { add_existing_error }
           3.times { add_resolved_failure }
+          3.times { add_resolved_error }
         end
 
-        it 'returns 2 new failures, and 1 of resolved and existing' do
-          expect(subject[:summary]).to include(total: 9, resolved: 3, failed: 6)
+        it 'returns 2 of each new category, and 1 of each resolved and existing' do
+          expect(subject[:summary]).to include(total: 18, resolved: 6, failed: 6, errored: 6)
           expect(subject[:new_failures].count).to eq(2)
+          expect(subject[:new_errors].count).to eq(2)
           expect(subject[:existing_failures].count).to eq(1)
+          expect(subject[:existing_errors].count).to eq(1)
           expect(subject[:resolved_failures].count).to eq(1)
+          expect(subject[:resolved_errors].count).to eq(1)
         end
       end
 
       context 'prefers existing over resolved' do
         before do
           3.times { add_existing_failure }
+          3.times { add_existing_error }
           3.times { add_resolved_failure }
+          3.times { add_resolved_error }
         end
 
-        it 'returns 2 existing failures, and 1 resolved' do
-          expect(subject[:summary]).to include(total: 6, resolved: 3, failed: 3)
+        it 'returns 2 of each existing category, and 1 of each resolved' do
+          expect(subject[:summary]).to include(total: 12, resolved: 6, failed: 3, errored: 3)
           expect(subject[:new_failures].count).to eq(0)
+          expect(subject[:new_errors].count).to eq(0)
           expect(subject[:existing_failures].count).to eq(2)
+          expect(subject[:existing_errors].count).to eq(2)
           expect(subject[:resolved_failures].count).to eq(1)
+          expect(subject[:resolved_errors].count).to eq(1)
         end
       end
 
       context 'limits amount of resolved' do
         before do
           3.times { add_resolved_failure }
+          3.times { add_resolved_error }
         end
 
-        it 'returns 2 resolved failures' do
-          expect(subject[:summary]).to include(total: 3, resolved: 3, failed: 0)
+        it 'returns 2 of each resolved category' do
+          expect(subject[:summary]).to include(total: 6, resolved: 6, failed: 0, errored: 0)
           expect(subject[:new_failures].count).to eq(0)
+          expect(subject[:new_errors].count).to eq(0)
           expect(subject[:existing_failures].count).to eq(0)
+          expect(subject[:existing_errors].count).to eq(0)
           expect(subject[:resolved_failures].count).to eq(2)
+          expect(subject[:resolved_errors].count).to eq(2)
         end
       end
 
@@ -134,19 +171,38 @@ describe TestSuiteComparerEntity do
         head_suite.add_test_case(failed_case)
       end
 
+      def add_new_error
+        error_case = create_test_case_rspec_error(SecureRandom.hex)
+        head_suite.add_test_case(error_case)
+      end
+
       def add_existing_failure
         failed_case = create_test_case_rspec_failed(SecureRandom.hex)
         base_suite.add_test_case(failed_case)
         head_suite.add_test_case(failed_case)
       end
 
+      def add_existing_error
+        error_case = create_test_case_rspec_error(SecureRandom.hex)
+        base_suite.add_test_case(error_case)
+        head_suite.add_test_case(error_case)
+      end
+
       def add_resolved_failure
         case_name = SecureRandom.hex
-        failed_case = create_test_case_rspec_failed(case_name)
-        success_case = create_test_case_rspec_success(case_name)
+        failed_case = create_test_case_java_failed(case_name)
+        success_case = create_test_case_java_success(case_name)
         base_suite.add_test_case(failed_case)
         head_suite.add_test_case(success_case)
       end
+
+      def add_resolved_error
+        case_name = SecureRandom.hex
+        error_case = create_test_case_java_error(case_name)
+        success_case = create_test_case_java_success(case_name)
+        base_suite.add_test_case(error_case)
+        head_suite.add_test_case(success_case)
+      end
     end
   end
 end