Merge branch 'mo-add-pa11y-parser' into 'master'

Add Pa11y accessibility parser

See merge request gitlab-org/gitlab!29964

lib/gitlab/ci/parsers.rb

@@ -9,7 +9,8 @@ module Gitlab
         {
           junit: ::Gitlab::Ci::Parsers::Test::Junit,
           cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura,
-          terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan
+          terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan,
+          accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y
         }
       end
 

lib/gitlab/ci/parsers/accessibility/pa11y.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Accessibility
+        class Pa11y
+          def parse!(json_data, accessibility_report)
+            root = Gitlab::Json.parse(json_data)
+
+            parse_all(root, accessibility_report)
+          rescue JSON::ParserError => e
+            accessibility_report.set_error_message("JSON parsing failed: #{e}")
+          rescue StandardError => e
+            accessibility_report.set_error_message("Pa11y parsing failed: #{e}")
+          end
+
+          private
+
+          def parse_all(root, accessibility_report)
+            return unless root.present?
+
+            root.dig("results").each do |url, value|
+              accessibility_report.add_url(url, value)
+            end
+
+            accessibility_report
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/reports/accessibility_reports.rb

@@ -4,20 +4,37 @@ module Gitlab
   module Ci
     module Reports
       class AccessibilityReports
-        attr_accessor :total, :passes, :errors
-        attr_reader :urls
+        attr_reader :urls, :error_message
 
         def initialize
           @urls = {}
-          @total = 0
-          @passes = 0
-          @errors = 0
+          @error_message = nil
         end
 
         def add_url(url, data)
-          return if url.empty?
+          if url.empty?
+            set_error_message("Empty URL detected in gl-accessibility.json")
+          else
+            urls[url] = data
+          end
+        end
+
+        def scans_count
+          @urls.size
+        end
+
+        def passes_count
+          @urls.count { |url, errors| errors.empty? }
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def errors_count
+          @urls.sum { |url, errors| errors.size }
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
 
-          urls[url] = data
+        def set_error_message(error)
+          @error_message = error
         end
       end
     end

spec/lib/gitlab/ci/parsers/accessibility/pa11y_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+describe Gitlab::Ci::Parsers::Accessibility::Pa11y do
+  describe '#parse!' do
+    subject { described_class.new.parse!(pa11y, accessibility_report) }
+
+    let(:accessibility_report) { Gitlab::Ci::Reports::AccessibilityReports.new }
+
+    context "when data is pa11y style JSON" do
+      context "when there are no URLs provided" do
+        let(:pa11y) do
+          {
+            "total": 1,
+            "passes": 0,
+            "errors": 0,
+            "results": {
+              "": [
+                {
+                  "message": "Protocol error (Page.navigate): Cannot navigate to invalid URL"
+                }
+              ]
+            }
+          }.to_json
+        end
+
+        it "returns an accessibility report" do
+          expect { subject }.not_to raise_error
+
+          expect(accessibility_report.errors_count).to eq(0)
+          expect(accessibility_report.passes_count).to eq(0)
+          expect(accessibility_report.scans_count).to eq(0)
+          expect(accessibility_report.urls).to be_empty
+          expect(accessibility_report.error_message).to eq("Empty URL detected in gl-accessibility.json")
+        end
+      end
+
+      context "when there are no errors" do
+        let(:pa11y) do
+          {
+            "total": 1,
+            "passes": 1,
+            "errors": 0,
+            "results": {
+              "http://pa11y.org/": []
+            }
+          }.to_json
+        end
+
+        it "returns an accessibility report" do
+          expect { subject }.not_to raise_error
+
+          expect(accessibility_report.urls['http://pa11y.org/']).to be_empty
+          expect(accessibility_report.errors_count).to eq(0)
+          expect(accessibility_report.passes_count).to eq(1)
+          expect(accessibility_report.scans_count).to eq(1)
+        end
+      end
+
+      context "when there are errors" do
+        let(:pa11y) do
+          {
+            "total": 1,
+            "passes": 0,
+            "errors": 1,
+            "results": {
+              "https://about.gitlab.com/": [
+                {
+                  "code": "WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent",
+                  "type": "error",
+                  "typeCode": 1,
+                  "message": "Anchor element found with a valid href attribute, but no link content has been supplied.",
+                  "context": "<a href=\"/\" class=\"navbar-brand animated\"><svg height=\"36\" viewBox=\"0 0 1...</a>",
+                  "selector": "#main-nav > div:nth-child(1) > a",
+                  "runner": "htmlcs",
+                  "runnerExtras": {}
+                }
+              ]
+            }
+          }.to_json
+        end
+
+        it "returns an accessibility report" do
+          expect { subject }.not_to raise_error
+
+          expect(accessibility_report.errors_count).to eq(1)
+          expect(accessibility_report.passes_count).to eq(0)
+          expect(accessibility_report.scans_count).to eq(1)
+          expect(accessibility_report.urls['https://about.gitlab.com/']).to be_present
+        end
+      end
+    end
+
+    context "when data is not a valid JSON string" do
+      let(:pa11y) do
+        {
+          "total": 1,
+          "passes": 1,
+          "errors": 0,
+          "results": {
+            "http://pa11y.org/": []
+          }
+        }
+      end
+
+      it "sets error_message" do
+        expect { subject }.not_to raise_error
+
+        expect(accessibility_report.error_message).to include('Pa11y parsing failed')
+        expect(accessibility_report.errors_count).to eq(0)
+        expect(accessibility_report.passes_count).to eq(0)
+        expect(accessibility_report.scans_count).to eq(0)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/parsers_spec.rb

@@ -22,6 +22,14 @@ describe Gitlab::Ci::Parsers do
       end
     end
 
+    context 'when file_type is accessibility' do
+      let(:file_type) { 'accessibility' }
+
+      it 'fabricates the class' do
+        is_expected.to be_a(described_class::Accessibility::Pa11y)
+      end
+    end
+
     context 'when file_type is terraform' do
       let(:file_type) { 'terraform' }
 

spec/lib/gitlab/ci/reports/accessibility_reports_spec.rb

@@ -4,37 +4,144 @@ require 'spec_helper'
 
 describe Gitlab::Ci::Reports::AccessibilityReports do
   let(:accessibility_report) { described_class.new }
+  let(:url) { 'https://gitlab.com' }
+  let(:data) do
+    [
+      {
+        "code": "WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent",
+        "type": "error",
+        "typeCode": 1,
+        "message": "Anchor element found with a valid href attribute, but no link content has been supplied.",
+        "context": "<a href=\"/customers/worldline\">\n<svg viewBox=\"0 0 509 89\" xmln...</a>",
+        "selector": "html > body > div:nth-child(9) > div:nth-child(2) > a:nth-child(17)",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent",
+        "type": "error",
+        "typeCode": 1,
+        "message": "Anchor element found with a valid href attribute, but no link content has been supplied.",
+        "context": "<a href=\"/customers/equinix\">\n<svg xmlns=\"http://www.w3.org/...</a>",
+        "selector": "html > body > div:nth-child(9) > div:nth-child(2) > a:nth-child(18)",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      }
+    ]
+  end
 
-  describe '#add_url' do
-    subject { accessibility_report.add_url(url, data) }
+  describe '#scans_count' do
+    subject { accessibility_report.scans_count }
+
+    context 'when data has errors' do
+      let(:different_url) { 'https://about.gitlab.com' }
+
+      before do
+        accessibility_report.add_url(url, data)
+        accessibility_report.add_url(different_url, data)
+      end
+
+      it 'returns the scans_count' do
+        expect(subject).to eq(2)
+      end
+    end
+
+    context 'when data has no errors' do
+      before do
+        accessibility_report.add_url(url, [])
+      end
+
+      it 'returns the scans_count' do
+        expect(subject).to eq(1)
+      end
+    end
+
+    context 'when data has no url' do
+      before do
+        accessibility_report.add_url("", [])
+      end
+
+      it 'returns the scans_count' do
+        expect(subject).to eq(0)
+      end
+    end
+  end
+
+  describe '#passes_count' do
+    subject { accessibility_report.passes_count }
+
+    context 'when data has errors' do
+      before do
+        accessibility_report.add_url(url, data)
+      end
+
+      it 'returns the passes_count' do
+        expect(subject).to eq(0)
+      end
+    end
+
+    context 'when data has no errors' do
+      before do
+        accessibility_report.add_url(url, [])
+      end
+
+      it 'returns the passes_count' do
+        expect(subject).to eq(1)
+      end
+    end
+
+    context 'when data has no url' do
+      before do
+        accessibility_report.add_url("", [])
+      end
+
+      it 'returns the scans_count' do
+        expect(subject).to eq(0)
+      end
+    end
+  end
+
+  describe '#errors_count' do
+    subject { accessibility_report.errors_count }
 
     context 'when data has errors' do
-      let(:url) { 'https://gitlab.com' }
-      let(:data) do
-        [
-          {
-            "code": "WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent",
-            "type": "error",
-            "typeCode": 1,
-            "message": "Anchor element found with a valid href attribute, but no link content has been supplied.",
-            "context": "<a href=\"/customers/worldline\">\n<svg viewBox=\"0 0 509 89\" xmln...</a>",
-            "selector": "html > body > div:nth-child(9) > div:nth-child(2) > a:nth-child(17)",
-            "runner": "htmlcs",
-            "runnerExtras": {}
-          },
-          {
-            "code": "WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent",
-            "type": "error",
-            "typeCode": 1,
-            "message": "Anchor element found with a valid href attribute, but no link content has been supplied.",
-            "context": "<a href=\"/customers/equinix\">\n<svg xmlns=\"http://www.w3.org/...</a>",
-            "selector": "html > body > div:nth-child(9) > div:nth-child(2) > a:nth-child(18)",
-            "runner": "htmlcs",
-            "runnerExtras": {}
-          }
-        ]
+      let(:different_url) { 'https://about.gitlab.com' }
+
+      before do
+        accessibility_report.add_url(url, data)
+        accessibility_report.add_url(different_url, data)
+      end
+
+      it 'returns the errors_count' do
+        expect(subject).to eq(4)
+      end
+    end
+
+    context 'when data has no errors' do
+      before do
+        accessibility_report.add_url(url, [])
+      end
+
+      it 'returns the errors_count' do
+        expect(subject).to eq(0)
       end
+    end
+
+    context 'when data has no url' do
+      before do
+        accessibility_report.add_url("", [])
+      end
+
+      it 'returns the errors_count' do
+        expect(subject).to eq(0)
+      end
+    end
+  end
 
+  describe '#add_url' do
+    subject { accessibility_report.add_url(url, data) }
+
+    context 'when data has errors' do
       it 'adds urls and data to accessibility report' do
         expect { subject }.not_to raise_error
 
@@ -44,7 +151,6 @@ describe Gitlab::Ci::Reports::AccessibilityReports do
     end
 
     context 'when data does not have errors' do
-      let(:url) { 'https://gitlab.com' }
       let(:data) { [] }
 
       it 'adds data to accessibility report' do
@@ -59,10 +165,37 @@ describe Gitlab::Ci::Reports::AccessibilityReports do
       let(:url) { '' }
       let(:data) { [{ message: "Protocol error (Page.navigate): Cannot navigate to invalid URL" }] }
 
-      it 'do not add data to accessibility report' do
+      it 'sets error_message and decreases total' do
         expect { subject }.not_to raise_error
 
-        expect(accessibility_report.urls).to be_empty
+        expect(accessibility_report.scans_count).to eq(0)
+        expect(accessibility_report.error_message).to eq('Empty URL detected in gl-accessibility.json')
+      end
+    end
+  end
+
+  describe '#set_error_message' do
+    let(:set_accessibility_error) { accessibility_report.set_error_message('error') }
+
+    context 'when error is nil' do
+      it 'returns the error' do
+        expect(set_accessibility_error).to eq('error')
+      end
+
+      it 'sets the error' do
+        set_accessibility_error
+
+        expect(accessibility_report.error_message).to eq('error')
+      end
+    end
+
+    context 'when a error has already been set' do
+      before do
+        accessibility_report.set_error_message('old error')
+      end
+
+      it 'overwrites the existing message' do
+        expect { set_accessibility_error }.to change(accessibility_report, :error_message).from('old error').to('error')
       end
     end
   end