Merge branch 'mo-add-accessibility-report-job-artifact' into 'master'

Add accessibility report as a JobArtifact

See merge request gitlab-org/gitlab!29934

app/models/ci/job_artifact.rb

@@ -12,6 +12,7 @@ module Ci
 
     TEST_REPORT_FILE_TYPES = %w[junit].freeze
     COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze
+    ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze
     NON_ERASABLE_FILE_TYPES = %w[trace].freeze
     TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze
     DEFAULT_FILE_NAMES = {
@@ -21,6 +22,7 @@ module Ci
       metrics_referee: nil,
       network_referee: nil,
       junit: 'junit.xml',
+      accessibility: 'gl-accessibility.json',
       codequality: 'gl-code-quality-report.json',
       sast: 'gl-sast-report.json',
       dependency_scanning: 'gl-dependency-scanning-report.json',
@@ -54,6 +56,7 @@ module Ci
       # All these file formats use `raw` as we need to store them uncompressed
       # for Frontend to fetch the files and do analysis
       # When they will be only used by backend, they can be `gzipped`.
+      accessibility: :raw,
       codequality: :raw,
       sast: :raw,
       dependency_scanning: :raw,
@@ -99,6 +102,10 @@ module Ci
       with_file_types(TEST_REPORT_FILE_TYPES)
     end
 
+    scope :accessibility_reports, -> do
+      with_file_types(ACCESSIBILITY_REPORT_FILE_TYPES)
+    end
+
     scope :coverage_reports, -> do
       with_file_types(COVERAGE_REPORT_FILE_TYPES)
     end
@@ -138,7 +145,8 @@ module Ci
       lsif: 15, # LSIF data for code navigation
       dotenv: 16,
       cobertura: 17,
-      terraform: 18 # Transformed json
+      terraform: 18, # Transformed json
+      accessibility: 19
     }
 
     enum file_format: {

lib/gitlab/ci/config/entry/reports.rb

@@ -14,7 +14,7 @@ module Gitlab
           ALLOWED_KEYS =
             %i[junit codequality sast dependency_scanning container_scanning
                dast performance license_management license_scanning metrics lsif
-               dotenv cobertura terraform].freeze
+               dotenv cobertura terraform accessibility].freeze
 
           attributes ALLOWED_KEYS
 
@@ -37,6 +37,7 @@ module Gitlab
               validates :dotenv, array_of_strings_or_string: true
               validates :cobertura, array_of_strings_or_string: true
               validates :terraform, array_of_strings_or_string: true
+              validates :accessibility, array_of_strings_or_string: true
             end
           end
 

lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml

@@ -8,12 +8,14 @@ stages:
 
 a11y:
   stage: accessibility
-  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.2
+  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.3
   script: /gitlab-accessibility.sh $a11y_urls
   allow_failure: true
   artifacts:
     when: always
     expose_as: 'Accessibility Reports'
     paths: ['reports/']
+    reports:
+      accessibility: reports/gl-accessibility.json
   rules:
     - if: $a11y_urls

spec/factories/ci/job_artifacts.rb

@@ -139,6 +139,36 @@ FactoryBot.define do
       end
     end
 
+    trait :accessibility do
+      file_type { :accessibility }
+      file_format { :raw }
+
+      after(:build) do |artifact, _evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/accessibility/pa11y_with_errors.json'), 'application/json')
+      end
+    end
+
+    trait :accessibility_with_invalid_url do
+      file_type { :accessibility }
+      file_format { :raw }
+
+      after(:build) do |artifact, _evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/accessibility/pa11y_with_invalid_url.json'), 'application/json')
+      end
+    end
+
+    trait :accessibility_without_errors do
+      file_type { :accessibility }
+      file_format { :raw }
+
+      after(:build) do |artifact, _evaluator|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/accessibility/pa11y_without_errors.json'), 'application/json')
+      end
+    end
+
     trait :cobertura do
       file_type { :cobertura }
       file_format { :gzip }

spec/fixtures/accessibility/pa11y_with_errors.json

+{
+  "total": 1,
+  "passes": 0,
+  "errors": 10,
+  "results": {
+    "https://about.gitlab.com/": [
+      {
+        "code": "WCAG2AA.Principle4.Guideline4_1.4_1_2.H91.A.NoContent",
+        "type": "error",
+        "typeCode": 1,
+        "message": "Anchor element found with a valid href attribute, but no link content has been supplied.",
+        "context": "<a href=\"/\" class=\"navbar-brand animated\"><svg height=\"36\" viewBox=\"0 0 1...</a>",
+        "selector": "#main-nav > div:nth-child(1) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a href=\"/stages-devops-lifecycle/\" class=\"main-nav-link\">Product</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > li:nth-child(1) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a href=\"/pricing/\" class=\"main-nav-link\">Pricing</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > li:nth-child(2) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a href=\"/resources/\" class=\"main-nav-link\">Resources</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > li:nth-child(3) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a href=\"/blog/\" class=\"main-nav-link\">Blog</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > li:nth-child(4) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a href=\"/support/\" class=\"main-nav-link\">Support</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > li:nth-child(5) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a href=\"/jobs/\" class=\"main-nav-link\">Jobs</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > li:nth-child(6) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 2.82:1. Recommendation:  change background to #d1470c.",
+        "context": "<a class=\"btn btn-nav-cta btn-nav-link-cta\" href=\"/free-trial\">\nGet free trial\n</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > div:nth-child(8) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a class=\"main-nav-link sign-up\" href=\"https://gitlab.com/explore\" target=\"_blank\">\nExplore\n</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > div:nth-child(9) > li:nth-child(1) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      },
+      {
+        "code": "WCAG2AA.Principle1.Guideline1_4.1_4_3.G18.Fail",
+        "type": "error",
+        "typeCode": 1,
+        "message": "This element has insufficient contrast at this conformance level. Expected a contrast ratio of at least 4.5:1, but text in this element has a contrast ratio of 1.05:1. Recommendation:  change background to #767676.",
+        "context": "<a class=\"main-nav-link sign-up\" href=\"https://gitlab.com/users/sign_in\">\nSign in\n</a>",
+        "selector": "#main-nav > div:nth-child(2) > ul > div:nth-child(9) > li:nth-child(2) > a",
+        "runner": "htmlcs",
+        "runnerExtras": {}
+      }
+    ]
+  }
+}

spec/fixtures/accessibility/pa11y_with_invalid_url.json

+{
+  "total": 1,
+  "passes": 0,
+  "errors": 0,
+  "results": {
+    "": [
+      {
+        "message": "Protocol error (Page.navigate): Cannot navigate to invalid URL"
+      }
+    ]
+  }
+}

spec/fixtures/accessibility/pa11y_without_errors.json

+{
+  "total": 1,
+  "passes": 1,
+  "errors": 0,
+  "results": {
+    "https://pa11y.org/": []
+  }
+}

spec/lib/gitlab/ci/config/entry/reports_spec.rb

@@ -47,6 +47,7 @@ describe Gitlab::Ci::Config::Entry::Reports do
         :dotenv | 'build.dotenv'
         :cobertura | 'cobertura-coverage.xml'
         :terraform | 'tfplan.json'
+        :accessibility | 'gl-accessibility.json'
       end
 
       with_them do

spec/models/ci/job_artifact_spec.rb

@@ -70,6 +70,22 @@ describe Ci::JobArtifact do
     end
   end
 
+  describe '.accessibility_reports' do
+    subject { described_class.accessibility_reports }
+
+    context 'when there is an accessibility report' do
+      let(:artifact) { create(:ci_job_artifact, :accessibility) }
+
+      it { is_expected.to eq([artifact]) }
+    end
+
+    context 'when there are no accessibility report' do
+      let(:artifact) { create(:ci_job_artifact, :archive) }
+
+      it { is_expected.to be_empty }
+    end
+  end
+
   describe '.coverage_reports' do
     subject { described_class.coverage_reports }
 

spec/services/ci/retry_build_service_spec.rb

@@ -38,7 +38,7 @@ describe Ci::RetryBuildService do
        job_artifacts_codequality job_artifacts_metrics scheduled_at
        job_variables waiting_for_resource_at job_artifacts_metrics_referee
        job_artifacts_network_referee job_artifacts_dotenv
-       job_artifacts_cobertura needs].freeze
+       job_artifacts_cobertura needs job_artifacts_accessibility].freeze
 
   IGNORE_ACCESSORS =
     %i[type lock_version target_url base_tags trace_sections