Merge branch 'qa-e2e-secure-container-scanning-tests' into 'master'

Added tests for Container Scanning

Closes gitlab-org/quality/testcases#130 and gitlab-org/quality/testcases#126

See merge request gitlab-org/gitlab-ee!15785

ee/app/assets/javascripts/vue_shared/security_reports/split_security_reports_app.vue

@@ -305,6 +305,7 @@ export default {
       :has-issues="sastContainer.newIssues.length > 0"
       :popover-options="sastContainerPopover"
       class="js-dependency-scanning-widget split-report-section"
+      data-qa-selector="container_scanning_report"
     />
 
     <report-section

qa/qa/ee/fixtures/secure_premade_reports/.gitlab-ci.yml

 include:
   template: Dependency-Scanning.gitlab-ci.yml
+  template: Container-Scanning.gitlab-ci.yml
 
 dependency_scanning:
   tags:
@@ -10,3 +11,16 @@ dependency_scanning:
   artifacts:
     reports:
       dependency_scanning: gl-dependency-scanning-report.json
+
+container_scanning:
+  tags:
+    - qa
+    - test
+  only: null # Template defaults to feature branches only
+  variables:
+    GIT_STRATEGY: fetch # Template defaults to none, which stops fetching the premade report
+  script:
+    - echo "Skipped"
+  artifacts:
+    reports:
+      container_scanning: gl-container-scanning-report.json

qa/qa/ee/fixtures/secure_premade_reports/gl-container-scanning-report.json

+{
+  "image": "registry.gitlab.com/groulot/container-scanning-test/master:5f21de6956aee99ddb68ae49498662d9872f50ff",
+  "unapproved": [
+    "CVE-2017-18269",
+    "CVE-2017-16997",
+    "CVE-2018-1000001",
+    "CVE-2016-10228",
+    "CVE-2018-18520",
+    "CVE-2010-4052",
+    "CVE-2018-16869",
+    "CVE-2018-18311"
+  ],
+  "vulnerabilities": [
+    {
+      "featurename": "glibc",
+      "featureversion": "2.24-11+deb9u3",
+      "vulnerability": "CVE-2017-18269",
+      "namespace": "debian:9",
+      "description": "SSE2-optimized memmove implementation problem.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2017-18269",
+      "severity": "Defcon1",
+      "fixedby": "2.24-11+deb9u4"
+    },
+    {
+      "featurename": "glibc",
+      "featureversion": "2.24-11+deb9u3",
+      "vulnerability": "CVE-2017-16997",
+      "namespace": "debian:9",
+      "description": "elf/dl-load.c in the GNU C Library (aka glibc or libc6) 2.19 through 2.26 mishandles RPATH and RUNPATH containing $ORIGIN for a privileged (setuid or AT_SECURE) program, which allows local users to gain privileges via a Trojan horse library in the current working directory, related to the fillin_rpath and decompose_rpath functions. This is associated with misinterpretion of an empty RPATH/RUNPATH token as the \"./\" directory. NOTE: this configuration of RPATH/RUNPATH for a privileged program is apparently very uncommon; most likely, no such program is shipped with any common Linux distribution.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2017-16997",
+      "severity": "Critical",
+      "fixedby": ""
+    },
+    {
+      "featurename": "glibc",
+      "featureversion": "2.24-11+deb9u3",
+      "vulnerability": "CVE-2018-1000001",
+      "namespace": "debian:9",
+      "description": "In glibc 2.26 and earlier there is confusion in the usage of getcwd() by realpath() which can be used to write before the destination buffer leading to a buffer underflow and potential code execution.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2018-1000001",
+      "severity": "High",
+      "fixedby": ""
+    },
+    {
+      "featurename": "glibc",
+      "featureversion": "2.24-11+deb9u3",
+      "vulnerability": "CVE-2016-10228",
+      "namespace": "debian:9",
+      "description": "The iconv program in the GNU C Library (aka glibc or libc6) 2.25 and earlier, when invoked with the -c option, enters an infinite loop when processing invalid multi-byte input sequences, leading to a denial of service.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2016-10228",
+      "severity": "Medium",
+      "fixedby": ""
+    },
+    {
+      "featurename": "elfutils",
+      "featureversion": "0.168-1",
+      "vulnerability": "CVE-2018-18520",
+      "namespace": "debian:9",
+      "description": "An Invalid Memory Address Dereference exists in the function elf_end in libelf in elfutils through v0.174. Although eu-size is intended to support ar files inside ar files, handle_ar in size.c closes the outer ar file before handling all inner entries. The vulnerability allows attackers to cause a denial of service (application crash) with a crafted ELF file.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2018-18520",
+      "severity": "Low",
+      "fixedby": ""
+    },
+    {
+      "featurename": "glibc",
+      "featureversion": "2.24-11+deb9u3",
+      "vulnerability": "CVE-2010-4052",
+      "namespace": "debian:9",
+      "description": "Stack consumption vulnerability in the regcomp implementation in the GNU C Library (aka glibc or libc6) through 2.11.3, and 2.12.x through 2.12.2, allows context-dependent attackers to cause a denial of service (resource exhaustion) via a regular expression containing adjacent repetition operators, as demonstrated by a {10,}{10,}{10,}{10,} sequence in the proftpd.gnu.c exploit for ProFTPD.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2010-4052",
+      "severity": "Negligible",
+      "fixedby": ""
+    },
+    {
+      "featurename": "nettle",
+      "featureversion": "3.3-1",
+      "vulnerability": "CVE-2018-16869",
+      "namespace": "debian:9",
+      "description": "A Bleichenbacher type side-channel based padding oracle attack was found in the way nettle handles endian conversion of RSA decrypted PKCS#1 v1.5 data. An attacker who is able to run a process on the same physical core as the victim process, could use this flaw extract plaintext or in some cases downgrade any TLS connections to a vulnerable server.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2018-16869",
+      "severity": "Unknown",
+      "fixedby": ""
+    },
+    {
+      "featurename": "perl",
+      "featureversion": "5.24.1-3+deb9u4",
+      "vulnerability": "CVE-2018-18311",
+      "namespace": "debian:9",
+      "description": "Perl before 5.26.3 and 5.28.x before 5.28.1 has a buffer overflow via a crafted regular expression that triggers invalid write operations.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2018-18311",
+      "severity": "Unknown",
+      "fixedby": "5.24.1-3+deb9u5"
+    },
+    {
+      "featurename": "foo",
+      "featureversion": "1.3",
+      "vulnerability": "CVE-2018-666",
+      "namespace": "debian:9",
+      "description": "Foo has a vulnerability nobody cares about and whitelist.",
+      "link": "https://security-tracker.debian.org/tracker/CVE-2018-666",
+      "severity": "Unknown",
+      "fixedby": "1.4"
+    }
+  ]
+}

qa/qa/ee/page/group/secure/show.rb

@@ -15,17 +15,19 @@ module QA
           end
 
           def filter_project(project)
-            find_element(:filter_project_dropdown).click
+            click_element(:filter_project_dropdown)
             within_element(:filter_dropdown_content) do
               click_on project
             end
           end
 
           def filter_report_type(report)
-            find_element(:filter_report_type_dropdown).click
+            click_element(:filter_report_type_dropdown)
             within_element(:filter_dropdown_content) do
               click_on report
             end
+            # Click the dropdown to close the modal and ensure it isn't open if this function is called again
+            click_element(:filter_report_type_dropdown)
           end
 
           def has_low_vulnerability_count_of?(expected)

qa/qa/ee/page/project/pipeline/show.rb

@@ -26,10 +26,12 @@ module QA::EE::Page
       end
 
       def filter_report_type(report)
-        find_element(:filter_report_type_dropdown).click
+        click_element(:filter_report_type_dropdown)
         within_element(:filter_dropdown_content) do
           click_on report
         end
+        # Click the dropdown to close the modal and ensure it isn't open if this function is called again
+        click_element(:filter_report_type_dropdown)
       end
     end
   end

qa/qa/ee/page/project/secure/show.rb

@@ -15,10 +15,12 @@ module QA
           end
 
           def filter_report_type(report)
-            find_element(:filter_report_type_dropdown).click
+            click_element(:filter_report_type_dropdown)
             within_element(:filter_dropdown_content) do
               click_on report
             end
+            # Click the dropdown to close the modal and ensure it isn't open if this function is called again
+            click_element(:filter_report_type_dropdown)
           end
 
           def has_low_vulnerability_count_of?(expected)

qa/qa/specs/features/ee/browser_ui/secure/create_merge_request_with_secure_spec.rb

@@ -5,6 +5,8 @@ require 'pathname'
 module QA
   context 'Secure', :docker do
     describe 'Security Reports in a Merge Request' do
+      let(:total_vuln_count) { 12 }
+
       after do
         Service::Runner.new(@executor).remove!
       end
@@ -51,7 +53,7 @@ module QA
       it 'displays the Security report in the merge request' do
         Page::MergeRequest::Show.perform do |mergerequest|
           expect(mergerequest).to have_vulnerability_report(timeout: 60)
-          expect(mergerequest).to have_detected_vulnerability_count_of "4"
+          expect(mergerequest).to have_detected_vulnerability_count_of total_vuln_count
         end
       end
 

qa/qa/specs/features/ee/browser_ui/secure/security_reports_spec.rb

@@ -5,21 +5,11 @@ require 'pathname'
 module QA
   context 'Secure', :docker do
     let(:number_of_dependencies_in_fixture) { 1309 }
+    let(:total_vuln_count) { 12 }
+    let(:dependency_scan_vuln_count) { 4 }
     let(:dependency_scan_example_vuln) { 'jQuery before 3.4.0' }
-
-    def login
-      Runtime::Browser.visit(:gitlab, Page::Main::Login)
-      Page::Main::Login.perform(&:sign_in_using_credentials)
-    end
-
-    def wait_for_job(job_name)
-      Page::Project::Pipeline::Show.perform do |pipeline|
-        pipeline.click_job(job_name)
-      end
-      Page::Project::Job::Show.perform do |job|
-        expect(job).to be_successful(timeout: 600)
-      end
-    end
+    let(:container_scan_vuln_count) { 8 }
+    let(:container_scan_example_vuln) { 'CVE-2017-18269 in glibc' }
 
     describe 'Security Reports' do
       after do
@@ -29,7 +19,8 @@ module QA
       before do
         @executor = "qa-runner-#{Time.now.to_i}"
 
-        login
+        Runtime::Browser.visit(:gitlab, Page::Main::Login)
+        Page::Main::Login.perform(&:sign_in_using_credentials)
 
         @project = Resource::Project.fabricate_via_api! do |p|
           p.name = Runtime::Env.auto_devops_project_name || 'project-with-secure'
@@ -53,51 +44,65 @@ module QA
 
         Page::Project::Menu.perform(&:click_ci_cd_pipelines)
         Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline)
-      end
 
-      it 'displays the Dependency Scanning report in the pipeline' do
         wait_for_job "dependency_scanning"
+      end
 
+      it 'displays security reports in the pipeline' do
         Page::Project::Menu.perform(&:click_ci_cd_pipelines)
         Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline)
 
         Page::Project::Pipeline::Show.perform do |pipeline|
           pipeline.click_on_security
-          pipeline.filter_report_type "Dependency Scanning"
-          expect(pipeline).to have_vulnerability_count_of 4
-          expect(pipeline).to have_content(dependency_scan_example_vuln)
-        end
+
+          filter_report_and_perform(pipeline, "Dependency Scanning") do
+            expect(pipeline).to have_vulnerability_count_of dependency_scan_vuln_count
+            expect(pipeline).to have_content dependency_scan_example_vuln
           end
 
-      it 'displays the Dependency Scanning report in the project security dashboard' do
-        wait_for_job "dependency_scanning"
+          filter_report_and_perform(pipeline, "Container Scanning") do
+            expect(pipeline).to have_vulnerability_count_of container_scan_vuln_count
+            expect(pipeline).to have_content container_scan_example_vuln
+          end
+        end
+      end
 
+      it 'displays security reports in the project security dashboard' do
         Page::Project::Menu.perform(&:click_project)
         Page::Project::Menu.perform(&:click_on_security_dashboard)
 
         EE::Page::Project::Secure::Show.perform do |dashboard|
-          dashboard.filter_report_type "Dependency Scanning"
-          expect(dashboard).to have_low_vulnerability_count_of "1"
-        end
+          filter_report_and_perform(dashboard, "Dependency Scanning") do
+            expect(dashboard).to have_low_vulnerability_count_of 1
           end
 
-      it 'displays the Dependency Scanning report in the group security dashboard' do
-        wait_for_job "dependency_scanning"
+          filter_report_and_perform(dashboard, "Container Scanning") do
+            expect(dashboard).to have_low_vulnerability_count_of 2
+          end
+        end
+      end
 
-        Page::Main::Menu.perform { |page| page.go_to_groups }
-        Page::Dashboard::Groups.perform { |page| page.click_group(@project.group.path) }
-        EE::Page::Group::Menu.perform { |page| page.click_group_security_link }
+      it 'displays security reports in the group security dashboard' do
+        Page::Main::Menu.perform(&:go_to_groups)
+        Page::Dashboard::Groups.perform do |page|
+          page.click_group @project.group.path
+        end
+        EE::Page::Group::Menu.perform(&:click_group_security_link)
 
         EE::Page::Group::Secure::Show.perform do |dashboard|
           dashboard.filter_project(@project.name)
-          dashboard.filter_report_type "Dependency Scanning"
+
+          filter_report_and_perform(dashboard, "Dependency Scanning") do
             expect(dashboard).to have_content dependency_scan_example_vuln
           end
+
+          filter_report_and_perform(dashboard, "Container Scanning") do
+            expect(dashboard).to have_content container_scan_example_vuln
+          end
+        end
       end
 
       it 'displays the Dependency List' do
-        wait_for_job "dependency_scanning"
-
         Page::Project::Menu.perform(&:click_on_dependency_list)
 
         EE::Page::Project::Secure::DependencyList.perform do |page|
@@ -105,5 +110,20 @@ module QA
         end
       end
     end
+
+    def wait_for_job(job_name)
+      Page::Project::Pipeline::Show.perform do |pipeline|
+        pipeline.click_job(job_name)
+      end
+      Page::Project::Job::Show.perform do |job|
+        expect(job).to be_successful(timeout: 600)
+      end
+    end
+
+    def filter_report_and_perform(page, report)
+      page.filter_report_type report
+      yield
+      page.filter_report_type report # Disable filter to avoid combining
+    end
   end
 end