Merge branch '213571-drop-lm-artifact-from-ci' into 'master'

Drop  support for  license-management artifacts

See merge request gitlab-org/gitlab!31247

app/models/ci/job_artifact.rb

@@ -15,6 +15,7 @@ module Ci
     ACCESSIBILITY_REPORT_FILE_TYPES = %w[accessibility].freeze
     NON_ERASABLE_FILE_TYPES = %w[trace].freeze
     TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze
+    UNSUPPORTED_FILE_TYPES = %i[license_management].freeze
     DEFAULT_FILE_NAMES = {
       archive: nil,
       metadata: nil,
@@ -100,7 +101,8 @@ module Ci
     mount_uploader :file, JobArtifactUploader
 
     validates :file_format, presence: true, unless: :trace?, on: :create
-    validate :valid_file_format?, unless: :trace?, on: :create
+    validate :validate_supported_file_format!, on: :create
+    validate :validate_file_format!, unless: :trace?, on: :create
     before_save :set_size, if: :file_changed?
 
     update_project_statistics project_statistics_name: :build_artifacts_size
@@ -202,7 +204,15 @@ module Ci
       raw: Gitlab::Ci::Build::Artifacts::Adapters::RawStream
     }.freeze
 
-    def valid_file_format?
+    def validate_supported_file_format!
+      return if Feature.disabled?(:drop_license_management_artifact, project, default_enabled: true)
+
+      if UNSUPPORTED_FILE_TYPES.include?(self.file_type&.to_sym)
+        errors.add(:base, _("File format is no longer supported"))
+      end
+    end
+
+    def validate_file_format!
       unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
         errors.add(:base, _('Invalid file format with specified file type'))
       end

changelogs/unreleased/213571-drop-lm-artifact-from-ci.yml

+---
+title: Drop support for `license_management` artifact
+merge_request: 31247
+author:
+type: removed

doc/ci/yaml/README.md

@@ -106,7 +106,7 @@ The following table lists available parameters for jobs:
 | [`when`](#when)                                    | When to run job. Also available: `when:manual` and `when:delayed`.                                                                                                                  |
 | [`environment`](#environment)                      | Name of an environment to which the job deploys. Also available: `environment:name`, `environment:url`, `environment:on_stop`, `environment:auto_stop_in` and `environment:action`. |
 | [`cache`](#cache)                                  | List of files that should be cached between subsequent runs. Also available: `cache:paths`, `cache:key`, `cache:untracked`, and `cache:policy`.                                     |
-| [`artifacts`](#artifacts)                          | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_management`, `artifacts:reports:performance` and `artifacts:reports:metrics`. |
+| [`artifacts`](#artifacts)                          | List of files and directories to attach to a job on success. Also available: `artifacts:paths`, `artifacts:expose_as`, `artifacts:name`, `artifacts:untracked`, `artifacts:when`, `artifacts:expire_in`, `artifacts:reports`, `artifacts:reports:junit`, `artifacts:reports:cobertura`, and `artifacts:reports:terraform`.<br><br>In GitLab [Enterprise Edition](https://about.gitlab.com/pricing/), these are available: `artifacts:reports:codequality`, `artifacts:reports:sast`, `artifacts:reports:dependency_scanning`, `artifacts:reports:container_scanning`, `artifacts:reports:dast`, `artifacts:reports:license_scanning`, `artifacts:reports:license_management` (removed in 13.0),`artifacts:reports:performance` and `artifacts:reports:metrics`. |
 | [`dependencies`](#dependencies)                    | Restrict which artifacts are passed to a specific job by providing a list of jobs to fetch artifacts from.                                                                          |
 | [`coverage`](#coverage)                            | Code coverage settings for a given job.                                                                                                                                             |
 | [`retry`](#retry)                                  | When and how many times a job can be auto-retried in case of a failure.                                                                                                             |
@@ -2757,7 +2757,7 @@ These are the available report types:
 | [`artifacts:reports:dependency_scanning`](../pipelines/job_artifacts.md#artifactsreportsdependency_scanning-ultimate) **(ULTIMATE)** | The `dependency_scanning` report collects Dependency Scanning vulnerabilities.   |
 | [`artifacts:reports:container_scanning`](../pipelines/job_artifacts.md#artifactsreportscontainer_scanning-ultimate) **(ULTIMATE)**   | The `container_scanning` report collects Container Scanning vulnerabilities.     |
 | [`artifacts:reports:dast`](../pipelines/job_artifacts.md#artifactsreportsdast-ultimate) **(ULTIMATE)**                               | The `dast` report collects Dynamic Application Security Testing vulnerabilities. |
-| [`artifacts:reports:license_management`](../pipelines/job_artifacts.md#artifactsreportslicense_management-ultimate) **(ULTIMATE)**   | The `license_management` report collects Licenses (*deprecated*).                |
+| [`artifacts:reports:license_management`](../pipelines/job_artifacts.md#artifactsreportslicense_management-ultimate) **(ULTIMATE)**   | The `license_management` report collects Licenses (*removed from 13.0*).                |
 | [`artifacts:reports:license_scanning`](../pipelines/job_artifacts.md#artifactsreportslicense_scanning-ultimate) **(ULTIMATE)**       | The `license_scanning` report collects Licenses.                                 |
 | [`artifacts:reports:performance`](../pipelines/job_artifacts.md#artifactsreportsperformance-premium) **(PREMIUM)**                   | The `performance` report collects Performance metrics.                           |
 | [`artifacts:reports:metrics`](../pipelines/job_artifacts.md#artifactsreportsmetrics-premium) **(PREMIUM)**                           | The `metrics` report collects Metrics.                                           |

doc/user/compliance/license_compliance/index.md

@@ -100,7 +100,7 @@ For GitLab versions earlier than 11.9, you can copy and use the job as defined
 that template.
 
 NOTE: **Note:**
-In GitLab 13.0, the `License-Management.gitlab-ci.yml` template is scheduled to be removed.
+In GitLab 13.0, the `License-Management.gitlab-ci.yml` template was removed.
 Use `License-Scanning.gitlab-ci.yml` instead.
 
 Add the following to your `.gitlab-ci.yml` file:
@@ -115,7 +115,7 @@ and scan your dependencies to find their licenses.
 
 NOTE: **Note:**
 Before GitLab 12.8, the `license_scanning` job was named `license_management`.
-In GitLab 13.0, the `license_management` job is scheduled to be removed completely,
+In GitLab 13.0, the `license_management` job was removed,
 so you're advised to migrate to the `license_scanning` job and used the new
 `License-Scanning.gitlab-ci.yml` template.
 
@@ -329,13 +329,13 @@ strict-ssl = false
 ### Migration from `license_management` to `license_scanning`
 
 In GitLab 12.8 a new name for `license_management` job was introduced. This change was made to improve clarity around the purpose of the scan, which is to scan and collect the types of licenses present in a projects dependencies.
-The support of `license_management` is scheduled to be dropped in GitLab 13.0.
+The support of `license_management` was dropped in GitLab 13.0.
 If you're using a custom setup for License Compliance, you're required
 to update your CI config accordingly:
 
 1. Change the CI template to `License-Scanning.gitlab-ci.yml`.
 1. Change the job name to `license_scanning` (if you mention it in `.gitlab-ci.yml`).
-1. Change the artifact name to `gl-license-scanning-report.json` (if you mention it in `.gitlab-ci.yml`).
+1. Change the artifact name to `license_scanning` and file name to `gl-license-scanning-report.json` (if you mention it in `.gitlab-ci.yml`).
 
 For example, the following `.gitlab-ci.yml`:
 
@@ -361,6 +361,16 @@ license_scanning:
       license_scanning: gl-license-scanning-report.json
 ```
 
+Since GitLab 13.0, if you use `license_management` artifact, you will encounter an error while running the License Compliance job:
+
+```text
+WARNING: Uploading artifacts to coordinator... failed id=:id responseStatus=400 Bad Request status=400 Bad Request token=:sha
+
+FATAL: invalid_argument
+```
+
+If you encounter this error, you're encouraged to follow the instructions described in this section.
+
 ## Running License Compliance in an offline environment
 
 For self-managed GitLab instances in an environment with limited, restricted, or intermittent access
@@ -467,7 +477,7 @@ Searching for Licenses:
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5491) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2.
 
 From your project's left sidebar, navigate to **CI/CD > Pipelines** and click on the
-pipeline ID that has a `license_management` job to see the Licenses tab with the listed
+pipeline ID that has a `license_scanning` job to see the Licenses tab with the listed
 licenses (if any).
 
 ![License Compliance Pipeline Tab](img/license_compliance_pipeline_tab_v13_0.png)

ee/spec/factories/ci/job_artifacts.rb

@@ -140,6 +140,8 @@ FactoryBot.define do
     end
 
     trait :license_management do
+      to_create { |instance| instance.save(validate: false) }
+
       file_type { :license_management }
       file_format { :raw }
 

ee/spec/factories/ci/pipelines.rb

@@ -65,7 +65,7 @@ FactoryBot.define do
       end
     end
 
-    trait :with_license_management_feature_branch do
+    trait :with_license_scanning_feature_branch do
       status { :success }
 
       after(:build) do |pipeline, evaluator|
@@ -73,7 +73,7 @@ FactoryBot.define do
       end
     end
 
-    trait :with_corrupted_license_management_report do
+    trait :with_corrupted_license_scanning_report do
       status { :success }
 
       after(:build) do |pipeline, evaluator|

ee/spec/features/projects/pipelines/pipeline_spec.rb

@@ -136,7 +136,7 @@ describe 'Pipeline', :js do
 
     context 'with a License Compliance artifact' do
       before do
-        create(:ee_ci_build, :license_management, pipeline: pipeline)
+        create(:ee_ci_build, :license_scanning, pipeline: pipeline)
 
         visit licenses_project_pipeline_path(project, pipeline)
       end

ee/spec/models/approval_merge_request_rule_spec.rb

@@ -409,7 +409,7 @@ describe ApprovalMergeRequestRule do
       let(:open_merge_request) { create(:merge_request, :opened, target_project: project, source_project: project) }
       let!(:project_approval_rule) { create(:approval_project_rule, :requires_approval, :license_scanning, project: project) }
       let(:project) { create(:project) }
-      let!(:open_pipeline) { create(:ee_ci_pipeline, :success, :with_license_management_report, project: project, merge_requests_as_head_pipeline: [open_merge_request]) }
+      let!(:open_pipeline) { create(:ee_ci_pipeline, :success, :with_license_scanning_report, project: project, merge_requests_as_head_pipeline: [open_merge_request]) }
       let!(:denied_policy) { create(:software_license_policy, project: project, software_license: license, classification: :denied) }
 
       before do

ee/spec/models/ci/pipeline_spec.rb

@@ -121,6 +121,7 @@ describe Ci::Pipeline do
 
     before do
       stub_licensed_features(license_scanning: true)
+      stub_feature_flags(drop_license_management_artifact: false)
     end
 
     [:license_scanning, :license_management].each do |artifact_type|

ee/spec/models/merge_request_spec.rb

@@ -123,6 +123,7 @@ describe MergeRequest do
       subject { merge_request.enabled_reports[report_type] }
 
       before do
+        stub_feature_flags(drop_license_management_artifact: false)
         stub_licensed_features({ feature => true })
       end
 

ee/spec/serializers/licenses_list_entity_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe LicensesListEntity do
-  let!(:pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
+  let!(:pipeline) { create(:ee_ci_pipeline, :with_license_scanning_report, project: project) }
   let(:license_compliance) { ::SCA::LicenseCompliance.new(project) }
 
   before do

ee/spec/serializers/licenses_list_serializer_spec.rb

@@ -11,7 +11,7 @@ describe LicensesListSerializer do
     end
 
     let(:project) { create(:project, :repository) }
-    let!(:pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
+    let!(:pipeline) { create(:ee_ci_pipeline, :with_license_scanning_report, project: project) }
     let(:license_compliance) { ::SCA::LicenseCompliance.new(project) }
     let(:user) { create(:user) }
     let(:ci_build) { create(:ee_ci_build, :success) }

ee/spec/services/ci/compare_license_scanning_reports_service_spec.rb

@@ -15,7 +15,7 @@ describe Ci::CompareLicenseScanningReportsService do
 
     context 'when head pipeline has license scanning reports' do
       let!(:base_pipeline) { nil }
-      let!(:head_pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
+      let!(:head_pipeline) { create(:ee_ci_pipeline, :with_license_scanning_report, project: project) }
 
       it 'reports new licenses' do
         expect(subject[:status]).to eq(:parsed)
@@ -25,8 +25,8 @@ describe Ci::CompareLicenseScanningReportsService do
     end
 
     context 'when base and head pipelines have test reports' do
-      let!(:base_pipeline) { create(:ee_ci_pipeline, :with_license_management_report, project: project) }
-      let!(:head_pipeline) { create(:ee_ci_pipeline, :with_license_management_feature_branch, project: project) }
+      let!(:base_pipeline) { create(:ee_ci_pipeline, :with_license_scanning_report, project: project) }
+      let!(:head_pipeline) { create(:ee_ci_pipeline, :with_license_scanning_feature_branch, project: project) }
 
       it 'reports status as parsed' do
         expect(subject[:status]).to eq(:parsed)
@@ -49,8 +49,8 @@ describe Ci::CompareLicenseScanningReportsService do
     end
 
     context 'when head pipeline has corrupted license scanning reports' do
-      let!(:base_pipeline) { build(:ee_ci_pipeline, :with_corrupted_license_management_report, project: project) }
-      let!(:head_pipeline) { build(:ee_ci_pipeline, :with_corrupted_license_management_report, project: project) }
+      let!(:base_pipeline) { build(:ee_ci_pipeline, :with_corrupted_license_scanning_report, project: project) }
+      let!(:head_pipeline) { build(:ee_ci_pipeline, :with_corrupted_license_scanning_report, project: project) }
 
       it 'returns status and error message' do
         expect(subject[:status]).to eq(:error)

ee/spec/services/security/sync_reports_to_approval_rules_service_spec.rb

@@ -64,6 +64,10 @@ describe Security::SyncReportsToApprovalRulesService, '#execute' do
       context "license compliance policy" do
         let!(:license_compliance_rule) { create(:report_approver_rule, :license_scanning, merge_request: merge_request, approvals_required: 1) }
 
+        before do
+          stub_feature_flags(drop_license_management_artifact: false)
+        end
+
         context "when a license violates the license compliance policy" do
           let!(:software_license_policy) { create(:software_license_policy, :denied, project: project, software_license: denied_license) }
           let(:denied_license) { create(:software_license, name: license_name) }
@@ -95,6 +99,10 @@ describe Security::SyncReportsToApprovalRulesService, '#execute' do
           context 'with an old report' do
             let!(:ci_build) { create(:ee_ci_build, :success, :license_management, pipeline: pipeline, project: project) }
 
+            before do
+              stub_feature_flags(drop_license_management_artifact: false)
+            end
+
             specify { expect { subject }.to change { license_compliance_rule.reload.approvals_required }.from(1).to(0) }
             specify { expect(subject[:status]).to be(:success) }
           end

locale/gitlab.pot

@@ -9419,6 +9419,9 @@ msgstr ""
 msgid "File deleted"
 msgstr ""
 
+msgid "File format is no longer supported"
+msgstr ""
+
 msgid "File hooks are similar to system hooks but are executed as files instead of sending data to a URL."
 msgstr ""
 

spec/models/ci/build_spec.rb

@@ -109,6 +109,10 @@ describe Ci::Build do
   describe '.with_downloadable_artifacts' do
     subject { described_class.with_downloadable_artifacts }
 
+    before do
+      stub_feature_flags(drop_license_management_artifact: false)
+    end
+
     context 'when job does not have a downloadable artifact' do
       let!(:job) { create(:ci_build) }
 
@@ -1427,6 +1431,8 @@ describe Ci::Build do
     subject { build.erase_erasable_artifacts! }
 
     before do
+      stub_feature_flags(drop_license_management_artifact: false)
+
       Ci::JobArtifact.file_types.keys.each do |file_type|
         create(:ci_job_artifact, job: build, file_type: file_type, file_format: Ci::JobArtifact::TYPE_AND_FORMAT_PAIRS[file_type.to_sym])
       end

spec/models/ci/job_artifact_spec.rb

@@ -239,9 +239,35 @@ describe Ci::JobArtifact do
     end
   end
 
+  describe 'validates if file format is supported' do
+    subject { artifact }
+
+    let(:artifact) { build(:ci_job_artifact, file_type: :license_management, file_format: :raw) }
+
+    context 'when license_management is supported' do
+      before do
+        stub_feature_flags(drop_license_management_artifact: false)
+      end
+
+      it { is_expected.to be_valid }
+    end
+
+    context 'when license_management is not supported' do
+      before do
+        stub_feature_flags(drop_license_management_artifact: true)
+      end
+
+      it { is_expected.not_to be_valid }
+    end
+  end
+
   describe 'validates file format' do
     subject { artifact }
 
+    before do
+      stub_feature_flags(drop_license_management_artifact: false)
+    end
+
     described_class::TYPE_AND_FORMAT_PAIRS.except(:trace).each do |file_type, file_format|
       context "when #{file_type} type with #{file_format} format" do
         let(:artifact) { build(:ci_job_artifact, file_type: file_type, file_format: file_format) }

spec/services/ci/retry_build_service_spec.rb

@@ -63,6 +63,9 @@ describe Ci::RetryBuildService do
     end
 
     before do
+      # Test correctly behaviour of deprecated artifact because it can be still in use
+      stub_feature_flags(drop_license_management_artifact: false)
+
       # Make sure that build has both `stage_id` and `stage` because FactoryBot
       # can reset one of the fields when assigning another. We plan to deprecate
       # and remove legacy `stage` column in the future.