Merge branch 'secure-workers-use-secure-queue' into 'master'

Secure workers use a dedicated queue

See merge request gitlab-org/gitlab!24340

app/workers/concerns/security_scans_queue.rb

+# frozen_string_literal: true
+
+##
+# Concern for setting Sidekiq settings for the various Secure product queues
+#
+module SecurityScansQueue
+  extend ActiveSupport::Concern
+
+  included do
+    queue_namespace :security_scans
+    feature_category :static_application_security_testing
+  end
+end

db/post_migrate/20200213220159_migrate_store_security_reports_sidekiq_queue.rb

+# frozen_string_literal: true
+
+class MigrateStoreSecurityReportsSidekiqQueue < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    sidekiq_queue_migrate 'pipeline_default:store_security_reports', to: 'security_scans:store_security_reports'
+  end
+
+  def down
+    sidekiq_queue_migrate 'security_scans:store_security_reports', to: 'pipeline_default:store_security_reports'
+  end
+end

db/post_migrate/20200213220211_migrate_sync_security_reports_to_report_approval_rules_sidekiq_queue.rb

+# frozen_string_literal: true
+
+class MigrateSyncSecurityReportsToReportApprovalRulesSidekiqQueue < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    sidekiq_queue_migrate 'pipeline_default:sync_security_reports_to_report_approval_rules',
+                          to: 'security_scans:sync_security_reports_to_report_approval_rules'
+  end
+
+  def down
+    sidekiq_queue_migrate 'security_scans:sync_security_reports_to_report_approval_rules',
+                          to: 'pipeline_default:sync_security_reports_to_report_approval_rules'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_02_13_204737) do
+ActiveRecord::Schema.define(version: 2020_02_13_220211) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"

ee/app/workers/all_queues.yml

@@ -339,24 +339,24 @@
   :latency_sensitive: 
   :resource_boundary: :unknown
   :weight: 1
-- :name: pipeline_default:store_security_reports
-  :feature_category: :continuous_integration
+- :name: security_scans:store_security_reports
+  :feature_category: :static_application_security_testing
   :has_external_dependencies: 
   :latency_sensitive: 
   :resource_boundary: :unknown
-  :weight: 3
-- :name: pipeline_default:sync_security_reports_to_report_approval_rules
-  :feature_category: :static_application_security_testing
-  :has_external_dependencies: 
-  :latency_sensitive: true
-  :resource_boundary: :cpu
-  :weight: 3
+  :weight: 2
 - :name: security_scans:store_security_scans
   :feature_category: :static_application_security_testing
   :has_external_dependencies: 
   :latency_sensitive: 
   :resource_boundary: :unknown
   :weight: 2
+- :name: security_scans:sync_security_reports_to_report_approval_rules
+  :feature_category: :static_application_security_testing
+  :has_external_dependencies: 
+  :latency_sensitive: true
+  :resource_boundary: :cpu
+  :weight: 2
 - :name: adjourned_project_deletion
   :feature_category: :authentication_and_authorization
   :has_external_dependencies: 

ee/app/workers/store_security_reports_worker.rb

@@ -4,7 +4,7 @@
 #
 class StoreSecurityReportsWorker
   include ApplicationWorker
-  include PipelineQueue
+  include SecurityScansQueue
 
   def perform(pipeline_id)
     Ci::Pipeline.find(pipeline_id).try do |pipeline|

ee/app/workers/store_security_scans_worker.rb

@@ -2,9 +2,7 @@
 
 class StoreSecurityScansWorker
   include ApplicationWorker
-
-  queue_namespace :security_scans
-  feature_category :static_application_security_testing
+  include SecurityScansQueue
 
   # rubocop: disable CodeReuse/ActiveRecord
   def perform(build_id)

ee/app/workers/sync_security_reports_to_report_approval_rules_worker.rb

@@ -4,9 +4,8 @@
 #
 class SyncSecurityReportsToReportApprovalRulesWorker
   include ApplicationWorker
-  include PipelineQueue
+  include SecurityScansQueue
 
-  feature_category :static_application_security_testing
   latency_sensitive_worker!
   worker_resource_boundary :cpu
 

spec/migrations/migrate_store_security_reports_sidekiq_queue_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200213220159_migrate_store_security_reports_sidekiq_queue.rb')
+
+describe MigrateStoreSecurityReportsSidekiqQueue, :redis do
+  include Gitlab::Database::MigrationHelpers
+  include StubWorker
+
+  context 'when there are jobs in the queue' do
+    it 'migrates queue when migrating up' do
+      Sidekiq::Testing.disable! do
+        stub_worker(queue: 'pipeline_default:store_security_reports').perform_async(1, 5)
+
+        described_class.new.up
+
+        expect(sidekiq_queue_length('pipeline_default:store_security_reports')).to eq 0
+        expect(sidekiq_queue_length('security_scans:store_security_reports')).to eq 1
+      end
+    end
+
+    it 'migrates queue when migrating down' do
+      Sidekiq::Testing.disable! do
+        stub_worker(queue: 'security_scans:store_security_reports').perform_async(1, 5)
+
+        described_class.new.down
+
+        expect(sidekiq_queue_length('pipeline_default:store_security_reports')).to eq 1
+        expect(sidekiq_queue_length('security_scans:store_security_reports')).to eq 0
+      end
+    end
+  end
+end

spec/migrations/migrate_sync_security_reports_to_report_approval_rules_sidekiq_queue_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200213220211_migrate_sync_security_reports_to_report_approval_rules_sidekiq_queue.rb')
+
+describe MigrateSyncSecurityReportsToReportApprovalRulesSidekiqQueue, :redis do
+  include Gitlab::Database::MigrationHelpers
+  include StubWorker
+
+  context 'when there are jobs in the queue' do
+    it 'migrates queue when migrating up' do
+      Sidekiq::Testing.disable! do
+        stub_worker(queue: 'pipeline_default:sync_security_reports_to_report_approval_rules').perform_async(1, 5)
+
+        described_class.new.up
+
+        expect(sidekiq_queue_length('pipeline_default:sync_security_reports_to_report_approval_rules')).to eq 0
+        expect(sidekiq_queue_length('security_scans:sync_security_reports_to_report_approval_rules')).to eq 1
+      end
+    end
+
+    it 'migrates queue when migrating down' do
+      Sidekiq::Testing.disable! do
+        stub_worker(queue: 'security_scans:sync_security_reports_to_report_approval_rules').perform_async(1, 5)
+
+        described_class.new.down
+
+        expect(sidekiq_queue_length('pipeline_default:sync_security_reports_to_report_approval_rules')).to eq 1
+        expect(sidekiq_queue_length('security_scans:sync_security_reports_to_report_approval_rules')).to eq 0
+      end
+    end
+  end
+end