Commit b89fa5ca authored by Yannis Roussos's avatar Yannis Roussos

Merge branch '251113-migrate-compliance-frameworks-enums' into 'master'

Migrate compliance frameworks to the new table

See merge request gitlab-org/gitlab!44290
parents b1505e24 9c07a832
# frozen_string_literal: true
class AddNamespaceColumnToFrameworks < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'idx_on_compliance_management_frameworks_namespace_id_name'
disable_ddl_transaction!
def up
unless column_exists?(:compliance_management_frameworks, :namespace_id)
add_column(:compliance_management_frameworks, :namespace_id, :integer)
end
add_concurrent_foreign_key(:compliance_management_frameworks, :namespaces, column: :namespace_id, on_delete: :cascade)
add_concurrent_index(:compliance_management_frameworks, [:namespace_id, :name], unique: true, name: INDEX_NAME)
end
def down
remove_concurrent_index_by_name(:compliance_management_frameworks, INDEX_NAME)
remove_foreign_key_if_exists(:compliance_management_frameworks, :namespaces, column: :namespace_id)
remove_column(:compliance_management_frameworks, :namespace_id)
end
end
# frozen_string_literal: true
class RemoveComplianceFrameworksGroupIdFk < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_compliance_management_frameworks_on_group_id_and_name'.freeze
class TmpComplianceFramework < ActiveRecord::Base
self.table_name = 'compliance_management_frameworks'
include EachBatch
end
disable_ddl_transaction!
def up
TmpComplianceFramework.each_batch(of: 100) do |query|
query.update_all('namespace_id = group_id') # Copy data in case we rolled back before...
end
change_column_null(:compliance_management_frameworks, :group_id, true)
remove_foreign_key_if_exists(:compliance_management_frameworks, :namespaces, column: :group_id)
remove_concurrent_index_by_name(:compliance_management_frameworks, INDEX_NAME)
end
def down
# This is just to make the rollback possible
TmpComplianceFramework.each_batch(of: 100) do |query|
query.update_all('group_id = namespace_id') # The group_id column is not in used at all
end
change_column_null(:compliance_management_frameworks, :group_id, false)
add_concurrent_foreign_key(:compliance_management_frameworks, :namespaces, column: :group_id, on_delete: :cascade)
add_concurrent_index(:compliance_management_frameworks, [:group_id, :name], unique: true, name: INDEX_NAME)
end
end
# frozen_string_literal: true
class AddFrameworkIdToProjectFrameworkSettings < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
unless column_exists?(:project_compliance_framework_settings, :framework_id)
with_lock_retries do
add_column(:project_compliance_framework_settings, :framework_id, :bigint)
end
end
add_concurrent_index(:project_compliance_framework_settings, :framework_id)
add_concurrent_foreign_key(
:project_compliance_framework_settings,
:compliance_management_frameworks,
column: :framework_id,
on_delete: :cascade
)
end
def down
remove_foreign_key_if_exists(:project_compliance_framework_settings, :compliance_management_frameworks, column: :framework_id)
with_lock_retries do
remove_column(:project_compliance_framework_settings, :framework_id)
end
end
end
# frozen_string_literal: true
class MigrateComplianceFrameworkEnumToDatabaseFrameworkRecord < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
class TmpComplianceFramework < ActiveRecord::Base
self.table_name = 'compliance_management_frameworks'
end
class TmpProjectSettings < ActiveRecord::Base
# Maps data between ComplianceManagement::ComplianceFramework::FRAMEWORKS(enum) and new ComplianceManagement::Framework model
ENUM_FRAMEWORK_MAPPING = {
1 => {
name: 'GDPR',
description: 'General Data Protection Regulation',
color: '#1aaa55'
}.freeze,
2 => {
name: 'HIPAA',
description: 'Health Insurance Portability and Accountability Act',
color: '#1f75cb'
}.freeze,
3 => {
name: 'PCI-DSS',
description: 'Payment Card Industry-Data Security Standard',
color: '#6666c4'
}.freeze,
4 => {
name: 'SOC 2',
description: 'Service Organization Control 2',
color: '#dd2b0e'
}.freeze,
5 => {
name: 'SOX',
description: 'Sarbanes-Oxley',
color: '#fc9403'
}.freeze
}.freeze
self.table_name = 'project_compliance_framework_settings'
include EachBatch
def raw_compliance_framework
# Because we have an `enum` definition in ComplianceManagement::ComplianceFramework::ProjectSettings, this is very unlikely to fail.
ENUM_FRAMEWORK_MAPPING.fetch(framework).merge(namespace_id: root_namespace_id)
end
end
def up
return unless Gitlab.ee?
TmpComplianceFramework.reset_column_information
TmpProjectSettings.reset_column_information
# This is our standard recursive namespace query, we use it to determine the root_namespace_id in the same query.
lateral_join = <<~SQL
INNER JOIN LATERAL (
WITH RECURSIVE "base_and_ancestors" AS (
(
SELECT "ns".* FROM "namespaces" as ns WHERE "ns"."id" = projects.namespace_id
) UNION
(
SELECT "ns".* FROM "namespaces" as ns, "base_and_ancestors" WHERE "ns"."id" = "base_and_ancestors"."parent_id"
)
) SELECT "namespaces".* FROM "base_and_ancestors" AS "namespaces" WHERE parent_id IS NULL LIMIT 1) AS root_namespaces ON TRUE
SQL
TmpProjectSettings.each_batch(of: 100) do |query|
project_settings_with_root_group = query
.select(:project_id, :framework, 'root_namespaces.id as root_namespace_id')
.from("(SELECT * FROM project_compliance_framework_settings) as project_compliance_framework_settings") # this is needed for the LATERAL JOIN
.joins("INNER JOIN projects on projects.id = project_compliance_framework_settings.project_id")
.joins(lateral_join)
.to_a
ActiveRecord::Base.transaction do
raw_frameworks = project_settings_with_root_group.map(&:raw_compliance_framework)
TmpComplianceFramework.insert_all(raw_frameworks.uniq) # Create compliance frameworks per group
unique_namespace_ids = project_settings_with_root_group.map(&:root_namespace_id).uniq
framework_records = TmpComplianceFramework.select(:id, :namespace_id, :name).where(namespace_id: unique_namespace_ids)
project_settings_with_root_group.each do |project_setting|
framework = framework_records.find do |record|
# name is unique within a group
record.name == project_setting.raw_compliance_framework[:name] && record[:namespace_id] == project_setting.raw_compliance_framework[:namespace_id]
end
project_setting.update_column(:framework_id, framework.id)
end
end
end
end
def down
# data migration, no-op
end
end
# frozen_string_literal: true
class AddNotNullConstraintToComplianceProjectSettings < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_not_null_constraint(:project_compliance_framework_settings, :framework_id)
change_column_null(:compliance_management_frameworks, :namespace_id, false)
end
def down
change_column_null(:compliance_management_frameworks, :namespace_id, true)
remove_not_null_constraint(:project_compliance_framework_settings, :framework_id)
end
end
550fb12fe5e180ab52bd6d012cf1869544130049e83ccbefd4b132831a074f71
\ No newline at end of file
5d9dbae8074627c41170e70849a3e6b71e20da473f312227df462c894c194efb
\ No newline at end of file
b59670c031a146e7a1a8277ba51080a6c120724a00c7612933ff1ed44bc8dd60
\ No newline at end of file
b024bc44406810a30a3aebb33dd1355468448b4ebf9c76fe7811148044241551
\ No newline at end of file
563f76e0635b54c2a5cb78cf5e87ea217acf3b6c853518588fcdeadef9dcc951
\ No newline at end of file
......@@ -11088,10 +11088,11 @@ ALTER SEQUENCE commit_user_mentions_id_seq OWNED BY commit_user_mentions.id;
CREATE TABLE compliance_management_frameworks (
id bigint NOT NULL,
group_id bigint NOT NULL,
group_id bigint,
name text NOT NULL,
description text NOT NULL,
color text NOT NULL,
namespace_id integer NOT NULL,
CONSTRAINT check_08cd34b2c2 CHECK ((char_length(color) <= 10)),
CONSTRAINT check_1617e0b87e CHECK ((char_length(description) <= 255)),
CONSTRAINT check_ab00bc2193 CHECK ((char_length(name) <= 255))
......@@ -14630,7 +14631,9 @@ ALTER SEQUENCE project_ci_cd_settings_id_seq OWNED BY project_ci_cd_settings.id;
CREATE TABLE project_compliance_framework_settings (
project_id bigint NOT NULL,
framework smallint NOT NULL
framework smallint NOT NULL,
framework_id bigint,
CONSTRAINT check_d348de9e2d CHECK ((framework_id IS NOT NULL))
);
CREATE SEQUENCE project_compliance_framework_settings_project_id_seq
......@@ -19509,6 +19512,8 @@ CREATE UNIQUE INDEX idx_metrics_users_starred_dashboard_on_user_project_dashboar
CREATE INDEX idx_mr_cc_diff_files_on_mr_cc_id_and_sha ON merge_request_context_commit_diff_files USING btree (merge_request_context_commit_id, sha);
CREATE UNIQUE INDEX idx_on_compliance_management_frameworks_namespace_id_name ON compliance_management_frameworks USING btree (namespace_id, name);
CREATE INDEX idx_packages_packages_on_project_id_name_version_package_type ON packages_packages USING btree (project_id, name, version, package_type);
CREATE UNIQUE INDEX idx_pkgs_dep_links_on_pkg_id_dependency_id_dependency_type ON packages_dependency_links USING btree (package_id, dependency_id, dependency_type);
......@@ -20039,8 +20044,6 @@ CREATE INDEX index_clusters_on_user_id ON clusters USING btree (user_id);
CREATE UNIQUE INDEX index_commit_user_mentions_on_note_id ON commit_user_mentions USING btree (note_id);
CREATE UNIQUE INDEX index_compliance_management_frameworks_on_group_id_and_name ON compliance_management_frameworks USING btree (group_id, name);
CREATE INDEX index_container_expiration_policies_on_next_run_at_and_enabled ON container_expiration_policies USING btree (next_run_at, enabled);
CREATE INDEX index_container_repositories_on_project_id ON container_repositories USING btree (project_id);
......@@ -20973,6 +20976,8 @@ CREATE UNIQUE INDEX index_project_auto_devops_on_project_id ON project_auto_devo
CREATE UNIQUE INDEX index_project_ci_cd_settings_on_project_id ON project_ci_cd_settings USING btree (project_id);
CREATE INDEX index_project_compliance_framework_settings_on_framework_id ON project_compliance_framework_settings USING btree (framework_id);
CREATE INDEX index_project_compliance_framework_settings_on_project_id ON project_compliance_framework_settings USING btree (project_id);
CREATE INDEX index_project_custom_attributes_on_key_and_value ON project_custom_attributes USING btree (key, value);
......@@ -22481,6 +22486,9 @@ ALTER TABLE ONLY project_access_tokens
ALTER TABLE ONLY protected_tag_create_access_levels
ADD CONSTRAINT fk_b4eb82fe3c FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY compliance_management_frameworks
ADD CONSTRAINT fk_b74c45b71f FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY issue_assignees
ADD CONSTRAINT fk_b7d881734a FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
......@@ -22496,6 +22504,9 @@ ALTER TABLE ONLY gitlab_subscriptions
ALTER TABLE ONLY metrics_users_starred_dashboards
ADD CONSTRAINT fk_bd6ae32fac FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
ALTER TABLE ONLY project_compliance_framework_settings
ADD CONSTRAINT fk_be413374a9 FOREIGN KEY (framework_id) REFERENCES compliance_management_frameworks(id) ON DELETE CASCADE;
ALTER TABLE ONLY snippets
ADD CONSTRAINT fk_be41fd4bb7 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
......@@ -23735,9 +23746,6 @@ ALTER TABLE ONLY requirements_management_test_reports
ALTER TABLE ONLY pool_repositories
ADD CONSTRAINT fk_rails_d2711daad4 FOREIGN KEY (source_project_id) REFERENCES projects(id) ON DELETE SET NULL;
ALTER TABLE ONLY compliance_management_frameworks
ADD CONSTRAINT fk_rails_d3240d6339 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY group_group_links
ADD CONSTRAINT fk_rails_d3a0488427 FOREIGN KEY (shared_group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
......
......@@ -9,5 +9,33 @@ module ComplianceManagement
soc_2: 4, # Service Organization Control 2
sox: 5 # Sarbanes-Oxley
}.freeze
ENUM_FRAMEWORK_MAPPING = {
FRAMEWORKS[:gdpr] => {
name: 'GDPR',
description: 'General Data Protection Regulation',
color: '#1aaa55'
}.freeze,
FRAMEWORKS[:hipaa] => {
name: 'HIPAA',
description: 'Health Insurance Portability and Accountability Act',
color: '#1f75cb'
}.freeze,
FRAMEWORKS[:pci_dss] => {
name: 'PCI-DSS',
description: 'Payment Card Industry-Data Security Standard',
color: '#6666c4'
}.freeze,
FRAMEWORKS[:soc_2] => {
name: 'SOC 2',
description: 'Service Organization Control 2',
color: '#dd2b0e'
}.freeze,
FRAMEWORKS[:sox] => {
name: 'SOX',
description: 'Sarbanes-Oxley',
color: '#fc9403'
}.freeze
}.freeze
end
end
......@@ -9,12 +9,33 @@ module ComplianceManagement
self.primary_key = :project_id
belongs_to :project
belongs_to :compliance_management_framework, class_name: "ComplianceManagement::Framework", foreign_key: :framework_id
enum framework: ::ComplianceManagement::ComplianceFramework::FRAMEWORKS
validates :project, presence: true
validates :framework, uniqueness: { scope: [:project_id] }
validates :framework, inclusion: { in: self.frameworks.keys }
before_save :ensure_compliance_framework_record
private
# Temporary callback for compatibility.
# This keeps the ComplianceManagement::Framework table in-sync with the `framework` enum column.
# At a later point the enum column will be removed so we can support custom frameworks.
def ensure_compliance_framework_record
framework_params = ComplianceManagement::ComplianceFramework::ENUM_FRAMEWORK_MAPPING[self.class.frameworks[framework]]
root_namespace = project.namespace.root_ancestor
# Framework is associated with the root group, there could be a case where the framework is already
# there. Using safe_find_or_create_by is not enough because some attributes (color) could be changed on the framework record, however
# the name is unique. For now we try to create the record and rescue RecordNotUnique error.
ComplianceManagement::Framework.create(framework_params.merge(namespace_id: root_namespace.id)) rescue ActiveRecord::RecordNotUnique
# We're sure that the framework record exists.
self.compliance_management_framework = ComplianceManagement::Framework.find_by!(namespace_id: root_namespace.id, name: framework_params[:name])
end
end
end
end
......@@ -3,17 +3,20 @@
module ComplianceManagement
class Framework < ApplicationRecord
include StripAttribute
include IgnorableColumns
self.table_name = 'compliance_management_frameworks'
ignore_columns :group_id, remove_after: '2020-12-06', remove_with: '13.7'
strip_attributes :name, :color
belongs_to :group
belongs_to :namespace
validates :group, presence: true
validates :name, presence: true, uniqueness: true, length: { maximum: 255 }
validates :namespace, presence: true
validates :name, presence: true, length: { maximum: 255 }
validates :description, presence: true, length: { maximum: 255 }
validates :color, color: true, allow_blank: false, length: { maximum: 10 }
validates :group_id, uniqueness: { scope: [:name] }
validates :namespace_id, uniqueness: { scope: :name }
end
end
......@@ -30,6 +30,8 @@ module EE
has_one :gitlab_subscription
has_one :elasticsearch_indexed_namespace
has_many :compliance_management_frameworks, class_name: "ComplianceManagement::Framework"
accepts_nested_attributes_for :gitlab_subscription, update_only: true
accepts_nested_attributes_for :namespace_limit
......
---
title: Migrate compliance framework enums to a new table
merge_request: 44290
author:
type: changed
......@@ -2,7 +2,7 @@
FactoryBot.define do
factory :compliance_framework, class: 'ComplianceManagement::Framework' do
association :group, factory: :group
namespace
name { 'GDPR' }
description { 'The General Data Protection Regulation (GDPR) is a regulation in EU law on data protection and privacy in the European Union (EU) and the European Economic Area (EEA).' }
......
......@@ -3,9 +3,13 @@
require 'spec_helper'
RSpec.describe ComplianceManagement::ComplianceFramework::ProjectSettings do
let_it_be(:group) { create(:group) }
let_it_be(:sub_group) { create(:group, parent: group) }
let_it_be(:project) { create(:project, group: sub_group) }
let(:known_frameworks) { ComplianceManagement::ComplianceFramework::ProjectSettings.frameworks.keys }
subject { build :compliance_framework_project_setting }
subject { build(:compliance_framework_project_setting, project: project) }
describe 'Associations' do
it 'belongs to project' do
......@@ -30,4 +34,20 @@ RSpec.describe ComplianceManagement::ComplianceFramework::ProjectSettings do
expect { build :compliance_framework_project_setting, framework: 'ABCDEFGH' }.to raise_error(ArgumentError).with_message(/is not a valid framework/)
end
end
describe 'creation of ComplianceManagement::Framework record' do
subject { create(:compliance_framework_project_setting, :sox, project: project) }
it 'creates a new record' do
expect(subject.reload.compliance_management_framework.name).to eq('SOX')
end
context 'when the framework record already exists for the group' do
let!(:existing_compliance_framework) { group.compliance_management_frameworks.create!(name: 'SOX', description: 'does not matter', color: '#004494') }
it 'creates a new record' do
expect(subject.reload.compliance_management_framework).to eq(existing_compliance_framework)
end
end
end
end
......@@ -8,7 +8,7 @@ RSpec.describe ComplianceManagement::Framework do
subject { framework }
it { is_expected.to validate_uniqueness_of(:group_id).scoped_to(:name) }
it { is_expected.to validate_uniqueness_of(:namespace_id).scoped_to(:name) }
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_length_of(:name).is_at_most(255) }
it { is_expected.to validate_length_of(:description).is_at_most(255) }
......
......@@ -18,6 +18,7 @@ RSpec.describe Group do
it { is_expected.to have_many(:value_streams) }
it { is_expected.to have_many(:ip_restrictions) }
it { is_expected.to have_many(:allowed_email_domains) }
it { is_expected.to have_many(:compliance_management_frameworks) }
it { is_expected.to have_one(:dependency_proxy_setting) }
it { is_expected.to have_one(:deletion_schedule) }
it { is_expected.to have_one(:group_wiki_repository) }
......
......@@ -31,6 +31,7 @@ RSpec.describe 'Database schema' do
ci_trigger_requests: %w[commit_id],
cluster_providers_aws: %w[security_group_id vpc_id access_key_id],
cluster_providers_gcp: %w[gcp_project_id operation_id],
compliance_management_frameworks: %w[group_id],
commit_user_mentions: %w[commit_id],
deploy_keys_projects: %w[deploy_key_id],
deployments: %w[deployable_id environment_id user_id],
......
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20201005094331_migrate_compliance_framework_enum_to_database_framework_record.rb')
RSpec.describe MigrateComplianceFrameworkEnumToDatabaseFrameworkRecord, schema: 20201005092753 do
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:project_compliance_framework_settings) { table(:project_compliance_framework_settings) }
let(:compliance_management_frameworks) { table(:compliance_management_frameworks) }
let(:gdpr_framework) { 1 }
let(:sox_framework) { 5 }
let!(:root_group) { namespaces.create!(type: 'Group', name: 'a', path: 'a') }
let!(:sub_group) { namespaces.create!(type: 'Group', name: 'b', path: 'b', parent_id: root_group.id) }
let!(:sub_sub_group) { namespaces.create!(type: 'Group', name: 'c', path: 'c', parent_id: sub_group.id) }
let!(:namespace) { namespaces.create!(name: 'd', path: 'd') }
let!(:project_on_root_level) { projects.create!(namespace_id: root_group.id) }
let!(:project_on_sub_sub_level_1) { projects.create!(namespace_id: sub_sub_group.id) }
let!(:project_on_sub_sub_level_2) { projects.create!(namespace_id: sub_sub_group.id) }
let!(:project_on_namespace) { projects.create!(namespace_id: namespace.id) }
let!(:project_on_root_level_compliance_setting) { project_compliance_framework_settings.create!(project_id: project_on_root_level.id, framework: gdpr_framework) }
let!(:project_on_sub_sub_level_compliance_setting_1) { project_compliance_framework_settings.create!(project_id: project_on_sub_sub_level_1.id, framework: sox_framework) }
let!(:project_on_sub_sub_level_compliance_setting_2) { project_compliance_framework_settings.create!(project_id: project_on_sub_sub_level_2.id, framework: gdpr_framework) }
let!(:project_on_namespace_level_compliance_setting) { project_compliance_framework_settings.create!(project_id: project_on_namespace.id, framework: gdpr_framework) }
subject { described_class.new.up }
context 'when Gitlab.ee? is true' do
before do
expect(Gitlab).to receive(:ee?).and_return(true)
end
it 'updates the project settings' do
subject
gdpr_framework = compliance_management_frameworks.find_by(namespace_id: root_group.id, name: 'GDPR')
expect(project_on_root_level_compliance_setting.reload.framework_id).to eq(gdpr_framework.id)
expect(project_on_sub_sub_level_compliance_setting_2.reload.framework_id).to eq(gdpr_framework.id)
sox_framework = compliance_management_frameworks.find_by(namespace_id: root_group.id, name: 'SOX')
expect(project_on_sub_sub_level_compliance_setting_1.reload.framework_id).to eq(sox_framework.id)
gdpr_framework = compliance_management_frameworks.find_by(namespace_id: namespace.id, name: 'GDPR')
expect(project_on_namespace_level_compliance_setting.reload.framework_id).to eq(gdpr_framework.id)
end
it 'adds two framework records' do
subject
expect(compliance_management_frameworks.count).to eq(3)
end
end
context 'when Gitlab.ee? is false' do
before do
expect(Gitlab).to receive(:ee?).and_return(false)
end
it 'does nothing' do
subject
expect(compliance_management_frameworks.count).to eq(0)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment