Merge branch 'add-ci_variables-environment_scope' into 'master'

Add environment_scope column to ci_variables table

See merge request !12363

changelogs/unreleased/add-ci_variables-environment_scope-mysql.yml

+---
+title: Rename duplicated variables with the same key for projects. Add environment_scope
+  column to variables and add unique constraint to make sure that no variables could
+  be created with the same key within a project
+merge_request: 12363
+author:

db/migrate/20170622135451_rename_duplicated_variable_key.rb

+class RenameDuplicatedVariableKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute(<<~SQL)
+      UPDATE ci_variables
+      SET #{key} = CONCAT(#{key}, #{underscore}, id)
+      WHERE id IN (
+        SELECT *
+        FROM ( -- MySQL requires an extra layer
+          SELECT dup.id
+          FROM ci_variables dup
+          INNER JOIN (SELECT max(id) AS id, #{key}, project_id
+                      FROM ci_variables tmp
+                      GROUP BY #{key}, project_id) var
+          USING (#{key}, project_id) where dup.id <> var.id
+        ) dummy
+      )
+    SQL
+  end
+
+  def down
+    # noop
+  end
+
+  def key
+    # key needs to be quoted in MySQL
+    quote_column_name('key')
+  end
+
+  def underscore
+    quote('_')
+  end
+end

db/migrate/20170622135628_add_environment_scope_to_ci_variables.rb

+class AddEnvironmentScopeToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:ci_variables, :environment_scope, :string, default: '*')
+  end
+
+  def down
+    remove_column(:ci_variables, :environment_scope)
+  end
+end

db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb

+class AddUniqueConstraintToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless this_index_exists?
+      add_concurrent_index(:ci_variables, columns, name: index_name, unique: true)
+    end
+  end
+
+  def down
+    if this_index_exists?
+      if Gitlab::Database.mysql? && !index_exists?(:ci_variables, :project_id)
+        # Need to add this index for MySQL project_id foreign key constraint
+        add_concurrent_index(:ci_variables, :project_id)
+      end
+
+      remove_concurrent_index(:ci_variables, columns, name: index_name)
+    end
+  end
+
+  private
+
+  def this_index_exists?
+    index_exists?(:ci_variables, columns, name: index_name)
+  end
+
+  def columns
+    @columns ||= [:project_id, :key, :environment_scope]
+  end
+
+  def index_name
+    'index_ci_variables_on_project_id_and_key_and_environment_scope'
+  end
+end

db/migrate/20170623080805_remove_ci_variables_project_id_index.rb

+class RemoveCiVariablesProjectIdIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    if index_exists?(:ci_variables, :project_id)
+      remove_concurrent_index(:ci_variables, :project_id)
+    end
+  end
+
+  def down
+    unless index_exists?(:ci_variables, :project_id)
+      add_concurrent_index(:ci_variables, :project_id)
+    end
+  end
+end

db/schema.rb

@@ -445,7 +445,7 @@ ActiveRecord::Schema.define(version: 20170627211700) do
     t.string "environment_scope", default: "*", null: false
   end
 
-  add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree
+  add_index "ci_variables", ["project_id", "key", "environment_scope"], name: "index_ci_variables_on_project_id_and_key_and_environment_scope", unique: true, using: :btree
 
   create_table "container_repositories", force: :cascade do |t|
     t.integer "project_id", null: false

spec/migrations/rename_duplicated_variable_key_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20170622135451_rename_duplicated_variable_key.rb')
+
+describe RenameDuplicatedVariableKey, :migration do
+  let(:variables) { table(:ci_variables) }
+  let(:projects) { table(:projects) }
+
+  before do
+    projects.create!(id: 1)
+    variables.create!(id: 1, key: 'key1', project_id: 1)
+    variables.create!(id: 2, key: 'key2', project_id: 1)
+    variables.create!(id: 3, key: 'keyX', project_id: 1)
+    variables.create!(id: 4, key: 'keyX', project_id: 1)
+    variables.create!(id: 5, key: 'keyY', project_id: 1)
+    variables.create!(id: 6, key: 'keyX', project_id: 1)
+    variables.create!(id: 7, key: 'key7', project_id: 1)
+    variables.create!(id: 8, key: 'keyY', project_id: 1)
+  end
+
+  it 'correctly remove duplicated records with smaller id' do
+    migrate!
+
+    expect(variables.pluck(:id, :key)).to contain_exactly(
+      [1, 'key1'],
+      [2, 'key2'],
+      [3, 'keyX_3'],
+      [4, 'keyX_4'],
+      [5, 'keyY_5'],
+      [6, 'keyX'],
+      [7, 'key7'],
+      [8, 'keyY']
+    )
+  end
+end

spec/models/ci/variable_spec.rb

@@ -5,14 +5,13 @@ describe Ci::Variable, models: true do
 
   let(:secret_value) { 'secret' }
 
-  it { is_expected.to include_module(HasVariable) }
-
   describe 'validations' do
     # EE
     before do
       stub_licensed_features(variable_environment_scope: true)
     end
 
+    it { is_expected.to include_module(HasVariable) }
     it { is_expected.to validate_presence_of(:key) }
     it { is_expected.to validate_uniqueness_of(:key).scoped_to(:project_id, :environment_scope) }
     it { is_expected.to validate_length_of(:key).is_at_most(255) }