Merge branch '5980-add-ce-upgrade-ee-test' into 'master'

Add jobs to verify that migrating from CE to EE works

Closes #5980

See merge request gitlab-org/gitlab-ee!5673

.gitlab-ci.yml

@@ -278,7 +278,6 @@ stages:
   <<: *use-pg
   variables:
     SETUP_DB: "false"
-    CREATE_DB_USER: "true"
   script:
     # Manually clone gitlab-test and only seed this project in
     # db/fixtures/development/04_project.rb thanks to SIZE=1 below
@@ -322,7 +321,7 @@ stages:
 .migration-paths: &migration-paths
   <<: *dedicated-no-docs-and-no-qa-pull-cache-job
   variables:
-    CREATE_DB_USER: "true"
+    SETUP_DB: "false"
   script:
     - git fetch https://gitlab.com/gitlab-org/gitlab-ee.git v9.3.0-ee
     - git checkout -f FETCH_HEAD
@@ -331,13 +330,31 @@ stages:
     - cp config/gitlab.yml.example config/gitlab.yml
     - bundle exec rake db:drop db:create db:schema:load db:seed_fu
     - date
-    - git checkout $CI_COMMIT_SHA
+    - git checkout -f $CI_COMMIT_SHA
     - bundle install $BUNDLE_INSTALL_FLAGS
     - date
     - . scripts/prepare_build.sh
     - date
     - bundle exec rake db:migrate
 
+.migration-paths-upgrade-ce-to-ee: &migration-paths-upgrade-ce-to-ee
+  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
+  variables:
+    SETUP_DB: "false"
+  script:
+    - CE_HEAD=$(ruby -r./scripts/ee_specific_check/ee_specific_check -e'puts EESpecificCheck.fetch_remote_ce_branch')
+    - git checkout -f $CE_HEAD
+    - . scripts/utils.sh
+    - . scripts/prepare_build.sh
+    - date
+    - setup_db
+    - date
+    - git checkout -f $CI_COMMIT_SHA
+    - date
+    - . scripts/prepare_build.sh
+    - date
+    - bundle exec rake db:migrate
+
 ##
 # Trigger a package build in omnibus-gitlab repository
 #
@@ -738,6 +755,14 @@ migration:path-mysql:
   <<: *migration-paths
   <<: *use-mysql
 
+migration:upgrade-pg-ce-to-ee:
+  <<: *migration-paths-upgrade-ce-to-ee
+  <<: *use-pg
+
+migration:upgrade-mysql-ce-to-ee:
+  <<: *migration-paths-upgrade-ce-to-ee
+  <<: *use-mysql
+
 .db-rollback: &db-rollback
   <<: *dedicated-no-docs-and-no-qa-pull-cache-job
   script:

scripts/create_mysql_user.sh

 #!/bin/bash
 
 mysql --user=root --host=mysql <<EOF
-CREATE DATABASE IF NOT EXISTS gitlabhq_test DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
 CREATE USER IF NOT EXISTS 'gitlab'@'%';
 GRANT ALL PRIVILEGES ON gitlabhq_test.* TO 'gitlab'@'%';
 FLUSH PRIVILEGES;

scripts/create_postgres_user.sh

 #!/bin/bash
 
 psql -h postgres -U postgres postgres <<EOF
-DROP DATABASE IF EXISTS gitlabhq_test;
-CREATE DATABASE gitlabhq_test;
 CREATE USER gitlab;
-GRANT ALL PRIVILEGES ON DATABASE gitlabhq_test TO gitlab;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO gitlab;
 EOF

scripts/ee_specific_check/ee_specific_check.rb

@@ -23,14 +23,12 @@ module EESpecificCheck
   end
 
   def say(message)
-    puts "\n#{message}", "\n" # puts would eat trailing newline
+    warn "\n#{message}", "\n" # puts would eat trailing newline
   end
 
   def find_compare_base
     git_clean
 
-    setup_canonical_remotes
-
     ce_fetch_head = fetch_remote_ce_branch
     ce_fetch_base = run_git_command("merge-base canonical-ce/master #{ce_fetch_head}")
     ce_merge_base = run_git_command("merge-base canonical-ce/master canonical-ee/master")
@@ -57,6 +55,8 @@ module EESpecificCheck
   end
 
   def fetch_remote_ce_branch
+    setup_canonical_remotes
+
     remote_to_fetch, branch_to_fetch = find_remote_ce_branch
 
     run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet")
@@ -220,7 +220,7 @@ module EESpecificCheck
 
   def run_command(*commands)
     commands.map do |cmd|
-      puts "=> Running `#{cmd}`"
+      warn "=> Running `#{cmd}`"
 
       `#{cmd}`.strip
     end

scripts/prepare_build.sh

@@ -58,23 +58,8 @@ sed -i 's/localhost/redis/g' config/redis.queues.yml
 cp config/redis.shared_state.yml.example config/redis.shared_state.yml
 sed -i 's/localhost/redis/g' config/redis.shared_state.yml
 
-# Some tasks (e.g. db:seed_fu) need to have a properly-configured database
-# user but not necessarily a full schema loaded
-if [ "$CREATE_DB_USER" != "false" ]; then
-    if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
-        . scripts/create_postgres_user.sh
-    else
-        . scripts/create_mysql_user.sh
-    fi
-fi
-
 if [ "$SETUP_DB" != "false" ]; then
-    bundle exec rake db:drop db:create db:schema:load db:migrate
-
-    if [ "$GITLAB_DATABASE" = "mysql" ]; then
-        bundle exec rake add_limits_mysql
-    fi
-
-    # EE-only
-    bundle exec rake geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate
+    setup_db
+elif getent hosts postgres || getent hosts mysql; then
+    setup_db_user_only
 fi

scripts/utils.sh

@@ -12,3 +12,24 @@ retry() {
     done
     return 1
 }
+
+setup_db_user_only() {
+    if [ "$GITLAB_DATABASE" = "postgresql" ]; then
+        . scripts/create_postgres_user.sh
+    else
+        . scripts/create_mysql_user.sh
+    fi
+}
+
+setup_db() {
+    setup_db_user_only
+
+    bundle exec rake db:drop db:create db:schema:load db:migrate
+
+    if [ "$GITLAB_DATABASE" = "mysql" ]; then
+        bundle exec rake add_limits_mysql
+    fi
+
+    # EE-only
+    bundle exec rake geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate
+}