Merge branch 'remove-ci-migration' into 'master'

Remove CI migration

See merge request !1656

CHANGELOG

@@ -19,6 +19,7 @@ v 8.1.0 (unreleased)
   - Allow removing of project without confirmation when JavaScript is disabled (Stan Hu)
   - Support filtering by "Any" milestone or issue and fix "No Milestone" and "No Label" filters (Stan Hu)
   - Improved performance of the trending projects page
+  - Remove CI migration task
   - Improved performance of finding projects by their namespace
   - Fix bug where transferring a project would result in stale commit links (Stan Hu)
   - Include full path of source and target branch names in New Merge Request page (Stan Hu)

lib/ci/migrate/builds.rb

-module Ci
-  module Migrate
-    class Builds
-      attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
-
-      def initialize
-        @app_builds_dir = Settings.gitlab_ci.builds_path
-        @backup_dir = Gitlab.config.backup.path
-        @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
-      end
-
-      def restore
-        backup_existing_builds_dir
-
-        FileUtils.mkdir_p(app_builds_dir, mode: 0700)
-        unless system('tar', '-C', app_builds_dir, '-zxf', backup_builds_tarball)
-          abort 'Restore failed'.red
-        end
-      end
-
-      def backup_existing_builds_dir
-        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
-        if File.exists?(app_builds_dir)
-          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
-        end
-      end
-    end
-  end
-end

lib/ci/migrate/database.rb

-require 'yaml'
-
-module Ci
-  module Migrate
-    class Database
-      attr_reader :config
-
-      def initialize
-        @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
-      end
-
-      def restore
-        decompress_rd, decompress_wr = IO.pipe
-        decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
-        decompress_wr.close
-
-        restore_pid = case config["adapter"]
-                        when /^mysql/ then
-                          $progress.print "Restoring MySQL database #{config['database']} ... "
-                          # Workaround warnings from MySQL 5.6 about passwords on cmd line
-                          ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-                          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
-                        when "postgresql" then
-                          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
-                          pg_env
-                          spawn('psql', config['database'], in: decompress_rd)
-                      end
-        decompress_rd.close
-
-        success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
-        abort 'Restore failed' unless success
-      end
-
-      protected
-
-      def db_file_name
-        File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
-      end
-
-      def mysql_args
-        args = {
-          'host' => '--host',
-          'port' => '--port',
-          'socket' => '--socket',
-          'username' => '--user',
-          'encoding' => '--default-character-set'
-        }
-        args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
-      end
-
-      def pg_env
-        ENV['PGUSER'] = config["username"] if config["username"]
-        ENV['PGHOST'] = config["host"] if config["host"]
-        ENV['PGPORT'] = config["port"].to_s if config["port"]
-        ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
-      end
-
-      def report_success(success)
-        if success
-          puts '[DONE]'.green
-        else
-          puts '[FAILED]'.red
-        end
-      end
-    end
-  end
-end

lib/ci/migrate/manager.rb

-module Ci
-  module Migrate
-    class Manager
-      CI_IMPORT_PREFIX = '8.0' # Only allow imports from CI 8.0.x
-
-      def cleanup
-        $progress.print "Deleting tmp directories ... "
-
-        backup_contents.each do |dir|
-          next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
-
-          if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
-            $progress.puts "done".green
-          else
-            puts "deleting tmp directory '#{dir}' failed".red
-            abort 'Backup failed'
-          end
-        end
-      end
-
-      def unpack
-        Dir.chdir(Gitlab.config.backup.path)
-
-        # check for existing backups in the backup dir
-        file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
-        puts "no backups found" if file_list.count == 0
-
-        if file_list.count > 1 && ENV["BACKUP"].nil?
-          puts "Found more than one backup, please specify which one you want to restore:"
-          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
-          exit 1
-        end
-
-        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
-
-        unless File.exists?(tar_file)
-          puts "The specified CI backup doesn't exist!"
-          exit 1
-        end
-
-        $progress.print "Unpacking backup ... "
-
-        unless Kernel.system(*%W(tar -xf #{tar_file}))
-          puts "unpacking backup failed".red
-          exit 1
-        else
-          $progress.puts "done".green
-        end
-
-        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
-
-        # restoring mismatching backups can lead to unexpected problems
-        if !settings[:gitlab_version].start_with?(CI_IMPORT_PREFIX)
-          puts "GitLab CI version mismatch:".red
-          puts "  Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
-          exit 1
-        end
-      end
-
-      private
-
-      def backup_contents
-        ["db", "builds", "backup_information.yml"]
-      end
-
-      def settings
-        @settings ||= YAML.load_file("backup_information.yml")
-      end
-    end
-  end
-end
-

lib/ci/migrate/tags.rb

-require 'yaml'
-
-module Ci
-  module Migrate
-    class Tags
-      def restore
-        puts 'Inserting tags...'
-        connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
-          begin
-            connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
-          rescue ActiveRecord::RecordNotUnique
-          end
-        end
-
-        ActiveRecord::Base.transaction do
-          puts 'Deleting old taggings...'
-          connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
-
-          puts 'Inserting taggings...'
-          connection.execute(
-            'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
-              "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
-              'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
-              'JOIN tags ON tags.name = ci_tags.name '
-          )
-
-          puts 'Resetting counters... '
-          connection.execute(
-            'UPDATE tags SET ' +
-              'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
-          )
-        end
-      end
-
-      protected
-
-      def connection
-        ActiveRecord::Base.connection
-      end
-    end
-  end
-end

lib/tasks/ci/migrate.rake

-namespace :ci do
-  desc 'GitLab | Import and migrate CI database'
-  task migrate: :environment do
-    warn_user_is_not_gitlab
-    configure_cron_mode
-
-    unless ENV['force'] == 'yes'
-      puts 'This will remove all CI related data and restore it from the provided backup.'
-      ask_to_continue
-      puts ''
-    end
-
-    # disable CI for time of migration
-    enable_ci(false)
-
-    # unpack archives
-    migrate = Ci::Migrate::Manager.new
-    migrate.unpack
-
-    Rake::Task['ci:migrate:db'].invoke
-    Rake::Task['ci:migrate:builds'].invoke
-    Rake::Task['ci:migrate:tags'].invoke
-    Rake::Task['ci:migrate:services'].invoke
-
-    # enable CI for time of migration
-    enable_ci(true)
-
-    migrate.cleanup
-  end
-
-  namespace :migrate do
-    desc 'GitLab | Import CI database'
-    task db: :environment do
-      configure_cron_mode
-      $progress.puts 'Restoring database ... '.blue
-      Ci::Migrate::Database.new.restore
-      $progress.puts 'done'.green
-    end
-
-    desc 'GitLab | Import CI builds'
-    task builds: :environment do
-      configure_cron_mode
-      $progress.puts 'Restoring builds ... '.blue
-      Ci::Migrate::Builds.new.restore
-      $progress.puts 'done'.green
-    end
-
-    desc 'GitLab | Migrate CI tags'
-    task tags: :environment do
-      configure_cron_mode
-      $progress.puts 'Migrating tags ... '.blue
-      ::Ci::Migrate::Tags.new.restore
-      $progress.puts 'done'.green
-    end
-
-    desc 'GitLab | Migrate CI auto-increments'
-    task autoincrements: :environment do
-      c = ActiveRecord::Base.connection
-      c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-        result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1")
-        if result
-          ai_val = result['id'].to_i + 1
-          puts "Resetting auto increment ID for #{table} to #{ai_val}"
-          if c.adapter_name == 'PostgreSQL'
-            c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}")
-          else
-            c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}")
-          end
-        end
-      end
-    end
-
-    desc 'GitLab | Migrate CI services'
-    task services: :environment do
-      $progress.puts 'Migrating services ... '.blue
-      c = ActiveRecord::Base.connection
-      c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
-      $progress.puts 'done'.green
-    end
-  end
-
-  def enable_ci(enabled)
-    settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
-    settings.ci_enabled = enabled
-    settings.save!
-  end
-end