wip: upload completed.

app/workers/gitlab_elt_data_dump_worker.rb

@@ -5,6 +5,16 @@ class GitlabEltDataDumpWorker
   def perform
     return unless Gitlab::CurrentSettings.elt_database_dump_enabled
 
-    Pseudonymity::Table.new.tables_to_csv
+    options = Pseudonymity::Options.new(
+      config: YAML.load_file(Rails.root.join(Gitlab.config.pseudonymizer.manifest)),
+      start_at: Time.now.utc
+    )
+
+    table = Pseudonymity::Table.new(options)
+    table.tables_to_csv
+
+    upload = Pseudonymity::UploadService.new(options)
+    upload.upload
+    upload.cleanup
   end
 end

db/schema.rb

@@ -197,8 +197,8 @@ ActiveRecord::Schema.define(version: 20180612175636) do
     t.string "external_authorization_service_url"
     t.string "external_authorization_service_default_label"
     t.boolean "pages_domain_verification_enabled", default: true, null: false
-    t.float "external_authorization_service_timeout", default: 0.5, null: false
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
+    t.float "external_authorization_service_timeout", default: 0.5
     t.text "external_auth_client_cert"
     t.text "encrypted_external_auth_client_key"
     t.string "encrypted_external_auth_client_key_iv"
@@ -206,6 +206,7 @@ ActiveRecord::Schema.define(version: 20180612175636) do
     t.string "encrypted_external_auth_client_key_pass_iv"
     t.string "email_additional_text"
     t.boolean "enforce_terms", default: false
+    t.boolean "elt_database_dump_enabled"
   end
 
   create_table "approvals", force: :cascade do |t|
@@ -1631,6 +1632,7 @@ ActiveRecord::Schema.define(version: 20180612175636) do
     t.text "title_html"
     t.text "description_html"
     t.integer "time_estimate"
+    t.boolean "squash", default: false, null: false
     t.integer "cached_markdown_version"
     t.datetime "last_edited_at"
     t.integer "last_edited_by_id"
@@ -2020,9 +2022,9 @@ ActiveRecord::Schema.define(version: 20180612175636) do
     t.datetime "next_execution_timestamp"
     t.string "status"
     t.string "jid"
+    t.text "last_error"
     t.datetime_with_timezone "last_update_at"
     t.datetime_with_timezone "last_successful_update_at"
-    t.text "last_error"
   end
 
   add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree

lib/pseudonymity/options.rb

+module Pseudonymity
+  class Options
+    attr_reader :config
+    attr_reader :start_at
+
+    def initialize(config: {}, start_at: Time.now.utc)
+      @config = config
+      @start_at = start_at
+    end
+
+    def output_dir
+      File.join('/tmp', 'gitlab-pseudonymizer', self.start_at.iso8601)
+    end
+
+    def upload_dir
+      File.join(self.start_at.iso8601)
+    end
+  end
+end

lib/pseudonymity/table.rb

@@ -27,11 +27,11 @@ module Pseudonymity
 
   class Table
     attr_accessor :config
-    attr_accessor :output_dir
 
-    def initialize
-      @config = parse_config
-      @output_dir = ""
+    def initialize(options)
+      @config = options.config
+      @output_dir = options.output_dir
+
       @schema = {}
       @output_files = []
     end
@@ -39,8 +39,7 @@ module Pseudonymity
     def tables_to_csv
       tables = config["tables"]
 
-      @output_dir = File.join("/tmp/", SecureRandom.hex)
-      Dir.mkdir(@output_dir) unless File.directory?(@output_dir)
+      FileUtils.mkdir_p(@output_dir) unless File.directory?(@output_dir)
 
       new_tables = tables.map do |k, v|
         @schema[k] = {}
@@ -101,10 +100,6 @@ module Pseudonymity
       @schema[table]["gl_mapping_key"] = "id"
     end
 
-    def parse_config
-      YAML.load_file(Rails.root.join(Gitlab.config.pseudonymizer.manifest))
-    end
-
     def write_to_csv_file(title, contents)
       Rails.logger.info "Writing #{title} ..."
       file_path = get_and_log_file_name("csv", title)

lib/pseudonymity/upload_service.rb

@@ -2,13 +2,14 @@ module Pseudonymity
   class UploadService
     RemoteStorageUnavailableError = Class.new(StandardError)
 
-    def initialize(output_dir, progress)
-      @progress = progress
-      @output_dir = output_dir
+    def initialize(options, progress = nil)
+      @progress = progress || $stdout
+      @output_dir = options.output_dir
+      @upload_dir = options.upload_dir
     end
 
     def upload
-      progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
+      progress.puts "Uploading output files to remote storage #{remote_directory} ... "
 
       file_list.each do |file|
         upload_file(file, remote_directory)
@@ -16,11 +17,14 @@ module Pseudonymity
     end
 
     def upload_file(file, directory)
-      progress.print "\tUploading #{file} ... "
-      if directory.files.create(key: File.basename(file), body: File.open(file), public: false)
+      progress.print "\t#{file} ... "
+
+      if directory.files.create(key: File.join(@upload_dir, File.basename(file)),
+                                body: File.open(file),
+                                public: false)
         progress.puts "done".color(:green)
       else
-        puts "uploading CSV to #{remote_directory} failed".color(:red)
+        progress.puts "uploading CSV to #{remote_directory} failed".color(:red)
       end
     end
 
@@ -67,7 +71,7 @@ module Pseudonymity
     end
 
     def file_list
-      Dir[File.join(@output_dir, "*.{csv,yml}")]
+      Dir[File.join(@output_dir, "*")]
     end
   end
 end

lib/tasks/gitlab/db.rake

@@ -76,10 +76,15 @@ namespace :gitlab do
 
     desc 'Output pseudonymity dump of selected tables'
     task pseudonymity_dump: :environment do
-      table = Pseudonymity::Table.new
+      options = Pseudonymity::Options.new(
+        config: YAML.load_file(Rails.root.join(Gitlab.config.pseudonymizer.manifest)),
+        start_at: Time.now.utc
+      )
+
+      table = Pseudonymity::Table.new(options)
       table.tables_to_csv
 
-      upload = Pseudonymity::UploadService.new(table.output_dir, progress)
+      upload = Pseudonymity::UploadService.new(options, progress)
       upload.upload
       upload.cleanup
     end

rakes.patch

+diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
+new file mode 100644
+index 00000000000..bfca4bfb3f7
+--- /dev/null
++++ b/lib/tasks/gitlab/artifacts/migrate.rake
+@@ -0,0 +1,25 @@
++require 'logger'
++require 'resolv-replace'
++
++desc "GitLab | Migrate files for artifacts to comply with new storage format"
++namespace :gitlab do
++  namespace :artifacts do
++    task migrate: :environment do
++      logger = Logger.new(STDOUT)
++      logger.info('Starting transfer of artifacts')
++
++      Ci::Build.joins(:project)
++        .with_artifacts_stored_locally
++        .find_each(batch_size: 10) do |build|
++        begin
++          build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
++          build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
++
++          logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
++        rescue => e
++          logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
++        end
++      end
++    end
++  end
++end
+diff --git a/lib/tasks/gitlab/exclusive_lease.rake b/lib/tasks/gitlab/exclusive_lease.rake
+new file mode 100644
+index 00000000000..83722bf6d94
+--- /dev/null
++++ b/lib/tasks/gitlab/exclusive_lease.rake
+@@ -0,0 +1,9 @@
++namespace :gitlab do
++  namespace :exclusive_lease do
++    desc 'GitLab | Clear existing exclusive leases for specified scope (default: *)'
++    task :clear, [:scope] => [:environment] do |_, args|
++      args[:scope].nil? ? Gitlab::ExclusiveLease.reset_all! : Gitlab::ExclusiveLease.reset_all!(args[:scope])
++      puts 'All exclusive lease entries were removed.'
++    end
++  end
++end
+diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
+new file mode 100644
+index 00000000000..a45e5ca91e0
+--- /dev/null
++++ b/lib/tasks/gitlab/lfs/migrate.rake
+@@ -0,0 +1,22 @@
++require 'logger'
++
++desc "GitLab | Migrate LFS objects to remote storage"
++namespace :gitlab do
++  namespace :lfs do
++    task migrate: :environment do
++      logger = Logger.new(STDOUT)
++      logger.info('Starting transfer of LFS files to object storage')
++
++      LfsObject.with_files_stored_locally
++        .find_each(batch_size: 10) do |lfs_object|
++          begin
++            lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
++
++            logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
++          rescue => e
++            logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
++          end
++        end
++    end
++  end
++end
+diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
+index 1ce2eedb89c..56f3a916c87 100644
+--- a/lib/tasks/gitlab/shell.rake
++++ b/lib/tasks/gitlab/shell.rake
+@@ -69,7 +69,7 @@ namespace :gitlab do
+         if File.exist?(path_to_repo)
+           print '-'
+         else
+-          if Gitlab::Shell.new.add_repository(project.repository_storage,
++          if Gitlab::Shell.new.create_repository(project.repository_storage,
+                                               project.disk_path)
+             print '.'
+           else
+diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
+new file mode 100644
+index 00000000000..c26c3ccb3be
+--- /dev/null
++++ b/lib/tasks/gitlab/uploads/migrate.rake
+@@ -0,0 +1,33 @@
++namespace :gitlab do
++  namespace :uploads do
++    desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
++    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
++      batch_size     = ENV.fetch('BATCH', 200).to_i
++      @to_store      = ObjectStorage::Store::REMOTE
++      @mounted_as    = args.mounted_as&.gsub(':', '')&.to_sym
++      @uploader_class = args.uploader_class.constantize
++      @model_class    = args.model_class.constantize
++
++      uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
++    end
++
++    def enqueue_batch(batch, index)
++      job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
++                                                         @mounted_as,
++                                                         @to_store)
++      puts "Enqueued job ##{index}: #{job}"
++    rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
++      # continue for the next batch
++      puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
++    end
++
++    def uploads
++      Upload.class_eval { include EachBatch } unless Upload < EachBatch
++
++      Upload
++        .where.not(store: @to_store)
++        .where(uploader: @uploader_class.to_s,
++               model_type: @model_class.base_class.sti_name)
++    end
++  end
++end