Merge branch 'export-rake-task' into 'master'

Export rake task

Closes #207847

See merge request gitlab-org/gitlab!25598

lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module AfterExportStrategies
+      class MoveFileStrategy < BaseAfterExportStrategy
+        def initialize(archive_path:)
+          @archive_path = archive_path
+        end
+
+        private
+
+        def strategy_execute
+          FileUtils.mv(project.export_file.path, @archive_path)
+        end
+      end
+    end
+  end
+end

lib/gitlab/profiler.rb

@@ -2,6 +2,8 @@
 
 module Gitlab
   module Profiler
+    extend WithRequestStore
+
     FILTERED_STRING = '[FILTERED]'
 
     IGNORE_BACKTRACES = %w[
@@ -58,28 +60,26 @@ module Gitlab
 
       logger = create_custom_logger(logger, private_token: private_token)
 
-      RequestStore.begin!
-
-      # Make an initial call for an asset path in development mode to avoid
-      # sprockets dominating the profiler output.
-      ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development?
+      result = with_request_store do
+        # Make an initial call for an asset path in development mode to avoid
+        # sprockets dominating the profiler output.
+        ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development?
 
-      # Rails loads internationalization files lazily the first time a
-      # translation is needed. Running this prevents this overhead from showing
-      # up in profiles.
-      ::I18n.t('.')[:test_string]
+        # Rails loads internationalization files lazily the first time a
+        # translation is needed. Running this prevents this overhead from showing
+        # up in profiles.
+        ::I18n.t('.')[:test_string]
 
-      # Remove API route mounting from the profile.
-      app.get('/api/v4/users')
+        # Remove API route mounting from the profile.
+        app.get('/api/v4/users')
 
-      result = with_custom_logger(logger) do
-        with_user(user) do
-          RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend
+        with_custom_logger(logger) do
+          with_user(user) do
+            RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend
+          end
         end
       end
 
-      RequestStore.end!
-
       log_load_times_by_model(logger)
 
       result

lib/gitlab/sidekiq_middleware/request_store_middleware.rb

@@ -3,12 +3,12 @@
 module Gitlab
   module SidekiqMiddleware
     class RequestStoreMiddleware
+      include Gitlab::WithRequestStore
+
       def call(worker, job, queue)
-        RequestStore.begin!
-        yield
-      ensure
-        RequestStore.end!
-        RequestStore.clear!
+        with_request_store do
+          yield
+        end
       end
     end
   end

lib/gitlab/utils/measuring.rb

+# frozen_string_literal: true
+
+require 'prometheus/pid_provider'
+
+module Gitlab
+  module Utils
+    class Measuring
+      def initialize(logger: Logger.new($stdout))
+        @logger = logger
+      end
+
+      def with_measuring
+        logger.info "Measuring enabled..."
+        with_gc_counter do
+          with_count_queries do
+            with_measure_time do
+              yield
+            end
+          end
+        end
+
+        logger.info "Memory usage: #{Gitlab::Metrics::System.memory_usage.to_f / 1024 / 1024} MiB"
+        logger.info "Label: #{::Prometheus::PidProvider.worker_id}"
+      end
+
+      private
+
+      attr_reader :logger
+
+      def with_count_queries(&block)
+        count = 0
+
+        counter_f = ->(_name, _started, _finished, _unique_id, payload) {
+          count += 1 unless payload[:name].in? %w[CACHE SCHEMA]
+        }
+
+        ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
+
+        logger.info "Number of sql calls: #{count}"
+      end
+
+      def with_gc_counter
+        gc_counts_before = GC.stat.select { |k, _v| k =~ /count/ }
+        yield
+        gc_counts_after = GC.stat.select { |k, _v| k =~ /count/ }
+        stats = gc_counts_before.merge(gc_counts_after) { |_k, vb, va| va - vb }
+
+        logger.info "Total GC count: #{stats[:count]}"
+        logger.info "Minor GC count: #{stats[:minor_gc_count]}"
+        logger.info "Major GC count: #{stats[:major_gc_count]}"
+      end
+
+      def with_measure_time
+        timing = Benchmark.realtime do
+          yield
+        end
+
+        logger.info "Time to finish: #{duration_in_numbers(timing)}"
+      end
+
+      def duration_in_numbers(duration_in_seconds)
+        seconds = duration_in_seconds % 1.minute
+        minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute)
+        hours = duration_in_seconds / 1.hour
+
+        if hours == 0
+          "%02d:%02d" % [minutes, seconds]
+        else
+          "%02d:%02d:%02d" % [hours, minutes, seconds]
+        end
+      end
+    end
+  end
+end

lib/gitlab/with_request_store.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module WithRequestStore
+    def with_request_store
+      RequestStore.begin!
+      yield
+    ensure
+      RequestStore.end!
+      RequestStore.clear!
+    end
+  end
+end

lib/tasks/gitlab/import_export/export.rake

+# frozen_string_literal: true
+
+require 'gitlab/with_request_store'
+
+# Export project to archive
+#
+# @example
+#   bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]"
+#
+namespace :gitlab do
+  namespace :import_export do
+    desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives'
+    task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
+      # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
+      require 'sidekiq/testing'
+
+      warn_user_is_not_gitlab
+
+      if ENV['IMPORT_DEBUG'].present?
+        ActiveRecord::Base.logger = Logger.new(STDOUT)
+        Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
+      end
+
+      GitlabProjectExport.new(
+        namespace_path: args.namespace_path,
+        project_path:   args.project_path,
+        username:       args.username,
+        file_path:      args.archive_path,
+        measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled)
+      ).export
+    end
+  end
+end
+
+class GitlabProjectExport
+  include Gitlab::WithRequestStore
+
+  def initialize(opts)
+    @project_path = opts.fetch(:project_path)
+    @file_path = opts.fetch(:file_path)
+    @current_user = User.find_by_username(opts.fetch(:username))
+    namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path))
+    @project = namespace.projects.find_by_path(@project_path)
+    @measurement_enabled = opts.fetch(:measurement_enabled)
+    @measurable = Gitlab::Utils::Measuring.new if @measurement_enabled
+  end
+
+  def export
+    validate_project
+    validate_file_path
+
+    with_export do
+      ::Projects::ImportExport::ExportService.new(project, current_user)
+        .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path))
+    end
+
+    puts 'Done!'
+  rescue StandardError => e
+    puts "Exception: #{e.message}"
+    puts e.backtrace
+    exit 1
+  end
+
+  private
+
+  attr_reader :measurable, :project, :current_user, :file_path, :project_path
+
+  def validate_project
+    unless project
+      puts "Error: Project with path: #{project_path} was not found. Please provide correct project path"
+      exit 1
+    end
+  end
+
+  def validate_file_path
+    directory = File.dirname(file_path)
+    unless Dir.exist?(directory)
+      puts "Error: Invalid file path: #{file_path}. Please provide correct file path"
+      exit 1
+    end
+  end
+
+  def with_export
+    with_request_store do
+      ::Gitlab::GitalyClient.allow_n_plus_1_calls do
+        measurement_enabled? ? measurable.with_measuring { yield } : yield
+      end
+    end
+  end
+
+  def measurement_enabled?
+    @measurement_enabled
+  end
+end

lib/tasks/gitlab/import_export/import.rake

 # frozen_string_literal: true
 
+require 'gitlab/with_request_store'
+
 # Import large project archives
 #
 # This task:
@@ -27,19 +29,22 @@ namespace :gitlab do
         project_path:   args.project_path,
         username:       args.username,
         file_path:      args.archive_path,
-        measurement_enabled: args.measurement_enabled == 'true'
+        measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled)
       ).import
     end
   end
 end
 
 class GitlabProjectImport
+  include Gitlab::WithRequestStore
+
   def initialize(opts)
     @project_path = opts.fetch(:project_path)
     @file_path    = opts.fetch(:file_path)
     @namespace    = Namespace.find_by_full_path(opts.fetch(:namespace_path))
     @current_user = User.find_by_username(opts.fetch(:username))
     @measurement_enabled = opts.fetch(:measurement_enabled)
+    @measurement = Gitlab::Utils::Measuring.new if @measurement_enabled
   end
 
   def import
@@ -49,11 +54,11 @@ class GitlabProjectImport
 
     show_import_failures_count
 
-    if @project&.import_state&.last_error
-      puts "ERROR: #{@project.import_state.last_error}"
+    if project&.import_state&.last_error
+      puts "ERROR: #{project.import_state.last_error}"
       exit 1
-    elsif @project.errors.any?
-      puts "ERROR: #{@project.errors.full_messages.join(', ')}"
+    elsif project.errors.any?
+      puts "ERROR: #{project.errors.full_messages.join(', ')}"
       exit 1
     else
       puts 'Done!'
@@ -66,60 +71,10 @@ class GitlabProjectImport
 
   private
 
-  def with_request_store
-    RequestStore.begin!
-    yield
-  ensure
-    RequestStore.end!
-    RequestStore.clear!
-  end
-
-  def with_count_queries(&block)
-    count = 0
-
-    counter_f = ->(name, started, finished, unique_id, payload) {
-      unless payload[:name].in? %w[CACHE SCHEMA]
-        count += 1
-      end
-    }
-
-    ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
-
-    puts "Number of sql calls: #{count}"
-  end
-
-  def with_gc_counter
-    gc_counts_before = GC.stat.select { |k, v| k =~ /count/ }
-    yield
-    gc_counts_after = GC.stat.select { |k, v| k =~ /count/ }
-    stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb }
-    puts "Total GC count: #{stats[:count]}"
-    puts "Minor GC count: #{stats[:minor_gc_count]}"
-    puts "Major GC count: #{stats[:major_gc_count]}"
-  end
-
-  def with_measure_time
-    timing = Benchmark.realtime do
-      yield
-    end
-
-    time = Time.at(timing).utc.strftime("%H:%M:%S")
-    puts "Time to finish: #{time}"
-  end
-
-  def with_measuring
-    puts "Measuring enabled..."
-    with_gc_counter do
-      with_count_queries do
-        with_measure_time do
-          yield
-        end
-      end
-    end
-  end
+  attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path
 
   def measurement_enabled?
-    @measurement_enabled != false
+    @measurement_enabled
   end
 
   # We want to ensure that all Sidekiq jobs are executed
@@ -135,7 +90,7 @@ class GitlabProjectImport
         # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
         # For development setups, this code-path will be excluded from n+1 detection.
         ::Gitlab::GitalyClient.allow_n_plus_1_calls do
-          measurement_enabled? ? with_measuring { yield } : yield
+          measurement_enabled? ? measurement.with_measuring { yield } : yield
         end
       end
 
@@ -158,11 +113,11 @@ class GitlabProjectImport
     # 2. Download of archive before unpacking
     disable_upload_object_storage do
       service = Projects::GitlabProjectsImportService.new(
-        @current_user,
+        current_user,
         {
-          namespace_id: @namespace.id,
-          path:         @project_path,
-          file:         File.open(@file_path)
+          namespace_id: namespace.id,
+          path:         project_path,
+          file:         File.open(file_path)
         }
       )
 
@@ -193,18 +148,18 @@ class GitlabProjectImport
   end
 
   def full_path
-    "#{@namespace.full_path}/#{@project_path}"
+    "#{namespace.full_path}/#{project_path}"
   end
 
   def show_import_start_message
-    puts "Importing GitLab export: #{@file_path} into GitLab" \
+    puts "Importing GitLab export: #{file_path} into GitLab" \
       " #{full_path}" \
-      " as #{@current_user.name}"
+      " as #{current_user.name}"
   end
 
   def show_import_failures_count
-    return unless @project.import_failures.exists?
+    return unless project.import_failures.exists?
 
-    puts "Total number of not imported relations: #{@project.import_failures.count}"
+    puts "Total number of not imported relations: #{project.import_failures.count}"
   end
 end

spec/support/shared_examples/tasks/gitlab/import_export/import_measurement_shared_examples.rb → spec/support/shared_examples/tasks/gitlab/import_export/measurable_shared_examples.rb

 # frozen_string_literal: true
 
-RSpec.shared_examples 'import measurement' do
+RSpec.shared_examples 'measurable' do
   context 'when measurement is enabled' do
     let(:measurement_enabled) { true }
 

spec/tasks/gitlab/import_export/export_rake_spec.rb

+# frozen_string_literal: true
+
+require 'rake_helper'
+
+describe 'gitlab:import_export:export rake task' do
+  let(:username) { 'root' }
+  let(:namespace_path) { username }
+  let!(:user) { create(:user, username: username) }
+  let(:measurement_enabled) { false }
+  let(:task_params) { [username, namespace_path, project_name, archive_path, measurement_enabled] }
+
+  before do
+    Rake.application.rake_require('tasks/gitlab/import_export/export')
+  end
+
+  subject { run_rake_task('gitlab:import_export:export', task_params) }
+
+  context 'when project is found' do
+    let(:project) { create(:project, creator: user, namespace: user.namespace) }
+    let(:project_name) { project.name }
+    let(:archive_path) { 'spec/fixtures/gitlab/import_export/test_project_export.tar.gz' }
+
+    around do |example|
+      example.run
+    ensure
+      File.delete(archive_path)
+    end
+
+    it 'performs project export successfully' do
+      expect { subject }.to output(/Done!/).to_stdout
+
+      expect(File).to exist(archive_path)
+    end
+
+    it_behaves_like 'measurable'
+  end
+end

spec/tasks/gitlab/import_export/import_rake_spec.rb

@@ -70,7 +70,7 @@ describe 'gitlab:import_export:import rake task' do
       subject
     end
 
-    it_behaves_like 'import measurement'
+    it_behaves_like 'measurable'
   end
 
   context 'when project import is invalid' do