Merge branch '321361-move-file-in-object-storage' into 'master'

Refactor how nuget package files are updated

See merge request gitlab-org/gitlab!66728

app/models/packages/package_file.rb

@@ -10,6 +10,9 @@ class Packages::PackageFile < ApplicationRecord
 
   belongs_to :package
 
+  # used to move the linked file within object storage
+  attribute :new_file_path, default: nil
+
   has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum'
   has_many :package_file_build_infos, inverse_of: :package_file, class_name: 'Packages::PackageFileBuildInfo'
   has_many :pipelines, through: :package_file_build_infos
@@ -80,6 +83,12 @@ class Packages::PackageFile < ApplicationRecord
 
   before_save :update_size_from_file
 
+  # if a new_file_path is provided, we need
+  # * disable the remove_previously_stored_file callback so that carrierwave doesn't take care of the file
+  # * enable a new after_commit callback that will move the file in object storage
+  skip_callback :commit, :after, :remove_previously_stored_file, if: :execute_move_in_object_storage?
+  after_commit :move_in_object_storage, if: :execute_move_in_object_storage?
+
   def download_path
     Gitlab::Routing.url_helpers.download_project_package_file_path(project, self)
   end
@@ -89,6 +98,17 @@ class Packages::PackageFile < ApplicationRecord
   def update_size_from_file
     self.size ||= file.size
   end
+
+  def execute_move_in_object_storage?
+    !file.file_storage? && new_file_path?
+  end
+
+  def move_in_object_storage
+    carrierwave_file = file.file
+
+    carrierwave_file.copy_to(new_file_path)
+    carrierwave_file.delete
+  end
 end
 
 Packages::PackageFile.prepend_mod_with('Packages::PackageFile')

app/services/packages/nuget/update_package_from_metadata_service.rb

@@ -21,6 +21,43 @@ module Packages
 
         try_obtain_lease do
           @package_file.transaction do
+            if use_new_package_file_updater?
+              new_execute
+            else
+              legacy_execute
+            end
+          end
+        end
+      rescue ActiveRecord::RecordInvalid => e
+        raise InvalidMetadataError, e.message
+      end
+
+      private
+
+      def new_execute
+        package_to_destroy = nil
+        target_package = @package_file.package
+
+        if existing_package
+          package_to_destroy = @package_file.package
+          target_package = existing_package
+        else
+          if symbol_package?
+            raise InvalidMetadataError, 'symbol package is invalid, matching package does not exist'
+          end
+
+          update_linked_package
+        end
+
+        update_package(target_package)
+
+        ::Packages::UpdatePackageFileService.new(@package_file, package_id: target_package.id, file_name: package_filename)
+                                            .execute
+
+        package_to_destroy&.destroy!
+      end
+
+      def legacy_execute
         if existing_package
           package = link_to_existing_package
         elsif symbol_package?
@@ -38,12 +75,10 @@ module Packages
           file: @package_file.file
         )
       end
-        end
-      rescue ActiveRecord::RecordInvalid => e
-        raise InvalidMetadataError, e.message
-      end
 
-      private
+      def use_new_package_file_updater?
+        ::Feature.enabled?(:packages_nuget_new_package_file_updater, @package_file.project, default_enabled: :yaml)
+      end
 
       def update_package(package)
         return if symbol_package?

app/services/packages/update_package_file_service.rb

+# frozen_string_literal: true
+
+module Packages
+  class UpdatePackageFileService
+    delegate :file, to: :@package_file
+
+    def initialize(package_file, params)
+      @package_file = package_file
+      @params = params
+    end
+
+    def execute
+      check_params
+
+      return if same_as_params?
+
+      # we need to access the file *before* updating the attributes linked to its path/key.
+      file_storage_mode = file.file_storage?
+
+      @package_file.package_id = package_id if package_id
+      @package_file.file_name = file_name if file_name
+
+      if file_storage_mode
+        # package file is in mode LOCAL: we can pass the `file` to the update
+        @package_file.file = file
+      else
+        # package file is in mode REMOTE: don't pass the `file` to the update
+        # instead, pass the new file path. This will move the file
+        # in object storage.
+        @package_file.new_file_path = File.join(file.store_dir, @package_file.file_name)
+      end
+
+      @package_file.save!
+    end
+
+    private
+
+    def check_params
+      raise ArgumentError, 'package_file not persisted' unless @package_file.persisted?
+      raise ArgumentError, 'package_id and file_name are blank' if package_id.blank? && file_name.blank?
+    end
+
+    def same_as_params?
+      return false if package_id && package_id != @package_file.package_id
+      return false if file_name && file_name != @package_file.file_name
+
+      true
+    end
+
+    def package_id
+      @params[:package_id]
+    end
+
+    def file_name
+      @params[:file_name]
+    end
+  end
+end

config/feature_flags/development/packages_nuget_new_package_file_updater.yml

+---
+name: packages_nuget_new_package_file_updater
+introduced_by_url:
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/336511
+milestone: '14.1'
+type: development
+group: group::package
+default_enabled: false

spec/models/packages/package_file_spec.rb

@@ -159,4 +159,71 @@ RSpec.describe Packages::PackageFile, type: :model do
       expect { subject }.to change { package_file.size }.from(nil).to(3513)
     end
   end
+
+  context 'update callbacks' do
+    subject { package_file.save! }
+
+    shared_examples 'executing the default callback' do
+      it 'executes the default callback' do
+        expect(package_file).to receive(:remove_previously_stored_file)
+        expect(package_file).not_to receive(:move_in_object_storage)
+
+        subject
+      end
+    end
+
+    context 'with object storage disabled' do
+      let(:package_file) { create(:package_file, file_name: 'file_name.txt') }
+
+      before do
+        stub_package_file_object_storage(enabled: false)
+      end
+
+      it_behaves_like 'executing the default callback'
+
+      context 'with new_file_path set' do
+        before do
+          package_file.new_file_path = 'test'
+        end
+
+        it_behaves_like 'executing the default callback'
+      end
+    end
+
+    context 'with object storage enabled' do
+      let(:package_file) do
+        create(
+          :package_file,
+          file_name: 'file_name.txt',
+          file: CarrierWaveStringFile.new_file(
+            file_content: 'content',
+            filename: 'file_name.txt',
+            content_type: 'text/plain'
+          ),
+          file_store: ::Packages::PackageFileUploader::Store::REMOTE
+        )
+      end
+
+      before do
+        stub_package_file_object_storage(enabled: true)
+      end
+
+      it_behaves_like 'executing the default callback'
+
+      context 'with new_file_path set' do
+        before do
+          package_file.new_file_path = 'test'
+        end
+
+        it 'executes the move_in_object_storage callback' do
+          expect(package_file).not_to receive(:remove_previously_stored_file)
+          expect(package_file).to receive(:move_in_object_storage).and_call_original
+          expect(package_file.file.file).to receive(:copy_to).and_call_original
+          expect(package_file.file.file).to receive(:delete).and_call_original
+
+          subject
+        end
+      end
+    end
+  end
 end

spec/services/packages/nuget/update_package_from_metadata_service_spec.rb

@@ -49,7 +49,7 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
           allow(service).to receive(:metadata).and_return(metadata)
         end
 
-        it 'does not update the package' do
+        it 'does not update the package', :aggregate_failures do
           expect(service).to receive(:try_obtain_lease).and_call_original
 
           expect { subject }
@@ -63,12 +63,13 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
       end
     end
 
+    shared_examples 'handling all conditions' do
       context 'with no existing package' do
         let(:package_id) { package.id }
 
-      it 'updates package and package file' do
+        it 'updates package and package file', :aggregate_failures do
           expect { subject }
-          .to change { ::Packages::Package.count }.by(1)
+            .to not_change { ::Packages::Package.count }
             .and change { Packages::Dependency.count }.by(1)
             .and change { Packages::DependencyLink.count }.by(1)
             .and change { ::Packages::Nuget::Metadatum.count }.by(0)
@@ -90,7 +91,7 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
         let!(:existing_package) { create(:nuget_package, project: package.project, name: package_name, version: package_version) }
         let(:package_id) { existing_package.id }
 
-      it 'link existing package and updates package file' do
+        it 'link existing package and updates package file', :aggregate_failures do
           expect(service).to receive(:try_obtain_lease).and_call_original
 
           expect { subject }
@@ -138,9 +139,9 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
           end
         end
 
-      it 'creates nuget metadatum' do
+        it 'creates nuget metadatum', :aggregate_failures do
           expect { subject }
-          .to change { ::Packages::Package.count }.by(1)
+            .to not_change { ::Packages::Package.count }
             .and change { ::Packages::Nuget::Metadatum.count }.by(1)
 
           metadatum = package_file.reload.package.nuget_metadatum
@@ -175,9 +176,9 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
           end
         end
 
-      it 'updates package and package file' do
+        it 'updates package and package file', :aggregate_failures do
           expect { subject }
-          .to change { ::Packages::Package.count }.by(1)
+            .to not_change { ::Packages::Package.count }
             .and change { Packages::Dependency.count }.by(4)
             .and change { Packages::DependencyLink.count }.by(4)
             .and change { Packages::Nuget::DependencyLinkMetadatum.count }.by(2)
@@ -272,4 +273,24 @@ RSpec.describe Packages::Nuget::UpdatePackageFromMetadataService, :clean_gitlab_
         end
       end
     end
+
+    context 'with packages_nuget_new_package_file_updater enabled' do
+      before do
+        expect(service).not_to receive(:legacy_execute)
+      end
+
+      it_behaves_like 'handling all conditions'
+    end
+
+    context 'with packages_nuget_new_package_file_updater disabled' do
+      before do
+        stub_feature_flags(packages_nuget_new_package_file_updater: false)
+        expect(::Packages::UpdatePackageFileService)
+          .not_to receive(:new).with(package_file, instance_of(Hash)).and_call_original
+        expect(service).not_to receive(:new_execute)
+      end
+
+      it_behaves_like 'handling all conditions'
+    end
+  end
 end

spec/services/packages/update_package_file_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Packages::UpdatePackageFileService do
+  let_it_be(:another_package) { create(:package) }
+  let_it_be(:old_file_name) { 'old_file_name.txt' }
+  let_it_be(:new_file_name) { 'new_file_name.txt' }
+
+  let(:package) { package_file.package }
+  let(:params) { { package_id: another_package.id, file_name: new_file_name } }
+  let(:service) { described_class.new(package_file, params) }
+
+  describe '#execute' do
+    subject { service.execute }
+
+    shared_examples 'updating package file with valid parameters' do
+      context 'with both parameters set' do
+        it 'updates the package file accordingly' do
+          expect { subject }
+            .to change { package.package_files.count }.from(1).to(0)
+            .and change { another_package.package_files.count }.from(0).to(1)
+            .and change { package_file.package_id }.from(package.id).to(another_package.id)
+            .and change { package_file.file_name }.from(old_file_name).to(new_file_name)
+        end
+      end
+
+      context 'with only file_name set' do
+        let(:params) { { file_name: new_file_name } }
+
+        it 'updates the package file accordingly' do
+          expect { subject }
+            .to not_change { package.package_files.count }
+            .and not_change { another_package.package_files.count }
+            .and not_change { package_file.package_id }
+            .and change { package_file.file_name }.from(old_file_name).to(new_file_name)
+        end
+      end
+
+      context 'with only package_id set' do
+        let(:params) { { package_id: another_package.id } }
+
+        it 'updates the package file accordingly' do
+          expect { subject }
+            .to change { package.package_files.count }.from(1).to(0)
+            .and change { another_package.package_files.count }.from(0).to(1)
+            .and change { package_file.package_id }.from(package.id).to(another_package.id)
+            .and not_change { package_file.file_name }
+        end
+      end
+    end
+
+    shared_examples 'not updating package with invalid parameters' do
+      context 'with blank parameters' do
+        let(:params) { {} }
+
+        it 'raise an argument error' do
+          expect { subject }.to raise_error(ArgumentError, 'package_id and file_name are blank')
+        end
+      end
+
+      context 'with non persisted package file' do
+        let(:package_file) { build(:package_file) }
+
+        it 'raise an argument error' do
+          expect { subject }.to raise_error(ArgumentError, 'package_file not persisted')
+        end
+      end
+    end
+
+    context 'with object storage disabled' do
+      let(:package_file) { create(:package_file, file_name: old_file_name) }
+
+      before do
+        stub_package_file_object_storage(enabled: false)
+      end
+
+      it_behaves_like 'updating package file with valid parameters' do
+        before do
+          expect(package_file).to receive(:remove_previously_stored_file).and_call_original
+          expect(package_file).not_to receive(:move_in_object_storage)
+        end
+      end
+
+      it_behaves_like 'not updating package with invalid parameters'
+    end
+
+    context 'with object storage enabled' do
+      let(:package_file) do
+        create(
+          :package_file,
+          file_name: old_file_name,
+          file: CarrierWaveStringFile.new_file(
+            file_content: 'content',
+            filename: old_file_name,
+            content_type: 'text/plain'
+          ),
+          file_store: ::Packages::PackageFileUploader::Store::REMOTE
+        )
+      end
+
+      before do
+        stub_package_file_object_storage(enabled: true)
+      end
+
+      it_behaves_like 'updating package file with valid parameters' do
+        before do
+          expect(package_file).not_to receive(:remove_previously_stored_file)
+          expect(package_file).to receive(:move_in_object_storage).and_call_original
+        end
+      end
+
+      it_behaves_like 'not updating package with invalid parameters' do
+        before do
+          expect(package_file.file.file).not_to receive(:copy_to)
+        end
+      end
+    end
+  end
+end