Move git operations for UpdateRemoteMirrorService into Gitlab::Git

ee/app/models/remote_mirror.rb

@@ -81,6 +81,10 @@ class RemoteMirror < ActiveRecord::Base
     update_status == 'started'
   end
 
+  def update_repository(options)
+    raw.update(options)
+  end
+
   def sync
     return unless enabled?
     return if Gitlab::Geo.secondary?
@@ -144,6 +148,10 @@ class RemoteMirror < ActiveRecord::Base
 
   private
 
+  def raw
+    @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, ref_name)
+  end
+
   def recently_scheduled?
     return false unless self.last_update_started_at
 

ee/app/services/projects/update_remote_mirror_service.rb

 module Projects
   class UpdateRemoteMirrorService < BaseService
-    attr_reader :mirror, :errors
+    attr_reader :errors
 
     def execute(remote_mirror)
-      @mirror = remote_mirror
       @errors = []
 
       return success unless remote_mirror.enabled?
 
       begin
-        repository.fetch_remote(mirror.ref_name, no_tags: true)
+        repository.fetch_remote(remote_mirror.ref_name, no_tags: true)
 
-        push_branches if changed_branches.present?
-        delete_branches if deleted_branches.present?
+        opts = {}
+        if remote_mirror.only_protected_branches?
+          opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
+        end
 
-        push_tags if changed_tags.present?
-        delete_tags if deleted_tags.present?
+        remote_mirror.update_repository(opts)
       rescue => e
         errors << e.message.strip
       end
@@ -26,105 +26,5 @@ module Projects
         success
       end
     end
-
-    private
-
-    def local_branches
-      @local_branches ||= repository.local_branches.each_with_object({}) do |branch, branches|
-        branches[branch.name] = branch unless skip_branch?(branch.name)
-      end
-    end
-
-    def remote_branches
-      @remote_branches ||= repository.remote_branches(mirror.ref_name).each_with_object({}) do |branch, branches|
-        branches[branch.name] = branch
-      end
-    end
-
-    def push_branches
-      default_branch, branches = changed_branches.partition { |name| project.default_branch == name }
-
-      # Push the default branch first so it works fine when remote mirror is empty.
-      branches.unshift(*default_branch)
-
-      repository.push_remote_branches(mirror.ref_name, branches)
-    end
-
-    def delete_branches
-      repository.delete_remote_branches(mirror.ref_name, deleted_branches)
-    end
-
-    def deleted_branches
-      @deleted_branches ||= refs_to_delete(:branches)
-    end
-
-    def changed_branches
-      @changed_branches ||= local_branches.each_with_object([]) do |(name, branch), branches|
-        remote_branch = remote_branches[name]
-
-        if remote_branch.nil?
-          branches << name
-        elsif branch.dereferenced_target == remote_branch.dereferenced_target
-          # Already up to date
-        else
-          branches << name
-        end
-      end
-    end
-
-    def local_tags
-      @local_tags ||= repository.tags.each_with_object({}) do |tag, tags|
-        tags[tag.name] = tag
-      end
-    end
-
-    def remote_tags
-      @remote_tags ||= repository.remote_tags(mirror.ref_name).each_with_object({}) do |tag, tags|
-        tags[tag.name] = tag
-      end
-    end
-
-    def push_tags
-      repository.push_remote_branches(mirror.ref_name, changed_tags)
-    end
-
-    def delete_tags
-      repository.delete_remote_branches(mirror.ref_name, deleted_tags)
-    end
-
-    def changed_tags
-      @changed_tags ||= local_tags.each_with_object([]) do |(name, tag), tags|
-        remote_tag = remote_tags[name]
-
-        if remote_tag.nil? || (tag.dereferenced_target != remote_tag.dereferenced_target)
-          tags << name
-        end
-      end
-    end
-
-    def deleted_tags
-      @deleted_tags ||= refs_to_delete(:tags)
-    end
-
-    def refs_to_delete(type)
-      remote_refs       = send("remote_#{type}") # rubocop:disable GitlabSecurity/PublicSend
-      local_refs        = send("local_#{type}") # rubocop:disable GitlabSecurity/PublicSend
-      default_branch_id = project.commit.id
-
-      remote_refs.each_with_object([]) do |(name, remote_ref), refs_to_delete|
-        next if local_refs[name] # skip if branch or tag exist in local repo
-        next if type == :branches && skip_branch?(name)
-
-        remote_ref_id = remote_ref.dereferenced_target.try(:id)
-
-        if remote_ref_id && project.repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
-          refs_to_delete << name
-        end
-      end
-    end
-
-    def skip_branch?(name)
-      mirror.only_protected_branches? && !ProtectedBranch.protected?(project, name)
-    end
   end
 end

lib/gitlab/git/remote_mirror.rb

+module Gitlab
+  module Git
+    class RemoteMirror
+      def initialize(repository, ref_name)
+        @repository = repository
+        @ref_name = ref_name
+      end
+
+      def update(only_branches_matching: [], only_tags_matching: [])
+        local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching)
+        remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching)
+
+        updated_branches = changed_refs(local_branches, remote_branches)
+        push_branches(updated_branches.keys) if updated_branches.present?
+
+        delete_refs(local_branches, remote_branches)
+
+        local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching)
+        remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching)
+
+        updated_tags = changed_refs(local_tags, remote_tags)
+        @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present?
+
+        delete_refs(local_tags, remote_tags)
+      end
+
+      private
+
+      def refs_obj(refs, only_refs_matching: [])
+        refs.each_with_object({}) do |ref, refs|
+          next if only_refs_matching.present? && !only_refs_matching.include?(ref.name)
+
+          refs[ref.name] = ref
+        end
+      end
+
+      def changed_refs(local_refs, remote_refs)
+        local_refs.select do |ref_name, ref|
+          remote_ref = remote_refs[ref_name]
+
+          remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target
+        end
+      end
+
+      def push_branches(branches)
+        default_branch, branches = branches.partition do |branch|
+          @repository.root_ref == branch
+        end
+
+        # Push the default branch first so it works fine when remote mirror is empty.
+        branches.unshift(*default_branch)
+
+        @repository.push_remote_branches(@ref_name, branches)
+      end
+
+      def delete_refs(local_refs, remote_refs)
+        refs = refs_to_delete(local_refs, remote_refs)
+
+        @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present?
+      end
+
+      def refs_to_delete(local_refs, remote_refs)
+        default_branch_id = @repository.commit.id
+
+        remote_refs.select do |remote_ref_name, remote_ref|
+          next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo
+
+          remote_ref_id = remote_ref.dereferenced_target.try(:id)
+
+          remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id)
+        end
+      end
+    end
+  end
+end

spec/services/projects/update_remote_mirror_service_spec.rb

@@ -4,6 +4,7 @@ describe Projects::UpdateRemoteMirrorService do
   let(:project) { create(:project, :repository) }
   let(:remote_project) { create(:forked_project_with_submodules) }
   let(:repository) { project.repository }
+  let(:raw_repository) { repository.raw }
   let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) }
 
   subject { described_class.new(project, project.creator) }
@@ -11,11 +12,10 @@ describe Projects::UpdateRemoteMirrorService do
   describe "#execute" do
     before do
       create_branch(repository, 'existing-branch')
-      allow(repository).to receive(:remote_tags) { generate_tags(repository, 'v1.0.0', 'v1.1.0') }
-
-      gitlab_shell = Gitlab::Shell.new
-      allow(repository).to receive(:gitlab_shell).and_return(gitlab_shell)
-      allow(gitlab_shell).to receive(:push_remote_branches).and_return(true)
+      allow(raw_repository).to receive(:remote_tags) do
+        generate_tags(repository, 'v1.0.0', 'v1.1.0')
+      end
+      allow(raw_repository).to receive(:push_remote_branches).and_return(true)
     end
 
     it 'does nothing when unlicensed' do
@@ -46,7 +46,7 @@ describe Projects::UpdateRemoteMirrorService do
       it "push all the branches the first time" do
         allow(repository).to receive(:fetch_remote)
 
-        expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, local_branch_names)
+        expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, local_branch_names)
 
         subject.execute(remote_mirror)
       end
@@ -54,7 +54,7 @@ describe Projects::UpdateRemoteMirrorService do
       it "does not push anything is remote is up to date" do
         allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, local_branch_names) }
 
-        expect(repository).not_to receive(:push_remote_branches)
+        expect(raw_repository).not_to receive(:push_remote_branches)
 
         subject.execute(remote_mirror)
       end
@@ -65,7 +65,7 @@ describe Projects::UpdateRemoteMirrorService do
         allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, current_branches) }
         create_branch(repository, 'my-new-branch')
 
-        expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['my-new-branch'])
+        expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['my-new-branch'])
 
         subject.execute(remote_mirror)
       end
@@ -76,7 +76,7 @@ describe Projects::UpdateRemoteMirrorService do
           update_branch(repository, 'existing-branch')
         end
 
-        expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
+        expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
 
         subject.execute(remote_mirror)
       end
@@ -98,7 +98,7 @@ describe Projects::UpdateRemoteMirrorService do
             update_branch(repository, protected_branch_name)
           end
 
-          expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
+          expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
 
           subject.execute(remote_mirror)
         end
@@ -109,7 +109,7 @@ describe Projects::UpdateRemoteMirrorService do
             update_branch(repository, 'existing-branch')
           end
 
-          expect(repository).not_to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
+          expect(raw_repository).not_to receive(:push_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
 
           subject.execute(remote_mirror)
         end
@@ -123,7 +123,7 @@ describe Projects::UpdateRemoteMirrorService do
               update_remote_branch(repository, remote_mirror.ref_name, 'markdown')
             end
 
-            expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['markdown'])
+            expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['markdown'])
 
             subject.execute(remote_mirror)
           end
@@ -138,7 +138,7 @@ describe Projects::UpdateRemoteMirrorService do
               delete_branch(repository, 'existing-branch')
             end
 
-            expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
+            expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
 
             subject.execute(remote_mirror)
           end
@@ -163,7 +163,7 @@ describe Projects::UpdateRemoteMirrorService do
                 delete_branch(repository, protected_branch_name)
               end
 
-              expect(repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
+              expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
 
               subject.execute(remote_mirror)
             end
@@ -174,7 +174,7 @@ describe Projects::UpdateRemoteMirrorService do
                 delete_branch(repository, 'existing-branch')
               end
 
-              expect(repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
+              expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['existing-branch'])
 
               subject.execute(remote_mirror)
             end
@@ -196,7 +196,7 @@ describe Projects::UpdateRemoteMirrorService do
                   create_remote_branch(repository, remote_mirror.ref_name, 'remote-branch', rev.id)
                 end
 
-                expect(repository).not_to receive(:delete_remote_branches)
+                expect(raw_repository).not_to receive(:delete_remote_branches)
 
                 subject.execute(remote_mirror)
               end
@@ -211,7 +211,7 @@ describe Projects::UpdateRemoteMirrorService do
                   create_remote_branch(repository, remote_mirror.ref_name, protected_branch_name, masterrev.id)
                 end
 
-                expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
+                expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, [protected_branch_name])
 
                 subject.execute(remote_mirror)
               end
@@ -229,7 +229,7 @@ describe Projects::UpdateRemoteMirrorService do
                 create_remote_branch(repository, remote_mirror.ref_name, 'remote-branch', rev.id)
               end
 
-              expect(repository).not_to receive(:delete_remote_branches)
+              expect(raw_repository).not_to receive(:delete_remote_branches)
 
               subject.execute(remote_mirror)
             end
@@ -244,7 +244,7 @@ describe Projects::UpdateRemoteMirrorService do
                 create_remote_branch(repository, remote_mirror.ref_name, 'remote-branch', masterrev.id)
               end
 
-              expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['remote-branch'])
+              expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['remote-branch'])
 
               subject.execute(remote_mirror)
             end
@@ -271,9 +271,9 @@ describe Projects::UpdateRemoteMirrorService do
 
       context 'when there are some tags to push' do
         it 'pushes tags to remote' do
-          allow(repository).to receive(:remote_tags) { {} }
+          allow(raw_repository).to receive(:remote_tags) { {} }
 
-          expect(repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['v1.0.0', 'v1.1.0'])
+          expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.ref_name, ['v1.0.0', 'v1.1.0'])
 
           subject.execute(remote_mirror)
         end
@@ -282,11 +282,11 @@ describe Projects::UpdateRemoteMirrorService do
       context 'when there are some tags to delete' do
         it 'deletes tags from remote' do
           remote_tags = generate_tags(repository, 'v1.0.0', 'v1.1.0')
-          allow(repository).to receive(:remote_tags) { remote_tags }
+          allow(raw_repository).to receive(:remote_tags) { remote_tags }
 
           repository.rm_tag(create(:user), 'v1.0.0')
 
-          expect(repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['v1.0.0'])
+          expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.ref_name, ['v1.0.0'])
 
           subject.execute(remote_mirror)
         end