Geo - Clean up deleted repositories in a secondary node

ee/app/services/ee/projects/destroy_service.rb

@@ -33,17 +33,46 @@ module EE
       # There is no need to do any database operation as it will be
       # replicated by itself.
       def geo_replicate
+        return unless ::Gitlab::Geo.secondary?
+
         # Flush the cache for both repositories. This has to be done _before_
         # removing the physical repositories as some expiration code depends on
         # Git data (e.g. a list of branch names).
         flush_caches(project)
 
         trash_repositories!
+        trash_repositories_cleanup!
+
         log_info("Project \"#{project.name}\" was removed")
       end
 
       private
 
+      # When we remove project we move the repository to path+deleted.git
+      # then outside the transaction we schedule removal of path+deleted
+      # with Sidekiq through the after_commit callback. In a Geo secondary
+      # node we don't have access to the original model anymore then we
+      # rebuild a Geo::DeletedProject model. Since this model is read-only,
+      # this callback will not be triggered letting us with stalled
+      # repositories on disk.
+      def trash_repositories_cleanup!
+        repo_removed_path = removal_path(repo_path)
+
+        if gitlab_shell.exists?(repository_storage_path, repo_removed_path + '.git')
+          GitlabShellWorker.perform_in(5.minutes, :remove_repository, repository_storage_path, repo_removed_path)
+        end
+
+        wiki_removed_path = removal_path(wiki_path)
+
+        if gitlab_shell.exists?(repository_storage_path, wiki_removed_path + '.git')
+          GitlabShellWorker.perform_in(5.minutes, :remove_repository, repository_storage_path, wiki_removed_path)
+        end
+      end
+
+      def repository_storage_path
+        project.repository_storage_path
+      end
+
       def log_audit_event(project)
         ::AuditEventService.new(
           current_user,

spec/services/geo/repository_destroy_service_spec.rb

 require 'spec_helper'
 
 describe Geo::RepositoryDestroyService do
+  set(:secondary) { create(:geo_node) }
   let(:project) { create(:project_empty_repo) }
+
   subject(:service) { described_class.new(project.id, project.name, project.disk_path, project.repository_storage) }
 
+  before do
+    allow(Gitlab::Geo).to receive(:current_node) { secondary }
+  end
+
   describe '#async_execute' do
     it 'starts the worker' do
       expect(GeoRepositoryDestroyWorker).to receive(:perform_async)
@@ -29,6 +35,15 @@ describe Geo::RepositoryDestroyService do
 
         expect(project.gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
       end
+
+      it 'cleans up deleted repositories' do
+        project.delete
+
+        expect(::GitlabShellWorker).to receive(:perform_in)
+          .with(5.minutes, :remove_repository, project.repository_storage_path, "#{project.disk_path}+#{project.id}+deleted")
+
+        service.execute
+      end
     end
 
     context 'hashed storage project' do
@@ -43,6 +58,15 @@ describe Geo::RepositoryDestroyService do
 
         expect(project.gitlab_shell.exists?(project.repository_storage_path, "#{project.disk_path}.git")).to be_falsey
       end
+
+      it 'cleans up deleted repositories' do
+        project.delete
+
+        expect(::GitlabShellWorker).to receive(:perform_in)
+          .with(5.minutes, :remove_repository, project.repository_storage_path, "#{project.disk_path}+#{project.id}+deleted")
+
+        service.execute
+      end
     end
   end
 end