Improve performance of the Container Registry delete tags API

Use the new DELETE /v2/<name>/tags/reference/<reference> endpoint
provided by the GitLab Container Registry to delete a tag by name
instead of digest (used as fallback for third-party registries).

app/models/container_repository.rb

@@ -77,7 +77,11 @@ class ContainerRepository < ApplicationRecord
   end
 
   def delete_tag_by_digest(digest)
-    client.delete_repository_tag(self.path, digest)
+    client.delete_repository_tag_by_digest(self.path, digest)
+  end
+
+  def delete_tag_by_name(name)
+    client.delete_repository_tag_by_name(self.path, name)
   end
 
   def self.build_from_path(path)

app/services/projects/container_repository/delete_tags_service.rb

@@ -14,12 +14,25 @@ module Projects
 
       private
 
+      # Delete tags by name with a single DELETE request. This is only supported
+      # by the GitLab Container Registry fork. See
+      # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23325 for details.
+      def fast_delete(container_repository, tag_names)
+        deleted_tags = tag_names.select do |name|
+          container_repository.delete_tag_by_name(name)
+        end
+
+        deleted_tags.any? ? success(deleted: deleted_tags) : error('could not delete tags')
+      end
+
       # Replace a tag on the registry with a dummy tag.
       # This is a hack as the registry doesn't support deleting individual
       # tags. This code effectively pushes a dummy image and assigns the tag to it.
       # This way when the tag is deleted only the dummy image is affected.
+      # This is used to preverse compatibility with third-party registries that
+      # don't support fast delete.
       # See https://gitlab.com/gitlab-org/gitlab/issues/15737 for a discussion
-      def smart_delete(container_repository, tag_names)
+      def slow_delete(container_repository, tag_names)
         # generates the blobs for the dummy image
         dummy_manifest = container_repository.client.generate_empty_manifest(container_repository.path)
         return error('could not generate manifest') if dummy_manifest.nil?
@@ -36,6 +49,15 @@ module Projects
         end
       end
 
+      def smart_delete(container_repository, tag_names)
+        fast_delete_enabled = Feature.enabled?(:container_registry_fast_tag_delete, default_enabled: true)
+        if fast_delete_enabled && container_repository.client.supports_tag_delete?
+          fast_delete(container_repository, tag_names)
+        else
+          slow_delete(container_repository, tag_names)
+        end
+      end
+
       # update the manifests of the tags with the new dummy image
       def replace_tag_manifests(container_repository, dummy_manifest, tag_names)
         deleted_tags = {}

changelogs/unreleased/31832-improve-performance-of-the-container-registry-delete-tags-api.yml

+---
+title: Improve performance of the Container Registry delete tags API
+merge_request: 23325
+author:
+type: performance

lib/container_registry/client.rb

@@ -6,6 +6,8 @@ require 'digest'
 
 module ContainerRegistry
   class Client
+    include Gitlab::Utils::StrongMemoize
+
     attr_accessor :uri
 
     DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
@@ -35,10 +37,25 @@ module ContainerRegistry
       response.headers['docker-content-digest'] if response.success?
     end
 
-    def delete_repository_tag(name, reference)
-      result = faraday.delete("/v2/#{name}/manifests/#{reference}")
+    def delete_repository_tag_by_digest(name, reference)
+      delete_if_exists("/v2/#{name}/manifests/#{reference}")
+    end
 
-      result.success? || result.status == 404
+    def delete_repository_tag_by_name(name, reference)
+      delete_if_exists("/v2/#{name}/tags/reference/#{reference}")
+    end
+
+    # Check if the registry supports tag deletion. This is only supported by the
+    # GitLab registry fork. The fastest and safest way to check this is to send
+    # an OPTIONS request to /v2/<name>/tags/reference/<tag>, using a random
+    # repository name and tag (the registry won't check if they exist).
+    # Registries that support tag deletion will reply with a 200 OK and include
+    # the DELETE method in the Allow header. Others reply with an 404 Not Found.
+    def supports_tag_delete?
+      strong_memoize(:supports_tag_delete) do
+        response = faraday.run_request(:options, '/v2/name/tags/reference/tag', '', {})
+        response.success? && response.headers['allow']&.include?('DELETE')
+      end
     end
 
     def upload_raw_blob(path, blob)
@@ -86,9 +103,7 @@ module ContainerRegistry
     end
 
     def delete_blob(name, digest)
-      result = faraday.delete("/v2/#{name}/blobs/#{digest}")
-
-      result.success? || result.status == 404
+      delete_if_exists("/v2/#{name}/blobs/#{digest}")
     end
 
     def put_tag(name, reference, manifest)
@@ -163,6 +178,12 @@ module ContainerRegistry
         conn.adapter :net_http
       end
     end
+
+    def delete_if_exists(path)
+      result = faraday.delete(path)
+
+      result.success? || result.status == 404
+    end
   end
 end
 

lib/container_registry/tag.rb

@@ -118,7 +118,7 @@ module ContainerRegistry
     def unsafe_delete
       return unless digest
 
-      client.delete_repository_tag(repository.path, digest)
+      client.delete_repository_tag_by_digest(repository.path, digest)
     end
   end
 end

spec/lib/container_registry/client_spec.rb

@@ -146,4 +146,57 @@ describe ContainerRegistry::Client do
       expect(subject).to eq 'sha256:123'
     end
   end
+
+  describe '#delete_repository_tag_by_name' do
+    subject { client.delete_repository_tag_by_name('group/test', 'a') }
+
+    context 'when the tag exists' do
+      before do
+        stub_request(:delete, "http://container-registry/v2/group/test/tags/reference/a")
+          .to_return(status: 200, body: "")
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when the tag does not exist' do
+      before do
+        stub_request(:delete, "http://container-registry/v2/group/test/tags/reference/a")
+          .to_return(status: 404, body: "")
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when an error occurs' do
+      before do
+        stub_request(:delete, "http://container-registry/v2/group/test/tags/reference/a")
+          .to_return(status: 500, body: "")
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '#supports_tag_delete?' do
+    subject { client.supports_tag_delete? }
+
+    context 'when the server supports tag deletion' do
+      before do
+        stub_request(:options, "http://container-registry/v2/name/tags/reference/tag")
+          .to_return(status: 200, body: "", headers: { 'Allow' => 'DELETE' })
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when the server does not support tag deletion' do
+      before do
+        stub_request(:options, "http://container-registry/v2/name/tags/reference/tag")
+          .to_return(status: 404, body: "")
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
 end

spec/models/container_repository_spec.rb

@@ -85,7 +85,7 @@ describe ContainerRepository do
     context 'when action succeeds' do
       it 'returns status that indicates success' do
         expect(repository.client)
-          .to receive(:delete_repository_tag)
+          .to receive(:delete_repository_tag_by_digest)
           .twice
           .and_return(true)
 
@@ -96,7 +96,7 @@ describe ContainerRepository do
     context 'when action fails' do
       it 'returns status that indicates failure' do
         expect(repository.client)
-          .to receive(:delete_repository_tag)
+          .to receive(:delete_repository_tag_by_digest)
           .twice
           .and_return(false)
 
@@ -105,6 +105,36 @@ describe ContainerRepository do
     end
   end
 
+  describe '#delete_tag_by_name' do
+    let(:repository) do
+      create(:container_repository, name: 'my_image',
+                                    tags: { latest: '123', rc1: '234' },
+                                    project: project)
+    end
+
+    context 'when action succeeds' do
+      it 'returns status that indicates success' do
+        expect(repository.client)
+          .to receive(:delete_repository_tag_by_name)
+          .with(repository.path, "latest")
+          .and_return(true)
+
+        expect(repository.delete_tag_by_name('latest')).to be_truthy
+      end
+    end
+
+    context 'when action fails' do
+      it 'returns status that indicates failure' do
+        expect(repository.client)
+          .to receive(:delete_repository_tag_by_name)
+          .with(repository.path, "latest")
+          .and_return(false)
+
+        expect(repository.delete_tag_by_name('latest')).to be_falsey
+      end
+    end
+  end
+
   describe '#location' do
     context 'when registry is running on a custom port' do
       before do

spec/services/projects/container_repository/cleanup_tags_service_spec.rb

@@ -41,7 +41,7 @@ describe Projects::ContainerRepository::CleanupTagsService do
       let(:params) { {} }
 
       it 'does not remove anything' do
-        expect_any_instance_of(ContainerRegistry::Client).not_to receive(:delete_repository_tag)
+        expect_any_instance_of(ContainerRegistry::Client).not_to receive(:delete_repository_tag_by_digest)
 
         is_expected.to include(status: :success, deleted: [])
       end
@@ -156,7 +156,7 @@ describe Projects::ContainerRepository::CleanupTagsService do
 
   def expect_delete(digest)
     expect_any_instance_of(ContainerRegistry::Client)
-      .to receive(:delete_repository_tag)
+      .to receive(:delete_repository_tag_by_digest)
       .with(repository.path, digest) { true }
   end
 end