Merge branch '335560-dp-manifest-workhorse-uploads' into 'master'

Dependency Proxy uses workhorse for manifest pulls

See merge request gitlab-org/gitlab!73033

app/controllers/groups/dependency_proxy_for_containers_controller.rb

@@ -11,8 +11,8 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
   before_action :ensure_token_granted!, only: [:blob, :manifest]
   before_action :ensure_feature_enabled!
 
-  before_action :verify_workhorse_api!, only: [:authorize_upload_blob, :upload_blob]
-  skip_before_action :verify_authenticity_token, only: [:authorize_upload_blob, :upload_blob]
+  before_action :verify_workhorse_api!, only: [:authorize_upload_blob, :upload_blob, :authorize_upload_manifest, :upload_manifest]
+  skip_before_action :verify_authenticity_token, only: [:authorize_upload_blob, :upload_blob, :authorize_upload_manifest, :upload_manifest]
 
   attr_reader :token
 
@@ -22,20 +22,11 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
     result = DependencyProxy::FindOrCreateManifestService.new(group, image, tag, token).execute
 
     if result[:status] == :success
-      response.headers['Docker-Content-Digest'] = result[:manifest].digest
-      response.headers['Content-Length'] = result[:manifest].size
-      response.headers['Docker-Distribution-Api-Version'] = DependencyProxy::DISTRIBUTION_API_VERSION
-      response.headers['Etag'] = "\"#{result[:manifest].digest}\""
-      content_type = result[:manifest].content_type
-
-      event_name = tracking_event_name(object_type: :manifest, from_cache: result[:from_cache])
-      track_package_event(event_name, :dependency_proxy, namespace: group, user: auth_user)
-      send_upload(
-        result[:manifest].file,
-        proxy: true,
-        redirect_params: { query: { 'response-content-type' => content_type } },
-        send_params: { type: content_type }
-      )
+      if result[:manifest]
+        send_manifest(result[:manifest], from_cache: result[:from_cache])
+      else
+        send_dependency(manifest_header, DependencyProxy::Registry.manifest_url(image, tag), manifest_file_name)
+      end
     else
       render status: result[:http_status], json: result[:message]
     end
@@ -59,7 +50,7 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
   def authorize_upload_blob
     set_workhorse_internal_api_content_type
 
-    render json: DependencyProxy::FileUploader.workhorse_authorize(has_length: false, maximum_size: 5.gigabytes)
+    render json: DependencyProxy::FileUploader.workhorse_authorize(has_length: false, maximum_size: DependencyProxy::Blob::MAX_FILE_SIZE)
   end
 
   def upload_blob
@@ -75,6 +66,27 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
     head :ok
   end
 
+  def authorize_upload_manifest
+    set_workhorse_internal_api_content_type
+
+    render json: DependencyProxy::FileUploader.workhorse_authorize(has_length: false, maximum_size: DependencyProxy::Manifest::MAX_FILE_SIZE)
+  end
+
+  def upload_manifest
+    @group.dependency_proxy_manifests.create!(
+      file_name: manifest_file_name,
+      content_type: request.headers[Gitlab::Workhorse::SEND_DEPENDENCY_CONTENT_TYPE_HEADER],
+      digest: request.headers['Docker-Content-Digest'],
+      file: params[:file],
+      size: params[:file].size
+    )
+
+    event_name = tracking_event_name(object_type: :manifest, from_cache: false)
+    track_package_event(event_name, :dependency_proxy, namespace: group, user: auth_user)
+
+    head :ok
+  end
+
   private
 
   def blob_via_workhorse
@@ -86,14 +98,38 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
 
       send_upload(blob.file)
     else
-      send_dependency(token, DependencyProxy::Registry.blob_url(image, params[:sha]), blob_file_name)
+      send_dependency(token_header, DependencyProxy::Registry.blob_url(image, params[:sha]), blob_file_name)
     end
   end
 
+  def send_manifest(manifest, from_cache:)
+    # Technical debt: change to read_at https://gitlab.com/gitlab-org/gitlab/-/issues/341536
+    manifest.touch
+    response.headers['Docker-Content-Digest'] = manifest.digest
+    response.headers['Content-Length'] = manifest.size
+    response.headers['Docker-Distribution-Api-Version'] = DependencyProxy::DISTRIBUTION_API_VERSION
+    response.headers['Etag'] = "\"#{manifest.digest}\""
+    content_type = manifest.content_type
+
+    event_name = tracking_event_name(object_type: :manifest, from_cache: from_cache)
+    track_package_event(event_name, :dependency_proxy, namespace: group, user: auth_user)
+
+    send_upload(
+      manifest.file,
+      proxy: true,
+      redirect_params: { query: { 'response-content-type' => content_type } },
+      send_params: { type: content_type }
+    )
+  end
+
   def blob_file_name
     @blob_file_name ||= params[:sha].sub('sha256:', '') + '.gz'
   end
 
+  def manifest_file_name
+    @manifest_file_name ||= "#{image}:#{tag}.json"
+  end
+
   def group
     strong_memoize(:group) do
       Group.find_by_full_path(params[:group_id], follow_redirects: true)
@@ -137,4 +173,12 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
       render status: result[:http_status], json: result[:message]
     end
   end
+
+  def token_header
+    { Authorization: ["Bearer #{token}"] }
+  end
+
+  def manifest_header
+    token_header.merge(Accept: ::ContainerRegistry::Client::ACCEPTED_TYPES)
+  end
 end

app/helpers/workhorse_helper.rb

@@ -41,8 +41,8 @@ module WorkhorseHelper
     head :ok
   end
 
-  def send_dependency(token, url, filename)
-    headers.store(*Gitlab::Workhorse.send_dependency(token, url))
+  def send_dependency(dependency_headers, url, filename)
+    headers.store(*Gitlab::Workhorse.send_dependency(dependency_headers, url))
     headers['Content-Disposition'] =
       ActionDispatch::Http::ContentDisposition.format(disposition: 'attachment', filename: filename)
     headers['Content-Type'] = 'application/gzip'

app/models/dependency_proxy/blob.rb

@@ -7,6 +7,8 @@ class DependencyProxy::Blob < ApplicationRecord
 
   belongs_to :group
 
+  MAX_FILE_SIZE = 5.gigabytes.freeze
+
   validates :group, presence: true
   validates :file, presence: true
   validates :file_name, presence: true

app/models/dependency_proxy/manifest.rb

@@ -7,6 +7,8 @@ class DependencyProxy::Manifest < ApplicationRecord
 
   belongs_to :group
 
+  MAX_FILE_SIZE = 10.megabytes.freeze
+
   validates :group, presence: true
   validates :file, presence: true
   validates :file_name, presence: true
@@ -14,10 +16,7 @@ class DependencyProxy::Manifest < ApplicationRecord
 
   mount_file_store_uploader DependencyProxy::FileUploader
 
-  def self.find_or_initialize_by_file_name_or_digest(file_name:, digest:)
-    result = find_by(file_name: file_name) || find_by(digest: digest)
-    return result if result
-
-    new(file_name: file_name, digest: digest)
+  def self.find_by_file_name_or_digest(file_name:, digest:)
+    find_by(file_name: file_name) || find_by(digest: digest)
   end
 end

app/services/dependency_proxy/find_or_create_manifest_service.rb

@@ -14,18 +14,18 @@ module DependencyProxy
     def execute
       @manifest = @group.dependency_proxy_manifests
                         .active
-                        .find_or_initialize_by_file_name_or_digest(file_name: @file_name, digest: @tag)
+                        .find_by_file_name_or_digest(file_name: @file_name, digest: @tag)
 
       head_result = DependencyProxy::HeadManifestService.new(@image, @tag, @token).execute
 
-      if cached_manifest_matches?(head_result)
-        @manifest.touch
+      return respond if cached_manifest_matches?(head_result)
 
-        return success(manifest: @manifest, from_cache: true)
+      if Feature.enabled?(:dependency_proxy_manifest_workhorse, @group, default_enabled: :yaml)
+        success(manifest: nil, from_cache: false)
+      else
+        pull_new_manifest
+        respond(from_cache: false)
       end
-
-      pull_new_manifest
-      respond(from_cache: false)
     rescue Timeout::Error, *Gitlab::HTTP::HTTP_ERRORS
       respond
     end
@@ -34,12 +34,19 @@ module DependencyProxy
 
     def pull_new_manifest
       DependencyProxy::PullManifestService.new(@image, @tag, @token).execute_with_manifest do |new_manifest|
-        @manifest.update!(
+        params = {
+          file_name: @file_name,
           content_type: new_manifest[:content_type],
           digest: new_manifest[:digest],
           file: new_manifest[:file],
           size: new_manifest[:file].size
-        )
+        }
+
+        if @manifest
+          @manifest.update!(params)
+        else
+          @manifest = @group.dependency_proxy_manifests.create!(params)
+        end
       end
     end
 
@@ -50,10 +57,7 @@ module DependencyProxy
     end
 
     def respond(from_cache: true)
-      if @manifest.persisted?
-        # Technical debt: change to read_at https://gitlab.com/gitlab-org/gitlab/-/issues/341536
-        @manifest.touch if from_cache
-
+      if @manifest
         success(manifest: @manifest, from_cache: from_cache)
       else
         error('Failed to download the manifest from the external registry', 503)

config/feature_flags/development/dependency_proxy_manifest_workhorse.yml

+---
+name: dependency_proxy_manifest_workhorse
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73033
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344216
+milestone: '14.4'
+type: development
+group: group::package
+default_enabled: false

config/routes/group.rb

@@ -155,5 +155,7 @@ scope format: false do
     get 'v2/*group_id/dependency_proxy/containers/*image/blobs/:sha' => 'groups/dependency_proxy_for_containers#blob' # rubocop:todo Cop/PutGroupRoutesUnderScope
     post 'v2/*group_id/dependency_proxy/containers/*image/blobs/:sha/upload/authorize' => 'groups/dependency_proxy_for_containers#authorize_upload_blob' # rubocop:todo Cop/PutGroupRoutesUnderScope
     post 'v2/*group_id/dependency_proxy/containers/*image/blobs/:sha/upload' => 'groups/dependency_proxy_for_containers#upload_blob' # rubocop:todo Cop/PutGroupRoutesUnderScope
+    post 'v2/*group_id/dependency_proxy/containers/*image/manifests/*tag/upload/authorize' => 'groups/dependency_proxy_for_containers#authorize_upload_manifest' # rubocop:todo Cop/PutGroupRoutesUnderScope
+    post 'v2/*group_id/dependency_proxy/containers/*image/manifests/*tag/upload' => 'groups/dependency_proxy_for_containers#upload_manifest' # rubocop:todo Cop/PutGroupRoutesUnderScope
   end
 end

doc/administration/instance_limits.md

@@ -831,6 +831,17 @@ Set the limit to `0` to allow any file size.
 
 When asking for versions of a given NuGet package name, the GitLab Package Registry returns a maximum of 300 versions.
 
+## Dependency Proxy Limits
+
+> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6396) in GitLab 14.5.
+
+The maximum file size for an image cached in the
+[Dependency Proxy](../user/packages/dependency_proxy/index.md)
+varies by file type:
+
+- Image blob: 5 GB
+- Image manifest: 10 MB
+
 ## Branch retargeting on merge
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/320902) in GitLab 13.9.

ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb

@@ -81,18 +81,23 @@ RSpec.describe Groups::DependencyProxyForContainersController do
   end
 
   describe 'GET #manifest' do
-    let_it_be(:manifest) { create(:dependency_proxy_manifest) }
+    let_it_be(:manifest) { create(:dependency_proxy_manifest, group: group) }
 
     let(:pull_response) { { status: :success, manifest: manifest, from_cache: false } }
+    let(:head_response) { { status: :success, digest: manifest.digest, content_type: manifest.content_type } }
+    let(:tag) { manifest.file_name.sub('.json', '').split(':').last }
 
     subject(:get_manifest) do
-      get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: '3.9.2' }
+      get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: tag }
     end
 
     before do
       allow_next_instance_of(DependencyProxy::FindOrCreateManifestService) do |instance|
         allow(instance).to receive(:execute).and_return(pull_response)
       end
+      allow_next_instance_of(DependencyProxy::HeadManifestService) do |instance|
+        allow(instance).to receive(:execute).and_return(head_response)
+      end
     end
 
     it_behaves_like 'when sso is enabled for the group', 'a successful manifest pull'

lib/gitlab/workhorse.rb

@@ -8,6 +8,7 @@ require 'uri'
 module Gitlab
   class Workhorse
     SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
+    SEND_DEPENDENCY_CONTENT_TYPE_HEADER = 'Workhorse-Proxy-Content-Type'
     VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'
     INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'
@@ -170,9 +171,9 @@ module Gitlab
         ]
       end
 
-      def send_dependency(token, url)
+      def send_dependency(headers, url)
         params = {
-          'Header' => { Authorization: ["Bearer #{token}"] },
+          'Header' => headers,
           'Url' => url
         }
 

spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb

@@ -124,6 +124,34 @@ RSpec.describe Groups::DependencyProxyForContainersController do
     end
   end
 
+  shared_examples 'authorize action with permission' do
+    context 'with a valid user' do
+      before do
+        group.add_guest(user)
+      end
+
+      it 'sends Workhorse local file instructions', :aggregate_failures do
+        subject
+
+        expect(response.headers['Content-Type']).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
+        expect(json_response['TempPath']).to eq(DependencyProxy::FileUploader.workhorse_local_upload_path)
+        expect(json_response['RemoteObject']).to be_nil
+        expect(json_response['MaximumSize']).to eq(maximum_size)
+      end
+
+      it 'sends Workhorse remote object instructions', :aggregate_failures do
+        stub_dependency_proxy_object_storage(direct_upload: true)
+
+        subject
+
+        expect(response.headers['Content-Type']).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
+        expect(json_response['TempPath']).to be_nil
+        expect(json_response['RemoteObject']).not_to be_nil
+        expect(json_response['MaximumSize']).to eq(maximum_size)
+      end
+    end
+  end
+
   before do
     allow(Gitlab.config.dependency_proxy)
       .to receive(:enabled).and_return(true)
@@ -136,9 +164,10 @@ RSpec.describe Groups::DependencyProxyForContainersController do
   end
 
   describe 'GET #manifest' do
-    let_it_be(:manifest) { create(:dependency_proxy_manifest) }
+    let_it_be(:manifest) { create(:dependency_proxy_manifest, group: group) }
 
     let(:pull_response) { { status: :success, manifest: manifest, from_cache: false } }
+    let(:tag) { 'latest1' }
 
     before do
       allow_next_instance_of(DependencyProxy::FindOrCreateManifestService) do |instance|
@@ -146,7 +175,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
       end
     end
 
-    subject { get_manifest }
+    subject { get_manifest(tag) }
 
     context 'feature enabled' do
       before do
@@ -207,11 +236,26 @@ RSpec.describe Groups::DependencyProxyForContainersController do
         it_behaves_like 'a successful manifest pull'
         it_behaves_like 'a package tracking event', described_class.name, 'pull_manifest'
 
-        context 'with a cache entry' do
-          let(:pull_response) { { status: :success, manifest: manifest, from_cache: true } }
+        context 'with workhorse response' do
+          let(:pull_response) { { status: :success, manifest: nil, from_cache: false } }
 
-          it_behaves_like 'returning response status', :success
-          it_behaves_like 'a package tracking event', described_class.name, 'pull_manifest_from_cache'
+          it 'returns Workhorse send-dependency instructions', :aggregate_failures do
+            subject
+
+            send_data_type, send_data = workhorse_send_data
+            header, url = send_data.values_at('Header', 'Url')
+
+            expect(send_data_type).to eq('send-dependency')
+            expect(header).to eq(
+              "Authorization" => ["Bearer abcd1234"],
+              "Accept" => ::ContainerRegistry::Client::ACCEPTED_TYPES
+            )
+            expect(url).to eq(DependencyProxy::Registry.manifest_url('alpine', tag))
+            expect(response.headers['Content-Type']).to eq('application/gzip')
+            expect(response.headers['Content-Disposition']).to eq(
+              ActionDispatch::Http::ContentDisposition.format(disposition: 'attachment', filename: manifest.file_name)
+            )
+          end
         end
       end
 
@@ -237,8 +281,8 @@ RSpec.describe Groups::DependencyProxyForContainersController do
 
     it_behaves_like 'not found when disabled'
 
-    def get_manifest
-      get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: '3.9.2' }
+    def get_manifest(tag)
+      get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: tag }
     end
   end
 
@@ -383,53 +427,71 @@ RSpec.describe Groups::DependencyProxyForContainersController do
 
   describe 'GET #authorize_upload_blob' do
     let(:blob_sha) { 'a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4' }
+    let(:maximum_size) { DependencyProxy::Blob::MAX_FILE_SIZE }
 
-    subject(:authorize_upload_blob) do
+    subject do
       request.headers.merge!(workhorse_internal_api_request_header)
 
       get :authorize_upload_blob, params: { group_id: group.to_param, image: 'alpine', sha: blob_sha }
     end
 
+    it_behaves_like 'without permission'
+    it_behaves_like 'authorize action with permission'
+  end
+
+  describe 'GET #upload_blob' do
+    let(:blob_sha) { 'a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4' }
+    let(:file) { fixture_file_upload("spec/fixtures/dependency_proxy/#{blob_sha}.gz", 'application/gzip') }
+
+    subject do
+      request.headers.merge!(workhorse_internal_api_request_header)
+
+      get :upload_blob, params: {
+        group_id: group.to_param,
+        image: 'alpine',
+        sha: blob_sha,
+        file: file
+      }
+    end
+
     it_behaves_like 'without permission'
 
     context 'with a valid user' do
       before do
         group.add_guest(user)
-      end
 
-      it 'sends Workhorse local file instructions', :aggregate_failures do
-        authorize_upload_blob
-
-        expect(response.headers['Content-Type']).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
-        expect(json_response['TempPath']).to eq(DependencyProxy::FileUploader.workhorse_local_upload_path)
-        expect(json_response['RemoteObject']).to be_nil
-        expect(json_response['MaximumSize']).to eq(5.gigabytes)
+        expect_next_found_instance_of(Group) do |instance|
+          expect(instance).to receive_message_chain(:dependency_proxy_blobs, :create!)
+        end
       end
 
-      it 'sends Workhorse remote object instructions', :aggregate_failures do
-        stub_dependency_proxy_object_storage(direct_upload: true)
+      it_behaves_like 'a package tracking event', described_class.name, 'pull_blob'
+    end
+  end
 
-        authorize_upload_blob
+  describe 'GET #authorize_upload_manifest' do
+    let(:maximum_size) { DependencyProxy::Manifest::MAX_FILE_SIZE }
 
-        expect(response.headers['Content-Type']).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
-        expect(json_response['TempPath']).to be_nil
-        expect(json_response['RemoteObject']).not_to be_nil
-        expect(json_response['MaximumSize']).to eq(5.gigabytes)
-      end
+    subject do
+      request.headers.merge!(workhorse_internal_api_request_header)
+
+      get :authorize_upload_manifest, params: { group_id: group.to_param, image: 'alpine', tag: 'latest' }
     end
+
+    it_behaves_like 'without permission'
+    it_behaves_like 'authorize action with permission'
   end
 
-  describe 'GET #upload_blob' do
-    let(:blob_sha) { 'a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4' }
-    let(:file) { fixture_file_upload("spec/fixtures/dependency_proxy/#{blob_sha}.gz", 'application/gzip') }
+  describe 'GET #upload_manifest' do
+    let(:file) { fixture_file_upload("spec/fixtures/dependency_proxy/manifest", 'application/json') }
 
     subject do
       request.headers.merge!(workhorse_internal_api_request_header)
 
-      get :upload_blob, params: {
+      get :upload_manifest, params: {
         group_id: group.to_param,
         image: 'alpine',
-        sha: blob_sha,
+        tag: 'latest',
         file: file
       }
     end
@@ -441,11 +503,11 @@ RSpec.describe Groups::DependencyProxyForContainersController do
         group.add_guest(user)
 
         expect_next_found_instance_of(Group) do |instance|
-          expect(instance).to receive_message_chain(:dependency_proxy_blobs, :create!)
+          expect(instance).to receive_message_chain(:dependency_proxy_manifests, :create!)
         end
       end
 
-      it_behaves_like 'a package tracking event', described_class.name, 'pull_blob'
+      it_behaves_like 'a package tracking event', described_class.name, 'pull_manifest'
     end
   end
 

spec/lib/gitlab/workhorse_spec.rb

@@ -512,6 +512,24 @@ RSpec.describe Gitlab::Workhorse do
     end
   end
 
+  describe '.send_dependency' do
+    let(:headers) { { Accept: 'foo', Authorization: 'Bearer asdf1234' } }
+    let(:url) { 'https://foo.bar.com/baz' }
+
+    subject { described_class.send_dependency(headers, url) }
+
+    it 'sets the header correctly', :aggregate_failures do
+      key, command, params = decode_workhorse_header(subject)
+
+      expect(key).to eq("Gitlab-Workhorse-Send-Data")
+      expect(command).to eq("send-dependency")
+      expect(params).to eq({
+        'Header' => headers,
+        'Url' => url
+      }.deep_stringify_keys)
+    end
+  end
+
   describe '.send_git_snapshot' do
     let(:url) { 'http://example.com' }
 

spec/models/dependency_proxy/manifest_spec.rb

@@ -31,18 +31,14 @@ RSpec.describe DependencyProxy::Manifest, type: :model do
     end
   end
 
-  describe '.find_or_initialize_by_file_name_or_digest' do
+  describe '.find_by_file_name_or_digest' do
     let_it_be(:file_name) { 'foo' }
     let_it_be(:digest) { 'bar' }
 
-    subject { DependencyProxy::Manifest.find_or_initialize_by_file_name_or_digest(file_name: file_name, digest: digest) }
+    subject { DependencyProxy::Manifest.find_by_file_name_or_digest(file_name: file_name, digest: digest) }
 
     context 'no manifest exists' do
-      it 'initializes a manifest' do
-        expect(DependencyProxy::Manifest).to receive(:new).with(file_name: file_name, digest: digest)
-
-        subject
-      end
+      it { is_expected.to be_nil }
     end
 
     context 'manifest exists and matches file_name' do

spec/requests/rack_attack_global_spec.rb

@@ -517,11 +517,15 @@ RSpec.describe 'Rack Attack global throttles', :use_clean_rails_memory_store_cac
       let(:path) { "/v2/#{group.path}/dependency_proxy/containers/alpine/manifests/latest" }
       let(:other_path) { "/v2/#{other_group.path}/dependency_proxy/containers/alpine/manifests/latest" }
       let(:pull_response) { { status: :success, manifest: manifest, from_cache: false } }
+      let(:head_response) { { status: :success } }
 
       before do
         allow_next_instance_of(DependencyProxy::FindOrCreateManifestService) do |instance|
           allow(instance).to receive(:execute).and_return(pull_response)
         end
+        allow_next_instance_of(DependencyProxy::HeadManifestService) do |instance|
+          allow(instance).to receive(:execute).and_return(head_response)
+        end
       end
 
       it_behaves_like 'rate-limited token-authenticated requests'

spec/routing/group_routing_spec.rb

@@ -85,6 +85,26 @@ RSpec.describe "Groups", "routing" do
       expect(get('/v2')).to route_to('groups/dependency_proxy_auth#authenticate')
     end
 
+    it 'routes to #upload_manifest' do
+      expect(post('v2/gitlabhq/dependency_proxy/containers/alpine/manifests/latest/upload'))
+        .to route_to('groups/dependency_proxy_for_containers#upload_manifest', group_id: 'gitlabhq', image: 'alpine', tag: 'latest')
+    end
+
+    it 'routes to #upload_blob' do
+      expect(post('v2/gitlabhq/dependency_proxy/containers/alpine/blobs/abc12345/upload'))
+        .to route_to('groups/dependency_proxy_for_containers#upload_blob', group_id: 'gitlabhq', image: 'alpine', sha: 'abc12345')
+    end
+
+    it 'routes to #upload_manifest_authorize' do
+      expect(post('v2/gitlabhq/dependency_proxy/containers/alpine/manifests/latest/upload/authorize'))
+        .to route_to('groups/dependency_proxy_for_containers#authorize_upload_manifest', group_id: 'gitlabhq', image: 'alpine', tag: 'latest')
+    end
+
+    it 'routes to #upload_blob_authorize' do
+      expect(post('v2/gitlabhq/dependency_proxy/containers/alpine/blobs/abc12345/upload/authorize'))
+        .to route_to('groups/dependency_proxy_for_containers#authorize_upload_blob', group_id: 'gitlabhq', image: 'alpine', sha: 'abc12345')
+    end
+
     context 'image name without namespace' do
       it 'routes to #manifest' do
         expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/manifests/2.3.6'))

spec/services/dependency_proxy/find_or_create_manifest_service_spec.rb

@@ -31,6 +31,14 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
       end
     end
 
+    shared_examples 'returning no manifest' do
+      it 'returns a nil manifest' do
+        expect(subject[:status]).to eq(:success)
+        expect(subject[:from_cache]).to eq false
+        expect(subject[:manifest]).to be_nil
+      end
+    end
+
     context 'when no manifest exists' do
       let_it_be(:image) { 'new-image' }
 
@@ -40,7 +48,15 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
           stub_manifest_download(image, tag, headers: headers)
         end
 
-        it_behaves_like 'downloading the manifest'
+        it_behaves_like 'returning no manifest'
+
+        context 'with dependency_proxy_manifest_workhorse feature disabled' do
+          before do
+            stub_feature_flags(dependency_proxy_manifest_workhorse: false)
+          end
+
+          it_behaves_like 'downloading the manifest'
+        end
       end
 
       context 'failed head request' do
@@ -49,7 +65,15 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
           stub_manifest_download(image, tag, headers: headers)
         end
 
-        it_behaves_like 'downloading the manifest'
+        it_behaves_like 'returning no manifest'
+
+        context 'with dependency_proxy_manifest_workhorse feature disabled' do
+          before do
+            stub_feature_flags(dependency_proxy_manifest_workhorse: false)
+          end
+
+          it_behaves_like 'downloading the manifest'
+        end
       end
     end
 
@@ -60,7 +84,7 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
 
       shared_examples 'using the cached manifest' do
         it 'uses cached manifest instead of downloading one', :aggregate_failures do
-          expect { subject }.to change { dependency_proxy_manifest.reload.updated_at }
+          subject
 
           expect(subject[:status]).to eq(:success)
           expect(subject[:manifest]).to be_a(DependencyProxy::Manifest)
@@ -80,12 +104,20 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
           stub_manifest_download(image, tag, headers: { 'docker-content-digest' => digest, 'content-type' => content_type })
         end
 
-        it 'downloads the new manifest and updates the existing record', :aggregate_failures do
-          expect(subject[:status]).to eq(:success)
-          expect(subject[:manifest]).to eq(dependency_proxy_manifest)
-          expect(subject[:manifest].content_type).to eq(content_type)
-          expect(subject[:manifest].digest).to eq(digest)
-          expect(subject[:from_cache]).to eq false
+        it_behaves_like 'returning no manifest'
+
+        context 'with dependency_proxy_manifest_workhorse feature disabled' do
+          before do
+            stub_feature_flags(dependency_proxy_manifest_workhorse: false)
+          end
+
+          it 'downloads the new manifest and updates the existing record', :aggregate_failures do
+            expect(subject[:status]).to eq(:success)
+            expect(subject[:manifest]).to eq(dependency_proxy_manifest)
+            expect(subject[:manifest].content_type).to eq(content_type)
+            expect(subject[:manifest].digest).to eq(digest)
+            expect(subject[:from_cache]).to eq false
+          end
         end
       end
 
@@ -96,7 +128,15 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
           stub_manifest_download(image, tag, headers: headers)
         end
 
-        it_behaves_like 'downloading the manifest'
+        it_behaves_like 'returning no manifest'
+
+        context 'with dependency_proxy_manifest_workhorse feature disabled' do
+          before do
+            stub_feature_flags(dependency_proxy_manifest_workhorse: false)
+          end
+
+          it_behaves_like 'downloading the manifest'
+        end
       end
 
       context 'failed connection' do

workhorse/internal/dependencyproxy/dependencyproxy.go

@@ -75,6 +75,19 @@ func (p *Injector) Inject(w http.ResponseWriter, r *http.Request, sendData strin
 		helper.Fail500(w, r, fmt.Errorf("dependency proxy: failed to create request: %w", err))
 	}
 	saveFileRequest.Header = helper.HeaderClone(r.Header)
+
+	// forward headers from dependencyResponse to rails and client
+	for key, values := range dependencyResponse.Header {
+		saveFileRequest.Header.Del(key)
+		w.Header().Del(key)
+		for _, value := range values {
+			saveFileRequest.Header.Add(key, value)
+			w.Header().Add(key, value)
+		}
+	}
+
+	// workhorse hijack overwrites the Content-Type header, but we need this header value
+	saveFileRequest.Header.Set("Workhorse-Proxy-Content-Type", dependencyResponse.Header.Get("Content-Type"))
 	saveFileRequest.ContentLength = dependencyResponse.ContentLength
 
 	nrw := &nullResponseWriter{header: make(http.Header)}

workhorse/internal/dependencyproxy/dependencyproxy_test.go

@@ -113,8 +113,14 @@ func TestInject(t *testing.T) {
 func TestSuccessfullRequest(t *testing.T) {
 	content := []byte("result")
 	contentLength := strconv.Itoa(len(content))
+	contentType := "foo"
+	dockerContentDigest := "sha256:asdf1234"
+	overriddenHeader := "originResourceServer"
 	originResourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Length", contentLength)
+		w.Header().Set("Content-Type", contentType)
+		w.Header().Set("Docker-Content-Digest", dockerContentDigest)
+		w.Header().Set("Overridden-Header", overriddenHeader)
 		w.Write(content)
 	}))
 
@@ -131,12 +137,16 @@ func TestSuccessfullRequest(t *testing.T) {
 
 	require.Equal(t, "/target/upload", uploadHandler.request.URL.Path)
 	require.Equal(t, int64(6), uploadHandler.request.ContentLength)
+	require.Equal(t, contentType, uploadHandler.request.Header.Get("Workhorse-Proxy-Content-Type"))
+	require.Equal(t, dockerContentDigest, uploadHandler.request.Header.Get("Docker-Content-Digest"))
+	require.Equal(t, overriddenHeader, uploadHandler.request.Header.Get("Overridden-Header"))
 
 	require.Equal(t, content, uploadHandler.body)
 
 	require.Equal(t, 200, response.Code)
 	require.Equal(t, string(content), response.Body.String())
 	require.Equal(t, contentLength, response.Header().Get("Content-Length"))
+	require.Equal(t, dockerContentDigest, response.Header().Get("Docker-Content-Digest"))
 }
 
 func TestIncorrectSendData(t *testing.T) {
@@ -177,6 +187,7 @@ func TestFailedOriginServer(t *testing.T) {
 func makeRequest(injector *Injector, data string) *httptest.ResponseRecorder {
 	w := httptest.NewRecorder()
 	r := httptest.NewRequest("GET", "/target", nil)
+	r.Header.Set("Overridden-Header", "request")
 
 	sendData := base64.StdEncoding.EncodeToString([]byte(data))
 	injector.Inject(w, r, sendData)