Proxy dependency proxy manifest content-type

Store the manifest content-type that is received
when pulling a manifest and use this content-type
in the manifest response.

app/controllers/groups/dependency_proxy_for_containers_controller.rb

@@ -16,7 +16,12 @@ class Groups::DependencyProxyForContainersController < Groups::ApplicationContro
     result = DependencyProxy::FindOrCreateManifestService.new(group, image, tag, token).execute
 
     if result[:status] == :success
-      send_upload(result[:manifest].file)
+      response.headers['Docker-Content-Digest'] = result[:manifest].digest
+      response.headers['Content-Length'] = result[:manifest].size
+      response.headers['Docker-Distribution-Api-Version'] = DependencyProxy::DISTRIBUTION_API_VERSION
+      response.headers['Etag'] = "\"#{result[:manifest].digest}\""
+
+      send_upload(result[:manifest].file, send_params: { type: result[:manifest].content_type })
     else
       render status: result[:http_status], json: result[:message]
     end

app/models/dependency_proxy.rb

 # frozen_string_literal: true
 module DependencyProxy
   URL_SUFFIX = '/dependency_proxy/containers'
+  DISTRIBUTION_API_VERSION = 'registry/2.0'
 
   def self.table_name_prefix
     'dependency_proxy_'

app/models/dependency_proxy/manifest.rb

@@ -12,5 +12,10 @@ class DependencyProxy::Manifest < ApplicationRecord
 
   mount_file_store_uploader DependencyProxy::FileUploader
 
-  scope :find_or_initialize_by_file_name, ->(file_name) { find_or_initialize_by(file_name: file_name) }
+  def self.find_or_initialize_by_file_name_or_digest(file_name:, digest:)
+    result = find_by(file_name: file_name) || find_by(digest: digest)
+    return result if result
+
+    new(file_name: file_name, digest: digest)
+  end
 end

app/services/dependency_proxy/find_or_create_manifest_service.rb

@@ -13,7 +13,7 @@ module DependencyProxy
 
     def execute
       @manifest = @group.dependency_proxy_manifests
-                        .find_or_initialize_by_file_name(@file_name)
+                        .find_or_initialize_by_file_name_or_digest(file_name: @file_name, digest: @tag)
 
       head_result = DependencyProxy::HeadManifestService.new(@image, @tag, @token).execute
 
@@ -30,6 +30,7 @@ module DependencyProxy
     def pull_new_manifest
       DependencyProxy::PullManifestService.new(@image, @tag, @token).execute_with_manifest do |new_manifest|
         @manifest.update!(
+          content_type: new_manifest[:content_type],
           digest: new_manifest[:digest],
           file: new_manifest[:file],
           size: new_manifest[:file].size
@@ -38,7 +39,9 @@ module DependencyProxy
     end
 
     def cached_manifest_matches?(head_result)
-      @manifest && @manifest.digest == head_result[:digest]
+      return false if head_result[:status] == :error
+
+      @manifest && @manifest.digest == head_result[:digest] && @manifest.content_type == head_result[:content_type]
     end
 
     def respond

app/services/dependency_proxy/head_manifest_service.rb

@@ -2,6 +2,8 @@
 
 module DependencyProxy
   class HeadManifestService < DependencyProxy::BaseService
+    ACCEPT_HEADERS = ::ContainerRegistry::Client::ACCEPTED_TYPES.join(',')
+
     def initialize(image, tag, token)
       @image = image
       @tag = tag
@@ -9,10 +11,10 @@ module DependencyProxy
     end
 
     def execute
-      response = Gitlab::HTTP.head(manifest_url, headers: auth_headers)
+      response = Gitlab::HTTP.head(manifest_url, headers: auth_headers.merge(Accept: ACCEPT_HEADERS))
 
       if response.success?
-        success(digest: response.headers['docker-content-digest'])
+        success(digest: response.headers['docker-content-digest'], content_type: response.headers['content-type'])
       else
         error(response.body, response.code)
       end

app/services/dependency_proxy/pull_manifest_service.rb

@@ -11,7 +11,7 @@ module DependencyProxy
     def execute_with_manifest
       raise ArgumentError, 'Block must be provided' unless block_given?
 
-      response = Gitlab::HTTP.get(manifest_url, headers: auth_headers)
+      response = Gitlab::HTTP.get(manifest_url, headers: auth_headers.merge(Accept: ::ContainerRegistry::Client::ACCEPTED_TYPES.join(',')))
 
       if response.success?
         file = Tempfile.new
@@ -20,7 +20,7 @@ module DependencyProxy
           file.write(response)
           file.flush
 
-          yield(success(file: file, digest: response.headers['docker-content-digest']))
+          yield(success(file: file, digest: response.headers['docker-content-digest'], content_type: response.headers['content-type']))
         ensure
           file.close
           file.unlink

doc/user/packages/dependency_proxy/index.md

@@ -4,12 +4,13 @@ group: Package
 info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
 ---
 
-# Dependency Proxy
+# Dependency Proxy **(FREE)**
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11.
-> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 13.6.
-> - [Support for private groups](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
-> - Anonymous access to images in public groups is no longer available starting in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
+> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to GitLab Free in GitLab 13.6.
+> - [Support for private groups](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab Free 13.7.
+> - Anonymous access to images in public groups is no longer available starting in GitLab Free 13.7.
+> - [Support for pull-by-digest and Docker version 20.x](https://gitlab.com/gitlab-org/gitlab/-/issues/290944) in GitLab Free 13.10.
 
 The GitLab Dependency Proxy is a local proxy you can use for your frequently-accessed
 upstream images.
@@ -17,11 +18,6 @@ upstream images.
 In the case of CI/CD, the Dependency Proxy receives a request and returns the
 upstream image from a registry, acting as a pull-through cache.
 
-NOTE:
-The Dependency Proxy is not compatible with Docker version 20.x and later.
-If you are using the Dependency Proxy, Docker version 19.x.x is recommended until
-[issue #290944](https://gitlab.com/gitlab-org/gitlab/-/issues/290944) is resolved.
-
 ## Prerequisites
 
 The Dependency Proxy must be [enabled by an administrator](../../../administration/packages/dependency_proxy.md).
@@ -60,7 +56,7 @@ Prerequisites:
 
 ### Authenticate with the Dependency Proxy
 
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab Free 13.7.
 > - It's [deployed behind a feature flag](../../feature_flags.md), enabled by default.
 > - It's enabled on GitLab.com.
 > - It's recommended for production use.
@@ -162,7 +158,7 @@ the [Dependency Proxy API](../../../api/dependency_proxy.md).
 
 ## Docker Hub rate limits and the Dependency Proxy
 
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241639) in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241639) in GitLab Free 13.7.
 
 <i class="fa fa-youtube-play youtube" aria-hidden="true"></i>
 Watch how to [use the Dependency Proxy to help avoid Docker Hub rate limits](https://youtu.be/Nc4nUo7Pq08).

spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb

@@ -130,7 +130,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
           }
         end
 
-        it 'proxies status from the remote token request' do
+        it 'proxies status from the remote token request', :aggregate_failures do
           subject
 
           expect(response).to have_gitlab_http_status(:service_unavailable)
@@ -147,7 +147,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
           }
         end
 
-        it 'proxies status from the remote manifest request' do
+        it 'proxies status from the remote manifest request', :aggregate_failures do
           subject
 
           expect(response).to have_gitlab_http_status(:bad_request)
@@ -156,7 +156,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
       end
 
       it 'sends a file' do
-        expect(controller).to receive(:send_file).with(manifest.file.path, {})
+        expect(controller).to receive(:send_file).with(manifest.file.path, type: manifest.content_type)
 
         subject
       end
@@ -165,6 +165,10 @@ RSpec.describe Groups::DependencyProxyForContainersController do
         subject
 
         expect(response).to have_gitlab_http_status(:ok)
+        expect(response.headers['Docker-Content-Digest']).to eq(manifest.digest)
+        expect(response.headers['Content-Length']).to eq(manifest.size)
+        expect(response.headers['Docker-Distribution-Api-Version']).to eq(DependencyProxy::DISTRIBUTION_API_VERSION)
+        expect(response.headers['Etag']).to eq("\"#{manifest.digest}\"")
         expect(response.headers['Content-Disposition']).to match(/^attachment/)
       end
     end
@@ -207,7 +211,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
           }
         end
 
-        it 'proxies status from the remote blob request' do
+        it 'proxies status from the remote blob request', :aggregate_failures do
           subject
 
           expect(response).to have_gitlab_http_status(:bad_request)
@@ -221,7 +225,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
         subject
       end
 
-      it 'returns Content-Disposition: attachment' do
+      it 'returns Content-Disposition: attachment', :aggregate_failures do
         subject
 
         expect(response).to have_gitlab_http_status(:ok)

spec/factories/dependency_proxy.rb

@@ -10,7 +10,8 @@ FactoryBot.define do
   factory :dependency_proxy_manifest, class: 'DependencyProxy::Manifest' do
     group
     file { fixture_file_upload('spec/fixtures/dependency_proxy/manifest') }
-    digest { 'sha256:5ab5a6872b264fe4fd35d63991b9b7d8425f4bc79e7cf4d563c10956581170c9' }
+    digest { 'sha256:d0710affa17fad5f466a70159cc458227bd25d4afb39514ef662ead3e6c99515' }
     file_name { 'alpine:latest.json' }
+    content_type { 'application/vnd.docker.distribution.manifest.v2+json' }
   end
 end

spec/fixtures/dependency_proxy/manifest

 {
-   "schemaVersion": 1,
-   "name": "library/alpine",
-   "tag": "latest",
-   "architecture": "amd64",
-   "fsLayers": [
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "config": {
+      "mediaType": "application/vnd.docker.container.image.v1+json",
+      "size": 1472,
+      "digest": "sha256:7731472c3f2a25edbb9c085c78f42ec71259f2b83485aa60648276d408865839"
+   },
+   "layers": [
       {
-         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
-      },
-      {
-         "blobSum": "sha256:188c0c94c7c576fff0792aca7ec73d67a2f7f4cb3a6e53a84559337260b36964"
-      }
-   ],
-   "history": [
-      {
-         "v1Compatibility": "{\"architecture\":\"amd64\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\"],\"ArgsEscaped\":true,\"Image\":\"sha256:3543079adc6fb5170279692361be8b24e89ef1809a374c1b4429e1d560d1459c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":null},\"container\":\"8c59eb170e19b8c3768b8d06c91053b0debf4a6fa6a452df394145fe9b885ea5\",\"container_config\":{\"Hostname\":\"8c59eb170e19\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) \",\"CMD [\\\"/bin/sh\\\"]\"],\"ArgsEscaped\":true,\"Image\":\"sha256:3543079adc6fb5170279692361be8b24e89ef1809a374c1b4429e1d560d1459c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"created\":\"2020-10-22T02:19:24.499382102Z\",\"docker_version\":\"18.09.7\",\"id\":\"c5f1aab5bb88eaf1aa62bea08ea6654547d43fd4d15b1a476c77e705dd5385ba\",\"os\":\"linux\",\"parent\":\"dc0b50cc52bc340d7848a62cfe8a756f4420592f4984f7a680ef8f9d258176ed\",\"throwaway\":true}"
-      },
-      {
-         "v1Compatibility": "{\"id\":\"dc0b50cc52bc340d7848a62cfe8a756f4420592f4984f7a680ef8f9d258176ed\",\"created\":\"2020-10-22T02:19:24.33416307Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) ADD file:f17f65714f703db9012f00e5ec98d0b2541ff6147c2633f7ab9ba659d0c507f4 in / \"]}}"
-      }
-   ],
-   "signatures": [
-      {
-         "header": {
-            "jwk": {
-               "crv": "P-256",
-               "kid": "XOTE:DZ4C:YBPJ:3O3L:YI4B:NYXU:T4VR:USH6:CXXN:SELU:CSCC:FVPE",
-               "kty": "EC",
-               "x": "cR1zye_3354mdbD7Dn-mtXNXvtPtmLlUVDa5vH6Lp74",
-               "y": "rldUXSllLit6_2BW6AV8aqkwWJXHoYPG9OwkIBouwxQ"
-            },
-            "alg": "ES256"
-         },
-         "signature": "DYB2iB-XKIisqp5Q0OXFOBIOlBOuRV7pnZuKy0cxVB2Qj1VFRhWX4Tq336y0VMWbF6ma1he5A1E_Vk4jazrJ9g",
-         "protected": "eyJmb3JtYXRMZW5ndGgiOjIxMzcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAyMC0xMS0yNFQyMjowMTo1MVoifQ"
+         "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+         "size": 2810825,
+         "digest": "sha256:596ba82af5aaa3e2fd9d6f955b8b94f0744a2b60710e3c243ba3e4a467f051d1"
       }
    ]
 }
\ No newline at end of file

spec/models/dependency_proxy/manifest_spec.rb

@@ -29,24 +29,32 @@ RSpec.describe DependencyProxy::Manifest, type: :model do
     end
   end
 
-  describe '.find_or_initialize_by_file_name' do
-    subject { DependencyProxy::Manifest.find_or_initialize_by_file_name(file_name) }
+  describe '.find_or_initialize_by_file_name_or_digest' do
+    let_it_be(:file_name) { 'foo' }
+    let_it_be(:digest) { 'bar' }
 
-    context 'no manifest exists' do
-      let_it_be(:file_name) { 'foo' }
+    subject { DependencyProxy::Manifest.find_or_initialize_by_file_name_or_digest(file_name: file_name, digest: digest) }
 
+    context 'no manifest exists' do
       it 'initializes a manifest' do
-        expect(DependencyProxy::Manifest).to receive(:new).with(file_name: file_name)
+        expect(DependencyProxy::Manifest).to receive(:new).with(file_name: file_name, digest: digest)
 
         subject
       end
     end
 
-    context 'manifest exists' do
+    context 'manifest exists and matches file_name' do
       let_it_be(:dependency_proxy_manifest) { create(:dependency_proxy_manifest) }
       let_it_be(:file_name) { dependency_proxy_manifest.file_name }
 
       it { is_expected.to eq(dependency_proxy_manifest) }
     end
+
+    context 'manifest exists and matches digest' do
+      let_it_be(:dependency_proxy_manifest) { create(:dependency_proxy_manifest) }
+      let_it_be(:digest) { dependency_proxy_manifest.digest }
+
+      it { is_expected.to eq(dependency_proxy_manifest) }
+    end
   end
 end

spec/services/dependency_proxy/find_or_create_manifest_service_spec.rb

@@ -10,7 +10,12 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
   let(:manifest) { dependency_proxy_manifest.file.read }
   let(:group) { dependency_proxy_manifest.group }
   let(:token) { Digest::SHA256.hexdigest('123') }
-  let(:headers) { { 'docker-content-digest' => dependency_proxy_manifest.digest } }
+  let(:headers) do
+    {
+      'docker-content-digest' => dependency_proxy_manifest.digest,
+      'content-type' => dependency_proxy_manifest.content_type
+    }
+  end
 
   describe '#execute' do
     subject { described_class.new(group, image, tag, token).execute }
@@ -18,22 +23,37 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
     context 'when no manifest exists' do
       let_it_be(:image) { 'new-image' }
 
-      before do
-        stub_manifest_head(image, tag, digest: dependency_proxy_manifest.digest)
-        stub_manifest_download(image, tag, headers: headers)
+      shared_examples 'downloading the manifest' do
+        it 'downloads manifest from remote registry if there is no cached one', :aggregate_failures do
+          expect { subject }.to change { group.dependency_proxy_manifests.count }.by(1)
+          expect(subject[:status]).to eq(:success)
+          expect(subject[:manifest]).to be_a(DependencyProxy::Manifest)
+          expect(subject[:manifest]).to be_persisted
+        end
       end
 
-      it 'downloads manifest from remote registry if there is no cached one', :aggregate_failures do
-        expect { subject }.to change { group.dependency_proxy_manifests.count }.by(1)
-        expect(subject[:status]).to eq(:success)
-        expect(subject[:manifest]).to be_a(DependencyProxy::Manifest)
-        expect(subject[:manifest]).to be_persisted
+      context 'successful head request' do
+        before do
+          stub_manifest_head(image, tag, headers: headers)
+          stub_manifest_download(image, tag, headers: headers)
+        end
+
+        it_behaves_like 'downloading the manifest'
+      end
+
+      context 'failed head request' do
+        before do
+          stub_manifest_head(image, tag, status: :error)
+          stub_manifest_download(image, tag, headers: headers)
+        end
+
+        it_behaves_like 'downloading the manifest'
       end
     end
 
     context 'when manifest exists' do
       before do
-        stub_manifest_head(image, tag, digest: dependency_proxy_manifest.digest)
+        stub_manifest_head(image, tag, headers: headers)
       end
 
       shared_examples 'using the cached manifest' do
@@ -48,15 +68,17 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
 
       context 'when digest is stale' do
         let(:digest) { 'new-digest' }
+        let(:content_type) { 'new-content-type' }
 
         before do
-          stub_manifest_head(image, tag, digest: digest)
-          stub_manifest_download(image, tag, headers: { 'docker-content-digest' => digest })
+          stub_manifest_head(image, tag, headers: { 'docker-content-digest' => digest, 'content-type' => content_type })
+          stub_manifest_download(image, tag, headers: { 'docker-content-digest' => digest, 'content-type' => content_type })
         end
 
         it 'downloads the new manifest and updates the existing record', :aggregate_failures do
           expect(subject[:status]).to eq(:success)
           expect(subject[:manifest]).to eq(dependency_proxy_manifest)
+          expect(subject[:manifest].content_type).to eq(content_type)
           expect(subject[:manifest].digest).to eq(digest)
         end
       end

spec/services/dependency_proxy/head_manifest_service_spec.rb

@@ -8,12 +8,19 @@ RSpec.describe DependencyProxy::HeadManifestService do
   let(:tag) { 'latest' }
   let(:token) { Digest::SHA256.hexdigest('123') }
   let(:digest) { '12345' }
+  let(:content_type) { 'foo' }
+  let(:headers) do
+    {
+      'docker-content-digest' => digest,
+      'content-type' => content_type
+    }
+  end
 
   subject { described_class.new(image, tag, token).execute }
 
   context 'remote request is successful' do
     before do
-      stub_manifest_head(image, tag, digest: digest)
+      stub_manifest_head(image, tag, headers: headers)
     end
 
     it { expect(subject[:status]).to eq(:success) }

spec/services/dependency_proxy/pull_manifest_service_spec.rb

@@ -9,7 +9,10 @@ RSpec.describe DependencyProxy::PullManifestService do
   let(:token) { Digest::SHA256.hexdigest('123') }
   let(:manifest) { { foo: 'bar' }.to_json }
   let(:digest) { '12345' }
-  let(:headers) { { 'docker-content-digest' => digest } }
+  let(:content_type) { 'foo' }
+  let(:headers) do
+    { 'docker-content-digest' => digest, 'content-type' => content_type }
+  end
 
   subject { described_class.new(image, tag, token).execute_with_manifest(&method(:check_response)) }
 
@@ -25,6 +28,7 @@ RSpec.describe DependencyProxy::PullManifestService do
         expect(response[:status]).to eq(:success)
         expect(response[:file].read).to eq(manifest)
         expect(response[:digest]).to eq(digest)
+        expect(response[:content_type]).to eq(content_type)
       end
 
       subject

spec/support/helpers/dependency_proxy_helpers.rb

@@ -18,11 +18,11 @@ module DependencyProxyHelpers
       .to_return(status: status, body: body || manifest, headers: headers)
   end
 
-  def stub_manifest_head(image, tag, status: 200, body: nil, digest: '123456')
+  def stub_manifest_head(image, tag, status: 200, body: nil, headers: {})
     manifest_url = registry.manifest_url(image, tag)
 
     stub_full_request(manifest_url, method: :head)
-      .to_return(status: status, body: body, headers: { 'docker-content-digest' => digest } )
+      .to_return(status: status, body: body, headers: headers )
   end
 
   def stub_blob_download(image, blob_sha, status = 200, body = '123456')