Commit 00b4f6fd authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch '290944-dependency-proxy-content-type' into 'master'

Proxy dependency proxy manifest content-type

See merge request gitlab-org/gitlab!53667
parents 2fbfa64f 9a794c07
......@@ -16,7 +16,18 @@ class Groups::DependencyProxyForContainersController < Groups::ApplicationContro
result = DependencyProxy::FindOrCreateManifestService.new(group, image, tag, token).execute
if result[:status] == :success
send_upload(result[:manifest].file)
response.headers['Docker-Content-Digest'] = result[:manifest].digest
response.headers['Content-Length'] = result[:manifest].size
response.headers['Docker-Distribution-Api-Version'] = DependencyProxy::DISTRIBUTION_API_VERSION
response.headers['Etag'] = "\"#{result[:manifest].digest}\""
content_type = result[:manifest].content_type
send_upload(
result[:manifest].file,
proxy: true,
redirect_params: { query: { 'response-content-type' => content_type } },
send_params: { type: content_type }
)
else
render status: result[:http_status], json: result[:message]
end
......
# frozen_string_literal: true
module DependencyProxy
URL_SUFFIX = '/dependency_proxy/containers'
DISTRIBUTION_API_VERSION = 'registry/2.0'
def self.table_name_prefix
'dependency_proxy_'
......
......@@ -12,5 +12,10 @@ class DependencyProxy::Manifest < ApplicationRecord
mount_file_store_uploader DependencyProxy::FileUploader
scope :find_or_initialize_by_file_name, ->(file_name) { find_or_initialize_by(file_name: file_name) }
def self.find_or_initialize_by_file_name_or_digest(file_name:, digest:)
result = find_by(file_name: file_name) || find_by(digest: digest)
return result if result
new(file_name: file_name, digest: digest)
end
end
......@@ -13,7 +13,7 @@ module DependencyProxy
def execute
@manifest = @group.dependency_proxy_manifests
.find_or_initialize_by_file_name(@file_name)
.find_or_initialize_by_file_name_or_digest(file_name: @file_name, digest: @tag)
head_result = DependencyProxy::HeadManifestService.new(@image, @tag, @token).execute
......@@ -30,6 +30,7 @@ module DependencyProxy
def pull_new_manifest
DependencyProxy::PullManifestService.new(@image, @tag, @token).execute_with_manifest do |new_manifest|
@manifest.update!(
content_type: new_manifest[:content_type],
digest: new_manifest[:digest],
file: new_manifest[:file],
size: new_manifest[:file].size
......@@ -38,7 +39,9 @@ module DependencyProxy
end
def cached_manifest_matches?(head_result)
@manifest && @manifest.digest == head_result[:digest]
return false if head_result[:status] == :error
@manifest && @manifest.digest == head_result[:digest] && @manifest.content_type == head_result[:content_type]
end
def respond
......
......@@ -2,6 +2,8 @@
module DependencyProxy
class HeadManifestService < DependencyProxy::BaseService
ACCEPT_HEADERS = ::ContainerRegistry::Client::ACCEPTED_TYPES.join(',')
def initialize(image, tag, token)
@image = image
@tag = tag
......@@ -9,10 +11,10 @@ module DependencyProxy
end
def execute
response = Gitlab::HTTP.head(manifest_url, headers: auth_headers)
response = Gitlab::HTTP.head(manifest_url, headers: auth_headers.merge(Accept: ACCEPT_HEADERS))
if response.success?
success(digest: response.headers['docker-content-digest'])
success(digest: response.headers['docker-content-digest'], content_type: response.headers['content-type'])
else
error(response.body, response.code)
end
......
......@@ -11,7 +11,7 @@ module DependencyProxy
def execute_with_manifest
raise ArgumentError, 'Block must be provided' unless block_given?
response = Gitlab::HTTP.get(manifest_url, headers: auth_headers)
response = Gitlab::HTTP.get(manifest_url, headers: auth_headers.merge(Accept: ::ContainerRegistry::Client::ACCEPTED_TYPES.join(',')))
if response.success?
file = Tempfile.new
......@@ -20,7 +20,7 @@ module DependencyProxy
file.write(response)
file.flush
yield(success(file: file, digest: response.headers['docker-content-digest']))
yield(success(file: file, digest: response.headers['docker-content-digest'], content_type: response.headers['content-type']))
ensure
file.close
file.unlink
......
......@@ -3,6 +3,7 @@
class DependencyProxy::FileUploader < GitlabUploader
include ObjectStorage::Concern
before :cache, :set_content_type
storage_options Gitlab.config.dependency_proxy
alias_method :upload, :model
......@@ -17,6 +18,17 @@ class DependencyProxy::FileUploader < GitlabUploader
private
# Docker manifests return a custom content type
# GCP will only use the content-type that is stored with the file
# and will not allow it to be overwritten when downloaded
# so we must store the custom content type in object storage.
# This does not apply to DependencyProxy::Blob uploads.
def set_content_type(file)
return unless model.class == DependencyProxy::Manifest
file.content_type = model.content_type
end
def dynamic_segment
Gitlab::HashedPath.new('dependency_proxy', model.group_id, 'files', model.id, root_hash: model.group_id)
end
......
......@@ -4,12 +4,13 @@ group: Package
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Dependency Proxy
# Dependency Proxy **(FREE)**
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11.
> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 13.6.
> - [Support for private groups](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
> - Anonymous access to images in public groups is no longer available starting in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to GitLab Free in GitLab 13.6.
> - [Support for private groups](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab Free 13.7.
> - Anonymous access to images in public groups is no longer available starting in GitLab Free 13.7.
> - [Support for pull-by-digest and Docker version 20.x](https://gitlab.com/gitlab-org/gitlab/-/issues/290944) in GitLab Free 13.10.
The GitLab Dependency Proxy is a local proxy you can use for your frequently-accessed
upstream images.
......@@ -17,11 +18,6 @@ upstream images.
In the case of CI/CD, the Dependency Proxy receives a request and returns the
upstream image from a registry, acting as a pull-through cache.
NOTE:
The Dependency Proxy is not compatible with Docker version 20.x and later.
If you are using the Dependency Proxy, Docker version 19.x.x is recommended until
[issue #290944](https://gitlab.com/gitlab-org/gitlab/-/issues/290944) is resolved.
## Prerequisites
The Dependency Proxy must be [enabled by an administrator](../../../administration/packages/dependency_proxy.md).
......@@ -60,7 +56,7 @@ Prerequisites:
### Authenticate with the Dependency Proxy
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab Free 13.7.
> - It's [deployed behind a feature flag](../../feature_flags.md), enabled by default.
> - It's enabled on GitLab.com.
> - It's recommended for production use.
......@@ -162,7 +158,7 @@ the [Dependency Proxy API](../../../api/dependency_proxy.md).
## Docker Hub rate limits and the Dependency Proxy
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241639) in [GitLab Core](https://about.gitlab.com/pricing/) 13.7.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241639) in GitLab Free 13.7.
<i class="fa fa-youtube-play youtube" aria-hidden="true"></i>
Watch how to [use the Dependency Proxy to help avoid Docker Hub rate limits](https://youtu.be/Nc4nUo7Pq08).
......
......@@ -130,7 +130,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
}
end
it 'proxies status from the remote token request' do
it 'proxies status from the remote token request', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(:service_unavailable)
......@@ -147,7 +147,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
}
end
it 'proxies status from the remote manifest request' do
it 'proxies status from the remote manifest request', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(:bad_request)
......@@ -156,7 +156,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
end
it 'sends a file' do
expect(controller).to receive(:send_file).with(manifest.file.path, {})
expect(controller).to receive(:send_file).with(manifest.file.path, type: manifest.content_type)
subject
end
......@@ -165,6 +165,10 @@ RSpec.describe Groups::DependencyProxyForContainersController do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.headers['Docker-Content-Digest']).to eq(manifest.digest)
expect(response.headers['Content-Length']).to eq(manifest.size)
expect(response.headers['Docker-Distribution-Api-Version']).to eq(DependencyProxy::DISTRIBUTION_API_VERSION)
expect(response.headers['Etag']).to eq("\"#{manifest.digest}\"")
expect(response.headers['Content-Disposition']).to match(/^attachment/)
end
end
......@@ -207,7 +211,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
}
end
it 'proxies status from the remote blob request' do
it 'proxies status from the remote blob request', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(:bad_request)
......@@ -221,7 +225,7 @@ RSpec.describe Groups::DependencyProxyForContainersController do
subject
end
it 'returns Content-Disposition: attachment' do
it 'returns Content-Disposition: attachment', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(:ok)
......
......@@ -10,7 +10,8 @@ FactoryBot.define do
factory :dependency_proxy_manifest, class: 'DependencyProxy::Manifest' do
group
file { fixture_file_upload('spec/fixtures/dependency_proxy/manifest') }
digest { 'sha256:5ab5a6872b264fe4fd35d63991b9b7d8425f4bc79e7cf4d563c10956581170c9' }
digest { 'sha256:d0710affa17fad5f466a70159cc458227bd25d4afb39514ef662ead3e6c99515' }
file_name { 'alpine:latest.json' }
content_type { 'application/vnd.docker.distribution.manifest.v2+json' }
end
end
{
"schemaVersion": 1,
"name": "library/alpine",
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 1472,
"digest": "sha256:7731472c3f2a25edbb9c085c78f42ec71259f2b83485aa60648276d408865839"
},
"layers": [
{
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
},
{
"blobSum": "sha256:188c0c94c7c576fff0792aca7ec73d67a2f7f4cb3a6e53a84559337260b36964"
}
],
"history": [
{
"v1Compatibility": "{\"architecture\":\"amd64\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\"],\"ArgsEscaped\":true,\"Image\":\"sha256:3543079adc6fb5170279692361be8b24e89ef1809a374c1b4429e1d560d1459c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":null},\"container\":\"8c59eb170e19b8c3768b8d06c91053b0debf4a6fa6a452df394145fe9b885ea5\",\"container_config\":{\"Hostname\":\"8c59eb170e19\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) \",\"CMD [\\\"/bin/sh\\\"]\"],\"ArgsEscaped\":true,\"Image\":\"sha256:3543079adc6fb5170279692361be8b24e89ef1809a374c1b4429e1d560d1459c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"created\":\"2020-10-22T02:19:24.499382102Z\",\"docker_version\":\"18.09.7\",\"id\":\"c5f1aab5bb88eaf1aa62bea08ea6654547d43fd4d15b1a476c77e705dd5385ba\",\"os\":\"linux\",\"parent\":\"dc0b50cc52bc340d7848a62cfe8a756f4420592f4984f7a680ef8f9d258176ed\",\"throwaway\":true}"
},
{
"v1Compatibility": "{\"id\":\"dc0b50cc52bc340d7848a62cfe8a756f4420592f4984f7a680ef8f9d258176ed\",\"created\":\"2020-10-22T02:19:24.33416307Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) ADD file:f17f65714f703db9012f00e5ec98d0b2541ff6147c2633f7ab9ba659d0c507f4 in / \"]}}"
}
],
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "XOTE:DZ4C:YBPJ:3O3L:YI4B:NYXU:T4VR:USH6:CXXN:SELU:CSCC:FVPE",
"kty": "EC",
"x": "cR1zye_3354mdbD7Dn-mtXNXvtPtmLlUVDa5vH6Lp74",
"y": "rldUXSllLit6_2BW6AV8aqkwWJXHoYPG9OwkIBouwxQ"
},
"alg": "ES256"
},
"signature": "DYB2iB-XKIisqp5Q0OXFOBIOlBOuRV7pnZuKy0cxVB2Qj1VFRhWX4Tq336y0VMWbF6ma1he5A1E_Vk4jazrJ9g",
"protected": "eyJmb3JtYXRMZW5ndGgiOjIxMzcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAyMC0xMS0yNFQyMjowMTo1MVoifQ"
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 2810825,
"digest": "sha256:596ba82af5aaa3e2fd9d6f955b8b94f0744a2b60710e3c243ba3e4a467f051d1"
}
]
}
\ No newline at end of file
......@@ -29,24 +29,32 @@ RSpec.describe DependencyProxy::Manifest, type: :model do
end
end
describe '.find_or_initialize_by_file_name' do
subject { DependencyProxy::Manifest.find_or_initialize_by_file_name(file_name) }
describe '.find_or_initialize_by_file_name_or_digest' do
let_it_be(:file_name) { 'foo' }
let_it_be(:digest) { 'bar' }
context 'no manifest exists' do
let_it_be(:file_name) { 'foo' }
subject { DependencyProxy::Manifest.find_or_initialize_by_file_name_or_digest(file_name: file_name, digest: digest) }
context 'no manifest exists' do
it 'initializes a manifest' do
expect(DependencyProxy::Manifest).to receive(:new).with(file_name: file_name)
expect(DependencyProxy::Manifest).to receive(:new).with(file_name: file_name, digest: digest)
subject
end
end
context 'manifest exists' do
context 'manifest exists and matches file_name' do
let_it_be(:dependency_proxy_manifest) { create(:dependency_proxy_manifest) }
let_it_be(:file_name) { dependency_proxy_manifest.file_name }
it { is_expected.to eq(dependency_proxy_manifest) }
end
context 'manifest exists and matches digest' do
let_it_be(:dependency_proxy_manifest) { create(:dependency_proxy_manifest) }
let_it_be(:digest) { dependency_proxy_manifest.digest }
it { is_expected.to eq(dependency_proxy_manifest) }
end
end
end
......@@ -10,7 +10,12 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
let(:manifest) { dependency_proxy_manifest.file.read }
let(:group) { dependency_proxy_manifest.group }
let(:token) { Digest::SHA256.hexdigest('123') }
let(:headers) { { 'docker-content-digest' => dependency_proxy_manifest.digest } }
let(:headers) do
{
'docker-content-digest' => dependency_proxy_manifest.digest,
'content-type' => dependency_proxy_manifest.content_type
}
end
describe '#execute' do
subject { described_class.new(group, image, tag, token).execute }
......@@ -18,22 +23,37 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
context 'when no manifest exists' do
let_it_be(:image) { 'new-image' }
before do
stub_manifest_head(image, tag, digest: dependency_proxy_manifest.digest)
stub_manifest_download(image, tag, headers: headers)
shared_examples 'downloading the manifest' do
it 'downloads manifest from remote registry if there is no cached one', :aggregate_failures do
expect { subject }.to change { group.dependency_proxy_manifests.count }.by(1)
expect(subject[:status]).to eq(:success)
expect(subject[:manifest]).to be_a(DependencyProxy::Manifest)
expect(subject[:manifest]).to be_persisted
end
end
it 'downloads manifest from remote registry if there is no cached one', :aggregate_failures do
expect { subject }.to change { group.dependency_proxy_manifests.count }.by(1)
expect(subject[:status]).to eq(:success)
expect(subject[:manifest]).to be_a(DependencyProxy::Manifest)
expect(subject[:manifest]).to be_persisted
context 'successful head request' do
before do
stub_manifest_head(image, tag, headers: headers)
stub_manifest_download(image, tag, headers: headers)
end
it_behaves_like 'downloading the manifest'
end
context 'failed head request' do
before do
stub_manifest_head(image, tag, status: :error)
stub_manifest_download(image, tag, headers: headers)
end
it_behaves_like 'downloading the manifest'
end
end
context 'when manifest exists' do
before do
stub_manifest_head(image, tag, digest: dependency_proxy_manifest.digest)
stub_manifest_head(image, tag, headers: headers)
end
shared_examples 'using the cached manifest' do
......@@ -48,15 +68,17 @@ RSpec.describe DependencyProxy::FindOrCreateManifestService do
context 'when digest is stale' do
let(:digest) { 'new-digest' }
let(:content_type) { 'new-content-type' }
before do
stub_manifest_head(image, tag, digest: digest)
stub_manifest_download(image, tag, headers: { 'docker-content-digest' => digest })
stub_manifest_head(image, tag, headers: { 'docker-content-digest' => digest, 'content-type' => content_type })
stub_manifest_download(image, tag, headers: { 'docker-content-digest' => digest, 'content-type' => content_type })
end
it 'downloads the new manifest and updates the existing record', :aggregate_failures do
expect(subject[:status]).to eq(:success)
expect(subject[:manifest]).to eq(dependency_proxy_manifest)
expect(subject[:manifest].content_type).to eq(content_type)
expect(subject[:manifest].digest).to eq(digest)
end
end
......
......@@ -8,12 +8,19 @@ RSpec.describe DependencyProxy::HeadManifestService do
let(:tag) { 'latest' }
let(:token) { Digest::SHA256.hexdigest('123') }
let(:digest) { '12345' }
let(:content_type) { 'foo' }
let(:headers) do
{
'docker-content-digest' => digest,
'content-type' => content_type
}
end
subject { described_class.new(image, tag, token).execute }
context 'remote request is successful' do
before do
stub_manifest_head(image, tag, digest: digest)
stub_manifest_head(image, tag, headers: headers)
end
it { expect(subject[:status]).to eq(:success) }
......
......@@ -9,7 +9,10 @@ RSpec.describe DependencyProxy::PullManifestService do
let(:token) { Digest::SHA256.hexdigest('123') }
let(:manifest) { { foo: 'bar' }.to_json }
let(:digest) { '12345' }
let(:headers) { { 'docker-content-digest' => digest } }
let(:content_type) { 'foo' }
let(:headers) do
{ 'docker-content-digest' => digest, 'content-type' => content_type }
end
subject { described_class.new(image, tag, token).execute_with_manifest(&method(:check_response)) }
......@@ -25,6 +28,7 @@ RSpec.describe DependencyProxy::PullManifestService do
expect(response[:status]).to eq(:success)
expect(response[:file].read).to eq(manifest)
expect(response[:digest]).to eq(digest)
expect(response[:content_type]).to eq(content_type)
end
subject
......
......@@ -18,11 +18,11 @@ module DependencyProxyHelpers
.to_return(status: status, body: body || manifest, headers: headers)
end
def stub_manifest_head(image, tag, status: 200, body: nil, digest: '123456')
def stub_manifest_head(image, tag, status: 200, body: nil, headers: {})
manifest_url = registry.manifest_url(image, tag)
stub_full_request(manifest_url, method: :head)
.to_return(status: status, body: body, headers: { 'docker-content-digest' => digest } )
.to_return(status: status, body: body, headers: headers )
end
def stub_blob_download(image, blob_sha, status = 200, body = '123456')
......
......@@ -2,25 +2,43 @@
require 'spec_helper'
RSpec.describe DependencyProxy::FileUploader do
let(:blob) { create(:dependency_proxy_blob) }
let(:uploader) { described_class.new(blob, :file) }
let(:path) { Gitlab.config.dependency_proxy.storage_path }
describe 'DependencyProxy::Blob uploader' do
let_it_be(:blob) { create(:dependency_proxy_blob) }
let_it_be(:path) { Gitlab.config.dependency_proxy.storage_path }
let(:uploader) { described_class.new(blob, :file) }
subject { uploader }
subject { uploader }
it_behaves_like "builds correct paths",
store_dir: %r[\h{2}/\h{2}],
cache_dir: %r[/dependency_proxy/tmp/cache],
work_dir: %r[/dependency_proxy/tmp/work]
it_behaves_like "builds correct paths",
store_dir: %r[\h{2}/\h{2}],
cache_dir: %r[/dependency_proxy/tmp/cache],
work_dir: %r[/dependency_proxy/tmp/work]
context 'object store is remote' do
before do
stub_dependency_proxy_object_storage
end
context 'object store is remote' do
before do
stub_dependency_proxy_object_storage
include_context 'with storage', described_class::Store::REMOTE
it_behaves_like "builds correct paths",
store_dir: %r[\h{2}/\h{2}]
end
end
include_context 'with storage', described_class::Store::REMOTE
describe 'DependencyProxy::Manifest uploader' do
let_it_be(:manifest) { create(:dependency_proxy_manifest) }
let_it_be(:initial_content_type) { 'application/json' }
let_it_be(:fixture_file) { fixture_file_upload('spec/fixtures/dependency_proxy/manifest', initial_content_type) }
let(:uploader) { described_class.new(manifest, :file) }
it_behaves_like "builds correct paths",
store_dir: %r[\h{2}/\h{2}]
subject { uploader }
it 'will change upload file content type to match the model content type', :aggregate_failures do
uploader.cache!(fixture_file)
expect(uploader.file.content_type).to eq(manifest.content_type)
expect(uploader.file.content_type).not_to eq(initial_content_type)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment