Merge branch 'dz-registry-proxy' into 'master'

Dependency proxy for containers

See merge request gitlab-org/gitlab-ee!9750

ee/app/controllers/groups/dependency_proxy_for_containers_controller.rb

+# frozen_string_literal: true
+
+class Groups::DependencyProxyForContainersController < Groups::ApplicationController
+  include SendFileUpload
+
+  before_action :ensure_feature_enabled!
+  before_action :ensure_token_granted!
+
+  attr_reader :token
+
+  def manifest
+    result = DependencyProxy::PullManifestService.new(image, tag, token).execute
+
+    if result[:status] == :success
+      render json: result[:manifest]
+    else
+      render status: result[:http_status], json: result[:message]
+    end
+  end
+
+  def blob
+    result = DependencyProxy::FindOrCreateBlobService
+      .new(group, image, token, params[:sha]).execute
+
+    if result[:status] == :success
+      send_upload(result[:blob].file)
+    else
+      head result[:http_status]
+    end
+  end
+
+  private
+
+  def image
+    params[:image]
+  end
+
+  def tag
+    params[:tag]
+  end
+
+  def ensure_feature_enabled!
+    render_404 unless Gitlab.config.dependency_proxy.enabled &&
+        group.feature_available?(:dependency_proxy) &&
+        group.dependency_proxy_setting&.enabled
+  end
+
+  def ensure_token_granted!
+    result = DependencyProxy::RequestTokenService.new(image).execute
+
+    if result[:status] == :success
+      @token = result[:token]
+    else
+      render status: result[:http_status], json: result[:message]
+    end
+  end
+end

ee/app/services/dependency_proxy/base_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class BaseService < ::BaseService
+    private
+
+    def registry
+      DependencyProxy::Registry
+    end
+
+    def auth_headers
+      {
+        Authorization: "Bearer #{@token}"
+      }
+    end
+  end
+end

ee/app/services/dependency_proxy/download_blob_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class DownloadBlobService < DependencyProxy::BaseService
+    class DownloadError < StandardError
+      attr_reader :http_status
+
+      def initialize(message, http_status)
+        @http_status = http_status
+
+        super(message)
+      end
+    end
+
+    def initialize(image, blob_sha, token)
+      @image = image
+      @blob_sha = blob_sha
+      @token = token
+      @temp_file = Tempfile.new
+    end
+
+    def execute
+      File.open(@temp_file.path, "wb") do |file|
+        Gitlab::HTTP.get(blob_url, headers: auth_headers, stream_body: true) do |fragment|
+          if [301, 302, 307].include?(fragment.code)
+            # do nothing
+          elsif fragment.code == 200
+            file.write(fragment)
+          else
+            raise DownloadError.new('Non-success response code on downloading blob fragment', fragment.code)
+          end
+        end
+      end
+
+      success(file: @temp_file)
+    rescue DownloadError => exception
+      error(exception.message, exception.http_status)
+    rescue Timeout::Error => exception
+      error(exception.message, 599)
+    end
+
+    private
+
+    def blob_url
+      registry.blob_url(@image, @blob_sha)
+    end
+  end
+end

ee/app/services/dependency_proxy/find_or_create_blob_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class FindOrCreateBlobService < DependencyProxy::BaseService
+    def initialize(group, image, token, blob_sha)
+      @group = group
+      @image = image
+      @token = token
+      @blob_sha = blob_sha
+    end
+
+    def execute
+      file_name = @blob_sha.sub('sha256:', '') + '.gz'
+      blob = @group.dependency_proxy_blobs.find_or_build(file_name)
+
+      unless blob.persisted?
+        result = DependencyProxy::DownloadBlobService
+          .new(@image, @blob_sha, @token).execute
+
+        if result[:status] == :error
+          log_failure(result)
+
+          return error('Failed to download the blob', result[:http_status])
+        end
+
+        blob.file = result[:file]
+        blob.size = result[:file].size
+        blob.save!
+      end
+
+      success(blob: blob)
+    end
+
+    private
+
+    def log_failure(result)
+      log_error(
+        "Dependency proxy: Failed to download the blob." \
+        "Blob sha: #{@blob_sha}." \
+        "Error message: #{result[:message][0, 100]}" \
+        "HTTP status: #{result[:http_status]}"
+      )
+    end
+  end
+end

ee/app/services/dependency_proxy/pull_manifest_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class PullManifestService < DependencyProxy::BaseService
+    def initialize(image, tag, token)
+      @image = image
+      @tag = tag
+      @token = token
+    end
+
+    def execute
+      response = Gitlab::HTTP.get(manifest_url, headers: auth_headers)
+
+      if response.success?
+        success(manifest: response.body)
+      else
+        error(response.body, response.code)
+      end
+    rescue Timeout::Error => exception
+      error(exception.message, 599)
+    end
+
+    private
+
+    def manifest_url
+      registry.manifest_url(@image, @tag)
+    end
+  end
+end

ee/app/services/dependency_proxy/request_token_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class RequestTokenService < DependencyProxy::BaseService
+    def initialize(image)
+      @image = image
+    end
+
+    def execute
+      response = Gitlab::HTTP.get(auth_url)
+
+      if response.success?
+        success(token: JSON.parse(response.body)['token'])
+      else
+        error('Expected 200 response code for an access token', response.code)
+      end
+    rescue Timeout::Error => exception
+      error(exception.message, 599)
+    rescue JSON::ParserError
+      error('Failed to parse a response body for an access token', 500)
+    end
+
+    private
+
+    def auth_url
+      registry.auth_url(@image)
+    end
+  end
+end

ee/changelogs/unreleased/dz-registry-proxy.yml

+---
+title: Add dependency proxy for containers
+merge_request: 9750
+author:
+type: added

ee/config/routes/group.rb

@@ -114,3 +114,11 @@ constraints(::Constraints::GroupUrlConstrainer.new) do
                                           :pipeline_quota, :hooks, :boards)
   end
 end
+
+# Dependency proxy for containers
+# Because docker adds v2 prefix to URI this need to be outside of usual group routes
+scope constraints: { format: nil } do
+  get 'v2', to: proc { [200, {}, ['']] }
+  get 'v2/*group_id/dependency_proxy/containers/:image/manifests/*tag' => 'groups/dependency_proxy_for_containers#manifest'
+  get 'v2/*group_id/dependency_proxy/containers/:image/blobs/:sha' => 'groups/dependency_proxy_for_containers#blob'
+end

ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Groups::DependencyProxyForContainersController do
+  let(:group) { create(:group) }
+  let(:token_response) { { status: :success, token: 'abcd1234' } }
+
+  before do
+    allow(Gitlab.config.dependency_proxy)
+      .to receive(:enabled).and_return(true)
+
+    allow_any_instance_of(DependencyProxy::RequestTokenService)
+      .to receive(:execute).and_return(token_response)
+  end
+
+  describe 'GET #manifest' do
+    let(:manifest) { { foo: 'bar' }.to_json }
+    let(:pull_response) { { status: :success, manifest: manifest } }
+
+    before do
+      allow_any_instance_of(DependencyProxy::PullManifestService)
+        .to receive(:execute).and_return(pull_response)
+    end
+
+    context 'feature enabled' do
+      before do
+        enable_dependency_proxy
+      end
+
+      context 'remote token request fails' do
+        let(:token_response) do
+          {
+            status: :error,
+            http_status: 503,
+            message: 'Service Unavailable'
+          }
+        end
+
+        it 'proxies status from the remote token request' do
+          get_manifest
+
+          expect(response).to have_gitlab_http_status(503)
+          expect(response.body).to eq('Service Unavailable')
+        end
+      end
+
+      context 'remote manifest request fails' do
+        let(:pull_response) do
+          {
+            status: :error,
+            http_status: 400,
+            message: ''
+          }
+        end
+
+        it 'proxies status from the remote manifest request' do
+          get_manifest
+
+          expect(response).to have_gitlab_http_status(400)
+          expect(response.body).to be_empty
+        end
+      end
+
+      it 'returns 200 with manifest file' do
+        get_manifest
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response.body).to eq(manifest)
+      end
+    end
+
+    it 'returns 404 when feature is disabled' do
+      get_manifest
+
+      expect(response).to have_gitlab_http_status(404)
+    end
+
+    def get_manifest
+      get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: '3.9.2' }
+    end
+  end
+
+  describe 'GET #blob' do
+    let(:blob) { create(:dependency_proxy_blob) }
+    let(:blob_sha) { blob.file_name.sub('.gz', '') }
+    let(:blob_response) { { status: :success, blob: blob } }
+
+    before do
+      allow_any_instance_of(DependencyProxy::FindOrCreateBlobService)
+        .to receive(:execute).and_return(blob_response)
+    end
+
+    context 'feature enabled' do
+      before do
+        enable_dependency_proxy
+      end
+
+      context 'remote blob request fails' do
+        let(:blob_response) do
+          {
+            status: :error,
+            http_status: 400,
+            message: ''
+          }
+        end
+
+        it 'proxies status from the remote blob request' do
+          get_blob
+
+          expect(response).to have_gitlab_http_status(400)
+          expect(response.body).to be_empty
+        end
+      end
+
+      it 'sends a file' do
+        expect(controller).to receive(:send_file).with(blob.file.path, {})
+
+        get_blob
+      end
+
+      it 'returns Content-Disposition: attachment' do
+        get_blob
+
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response.headers['Content-Disposition']).to match(/^attachment/)
+      end
+    end
+
+    it 'returns 404 when feature is disabled' do
+      get_blob
+
+      expect(response).to have_gitlab_http_status(404)
+    end
+
+    def get_blob
+      get :blob, params: { group_id: group.to_param, image: 'alpine', sha: blob_sha }
+    end
+  end
+
+  def enable_dependency_proxy
+    stub_licensed_features(dependency_proxy: true)
+    group.create_dependency_proxy_setting!(enabled: true)
+  end
+end

ee/spec/routing/group_routing_spec.rb

@@ -87,4 +87,20 @@ describe 'Group routing', "routing" do
       expect(get('/groups/gitlabhq/-/security/vulnerabilities/history')).to route_to('groups/security/vulnerabilities#history', group_id: 'gitlabhq')
     end
   end
+
+  describe 'dependency proxy for containers' do
+    before do
+      allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true)
+    end
+
+    it 'routes to #manifest' do
+      expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/manifests/2.3.6'))
+        .to route_to('groups/dependency_proxy_for_containers#manifest', group_id: 'gitlabhq', image: 'ruby', tag: '2.3.6')
+    end
+
+    it 'routes to #blob' do
+      expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/blobs/abc12345'))
+        .to route_to('groups/dependency_proxy_for_containers#blob', group_id: 'gitlabhq', image: 'ruby', sha: 'abc12345')
+    end
+  end
 end

ee/spec/services/dependency_proxy/download_blob_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::DownloadBlobService do
+  include EE::DependencyProxyHelpers
+
+  let(:image) { 'alpine' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:blob_sha) { Digest::SHA256.hexdigest('ruby:2.3.9') }
+
+  subject { described_class.new(image, blob_sha, token).execute }
+
+  context 'remote request is successful' do
+    before do
+      stub_blob_download(image, blob_sha)
+    end
+
+    it { expect(subject[:status]).to eq(:success) }
+    it { expect(subject[:file]).to be_a(Tempfile) }
+    it { expect(subject[:file].size).to eq(6) }
+  end
+
+  context 'remote request is not found' do
+    before do
+      stub_blob_download(image, blob_sha, 404)
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(404) }
+    it { expect(subject[:message]).to eq('Non-success response code on downloading blob fragment') }
+  end
+
+  context 'net timeout exception' do
+    before do
+      blob_url = DependencyProxy::Registry.blob_url(image, blob_sha)
+
+      stub_request(:get, blob_url).to_timeout
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(599) }
+    it { expect(subject[:message]).to eq('execution expired') }
+  end
+end

ee/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::FindOrCreateBlobService do
+  include EE::DependencyProxyHelpers
+
+  let(:blob)  { create(:dependency_proxy_blob) }
+  let(:group) { blob.group }
+  let(:image) { 'alpine' }
+  let(:tag)   { '3.9' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:blob_sha) { '40bd001563085fc35165329ea1ff5c5ecbdbbeef' }
+
+  subject { described_class.new(group, image, token, blob_sha).execute }
+
+  before do
+    stub_registry_auth(image, token)
+  end
+
+  context 'no cache' do
+    before do
+      stub_blob_download(image, blob_sha)
+    end
+
+    it 'downloads blob from remote registry if there is no cached one' do
+      expect(subject[:status]).to eq(:success)
+      expect(subject[:blob]).to be_a(DependencyProxy::Blob)
+      expect(subject[:blob]).to be_persisted
+    end
+  end
+
+  context 'cached blob' do
+    let(:blob_sha) { blob.file_name.sub('.gz', '') }
+
+    it 'uses cached blob instead of downloading one' do
+      expect(subject[:status]).to eq(:success)
+      expect(subject[:blob]).to be_a(DependencyProxy::Blob)
+      expect(subject[:blob]).to eq(blob)
+    end
+  end
+
+  context 'no such blob exists remotely' do
+    before do
+      stub_blob_download(image, blob_sha, 404)
+    end
+
+    it 'returns error message and http status' do
+      expect(subject[:status]).to eq(:error)
+      expect(subject[:message]).to eq('Failed to download the blob')
+      expect(subject[:http_status]).to eq(404)
+    end
+  end
+end

ee/spec/services/dependency_proxy/pull_manifest_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::PullManifestService do
+  include EE::DependencyProxyHelpers
+
+  let(:image) { 'alpine' }
+  let(:tag) { '3.9' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+  let(:manifest) { { foo: 'bar' }.to_json }
+
+  subject { described_class.new(image, tag, token).execute }
+
+  context 'remote request is successful' do
+    before do
+      stub_manifest_download(image, tag)
+    end
+
+    it { expect(subject[:status]).to eq(:success) }
+    it { expect(subject[:manifest]).to eq(manifest) }
+  end
+
+  context 'remote request is not found' do
+    before do
+      stub_manifest_download(image, tag, 404, 'Not found')
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(404) }
+    it { expect(subject[:message]).to eq('Not found') }
+  end
+
+  context 'net timeout exception' do
+    before do
+      manifest_link = DependencyProxy::Registry.manifest_url(image, tag)
+
+      stub_request(:get, manifest_link).to_timeout
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(599) }
+    it { expect(subject[:message]).to eq('execution expired') }
+  end
+end

ee/spec/services/dependency_proxy/request_token_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe DependencyProxy::RequestTokenService do
+  include EE::DependencyProxyHelpers
+
+  let(:image) { 'alpine:3.9' }
+  let(:token) { Digest::SHA256.hexdigest('123') }
+
+  subject { described_class.new(image).execute }
+
+  context 'remote request is successful' do
+    before do
+      stub_registry_auth(image, token)
+    end
+
+    it { expect(subject[:status]).to eq(:success) }
+    it { expect(subject[:token]).to eq(token) }
+  end
+
+  context 'remote request is not found' do
+    before do
+      stub_registry_auth(image, token, 404)
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(404) }
+    it { expect(subject[:message]).to eq('Expected 200 response code for an access token') }
+  end
+
+  context 'failed to parse response body' do
+    before do
+      stub_registry_auth(image, token, 200, 'dasd1321: wow')
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(500) }
+    it { expect(subject[:message]).to eq('Failed to parse a response body for an access token') }
+  end
+
+  context 'net timeout exception' do
+    before do
+      auth_link = DependencyProxy::Registry.auth_url(image)
+
+      stub_request(:any, auth_link).to_timeout
+    end
+
+    it { expect(subject[:status]).to eq(:error) }
+    it { expect(subject[:http_status]).to eq(599) }
+    it { expect(subject[:message]).to eq('execution expired') }
+  end
+end

ee/spec/support/helpers/ee/dependency_proxy_helpers.rb

+# frozen_string_literal: true
+
+module EE
+  module DependencyProxyHelpers
+    def stub_registry_auth(image, token, status = 200, body = nil)
+      auth_body = { 'token' => token }.to_json
+      auth_link = registry.auth_url(image)
+
+      stub_request(:get, auth_link)
+        .to_return(status: status, body: body || auth_body)
+    end
+
+    def stub_manifest_download(image, tag, status = 200, body = nil)
+      manifest_url = registry.manifest_url(image, tag)
+
+      stub_request(:get, manifest_url)
+        .to_return(status: status, body: body || manifest)
+    end
+
+    def stub_blob_download(image, blob_sha, status = 200, body = '123456')
+      download_link = registry.blob_url(image, blob_sha)
+
+      stub_request(:get, download_link)
+        .to_return(status: status, body: body)
+    end
+
+    private
+
+    def registry
+      @registry ||= DependencyProxy::Registry
+    end
+  end
+end