Pypi skeleton code

- Supports token authentication
- Supports `sample-project` package instalation
- Base API skeleton

ee/app/models/packages/package.rb

@@ -30,7 +30,7 @@ class Packages::Package < ApplicationRecord
   validate :valid_npm_package_name, if: :npm?
   validate :package_already_taken, if: :npm?
 
-  enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4 }
+  enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5 }
 
   scope :with_name, ->(name) { where(name: name) }
   scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) }

ee/lib/api/helpers/packages/basic_auth_helpers.rb

+# frozen_string_literal: true
+
+module API
+  module Helpers
+    module Packages
+      module BasicAuthHelpers
+        module Constants
+          AUTHENTICATE_REALM_HEADER = 'Www-Authenticate: Basic realm'
+          AUTHENTICATE_REALM_NAME = 'GitLab Packages Registry'
+        end
+
+        include Constants
+
+        def find_personal_access_token
+          find_personal_access_token_from_http_basic_auth
+        end
+
+        def authorized_user_project
+          @authorized_user_project ||= authorized_project_find!(params[:id])
+        end
+
+        def authorized_project_find!(id)
+          project = find_project(id)
+
+          unless project && can?(current_user, :read_project, project)
+            return unauthorized_or! { not_found! }
+          end
+
+          project
+        end
+
+        def authorize!(action, subject = :global, reason = nil)
+          return if can?(current_user, action, subject)
+
+          unauthorized_or! { forbidden!(reason) }
+        end
+
+        def unauthorized_or!
+          current_user ? yield : unauthorized_with_header!
+        end
+
+        def unauthorized_with_header!
+          header(AUTHENTICATE_REALM_HEADER, AUTHENTICATE_REALM_NAME)
+          unauthorized!
+        end
+      end
+    end
+  end
+end

ee/lib/api/pypi_packages.rb

+# frozen_string_literal: true
+
+# PyPI Package Manager Client API
+#
+# These API endpoints are not meant to be consumed directly by users. They are
+# called by the PyPI package manager client when users run commands
+# like `pip install` or `twine upload`.
+module API
+  class PypiPackages < Grape::API
+    helpers ::API::Helpers::PackagesManagerClientsHelpers
+    helpers ::API::Helpers::RelatedResourcesHelpers
+    helpers ::API::Helpers::Packages::BasicAuthHelpers
+    include ::API::Helpers::Packages::BasicAuthHelpers::Constants
+
+    default_format :json
+
+    rescue_from ArgumentError do |e|
+      render_api_error!(e.message, 400)
+    end
+
+    before do
+      require_packages_enabled!
+    end
+
+    params do
+      requires :id, type: Integer, desc: 'The ID of a project'
+    end
+
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      before do
+        unless ::Feature.enabled?(:pypi_packages, authorized_user_project)
+          not_found!
+        end
+
+        authorize_packages_feature!(authorized_user_project)
+      end
+
+      namespace ':id/packages/pypi' do
+        desc 'The PyPi package download endpoint' do
+          detail 'This feature was introduced in GitLab 12.10'
+        end
+
+        params do
+          requires :file_identifier, type: String, desc: 'The PyPi package file identifier', regexp: API::NO_SLASH_URL_PART_REGEX
+        end
+
+        get 'files/*file_identifier', :txt do
+          authorize_read_package!(authorized_user_project)
+        end
+
+        desc 'The PyPi Simple Endpoint' do
+          detail 'This feature was introduced in GitLab 12.10'
+        end
+
+        params do
+          requires :package_name, type: String, desc: 'The PyPi package name', regexp: API::NO_SLASH_URL_PART_REGEX
+        end
+
+        get 'simple/*package_name', format: :txt do
+          authorize_read_package!(authorized_user_project)
+        end
+
+        desc 'The PyPi Package upload endpoint' do
+          detail 'This feature was introduced in GitLab 12.10'
+        end
+
+        params do
+          use :workhorse_upload_params
+        end
+
+        post do
+          authorize_upload!(authorized_user_project)
+
+          created!
+        rescue ObjectStorage::RemoteStoreError => e
+          Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:name], project_id: authorized_user_project.id })
+
+          forbidden!
+        end
+
+        post 'authorize' do
+          authorize_workhorse!(subject: authorized_user_project, has_length: false)
+        end
+      end
+    end
+  end
+end

ee/lib/ee/api/api.rb

@@ -32,6 +32,7 @@ module EE
         mount ::API::ProjectMirror
         mount ::API::ProjectPushRule
         mount ::API::NugetPackages
+        mount ::API::PypiPackages
         mount ::API::ConanPackages
         mount ::API::MavenPackages
         mount ::API::NpmPackages

ee/spec/factories/packages.rb

@@ -49,6 +49,12 @@ FactoryBot.define do
       end
     end
 
+    factory :pypi_package do
+      sequence(:name) { |n| "pypi-package-#{n}"}
+      sequence(:version) { |n| "1.0.#{n}" }
+      package_type { :pypi }
+    end
+
     factory :conan_package do
       conan_metadatum
 

ee/spec/requests/api/pypi_packages_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe API::PypiPackages do
+  include WorkhorseHelpers
+  include EE::PackagesManagerApiSpecHelpers
+
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project, reload: true) { create(:project, :public) }
+  let_it_be(:personal_access_token) { create(:personal_access_token, user: user) }
+
+  describe 'GET /api/v4/projects/:id/packages/pypi/simple/:package_name' do
+    let(:url) { "/projects/#{project.id}/packages/pypi/simple/sample-project" }
+
+    subject { get api(url) }
+
+    context 'with packages features enabled' do
+      before do
+        stub_licensed_features(packages: true)
+      end
+
+      context 'with valid project' do
+        using RSpec::Parameterized::TableSyntax
+
+        where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
+          'PUBLIC'  | :developer  | true  | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | true  | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :developer  | true  | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | true  | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :developer  | false | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | false | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :developer  | false | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | false | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :anonymous  | false | true  | 'process PyPi api request' | :success
+          'PRIVATE' | :developer  | true  | true  | 'process PyPi api request' | :success
+          'PRIVATE' | :guest      | true  | true  | 'process PyPi api request' | :forbidden
+          'PRIVATE' | :developer  | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :developer  | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :guest      | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :developer  | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :anonymous  | false | true  | 'process PyPi api request' | :unauthorized
+        end
+
+        with_them do
+          let(:token) { user_token ? personal_access_token.token : 'wrong' }
+          let(:headers) { user_role == :anonymous ? {} : build_basic_auth_header(user.username, token) }
+
+          subject { get api(url), headers: headers }
+
+          before do
+            project.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
+          end
+
+          it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+        end
+      end
+
+      it_behaves_like 'rejects PyPI access with unknown project id'
+    end
+
+    it_behaves_like 'rejects PyPI packages access with packages features disabled'
+  end
+
+  describe 'POST /api/v4/projects/:id/packages/pypi/authorize' do
+    let_it_be(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') }
+    let_it_be(:workhorse_header) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } }
+    let(:url) { "/projects/#{project.id}/packages/pypi/authorize" }
+    let(:headers) { {} }
+
+    subject { post api(url), headers: headers }
+
+    context 'with packages features enabled' do
+      before do
+        stub_licensed_features(packages: true)
+      end
+
+      context 'with valid project' do
+        using RSpec::Parameterized::TableSyntax
+
+        where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
+          'PUBLIC'  | :developer  | true  | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | true  | true  | 'process PyPi api request' | :forbidden
+          'PUBLIC'  | :developer  | true  | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :guest      | true  | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :developer  | false | true  | 'process PyPi api request' | :forbidden
+          'PUBLIC'  | :guest      | false | true  | 'process PyPi api request' | :forbidden
+          'PUBLIC'  | :developer  | false | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :guest      | false | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :anonymous  | false | true  | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :developer  | true  | true  | 'process PyPi api request' | :success
+          'PRIVATE' | :guest      | true  | true  | 'process PyPi api request' | :forbidden
+          'PRIVATE' | :developer  | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :developer  | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :guest      | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :developer  | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :anonymous  | false | true  | 'process PyPi api request' | :unauthorized
+        end
+
+        with_them do
+          let(:token) { user_token ? personal_access_token.token : 'wrong' }
+          let(:user_headers) { user_role == :anonymous ? {} : build_basic_auth_header(user.username, token) }
+          let(:headers) { user_headers.merge(workhorse_header) }
+
+          before do
+            project.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
+          end
+
+          it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+        end
+      end
+
+      it_behaves_like 'rejects PyPI access with unknown project id'
+    end
+
+    it_behaves_like 'rejects PyPI packages access with packages features disabled'
+  end
+
+  describe 'POST /api/v4/projects/:id/packages/pypi' do
+    let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') }
+    let(:workhorse_header) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } }
+    let_it_be(:file_name) { 'package.whl' }
+    let(:url) { "/projects/#{project.id}/packages/pypi" }
+    let(:headers) { {} }
+    let(:params) { { content: temp_file(file_name) } }
+
+    subject do
+      workhorse_finalize(
+        api(url),
+        method: :post,
+        file_key: :content,
+        params: params,
+        headers: headers
+      )
+    end
+
+    context 'with packages features enabled' do
+      before do
+        stub_licensed_features(packages: true)
+      end
+
+      context 'with valid project' do
+        using RSpec::Parameterized::TableSyntax
+
+        where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
+          'PUBLIC'  | :developer  | true  | true  | 'process PyPi api request' | :created
+          'PUBLIC'  | :guest      | true  | true  | 'process PyPi api request' | :forbidden
+          'PUBLIC'  | :developer  | true  | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :guest      | true  | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :developer  | false | true  | 'process PyPi api request' | :forbidden
+          'PUBLIC'  | :guest      | false | true  | 'process PyPi api request' | :forbidden
+          'PUBLIC'  | :developer  | false | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :guest      | false | false | 'process PyPi api request' | :unauthorized
+          'PUBLIC'  | :anonymous  | false | true  | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :developer  | true  | true  | 'process PyPi api request' | :created
+          'PRIVATE' | :guest      | true  | true  | 'process PyPi api request' | :forbidden
+          'PRIVATE' | :developer  | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :developer  | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :guest      | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :developer  | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :anonymous  | false | true  | 'process PyPi api request' | :unauthorized
+        end
+
+        with_them do
+          let(:token) { user_token ? personal_access_token.token : 'wrong' }
+          let(:user_headers) { user_role == :anonymous ? {} : build_basic_auth_header(user.username, token) }
+          let(:headers) { user_headers.merge(workhorse_header) }
+
+          before do
+            project.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
+          end
+
+          it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+        end
+      end
+
+      it_behaves_like 'rejects PyPI access with unknown project id'
+    end
+
+    it_behaves_like 'rejects PyPI packages access with packages features disabled'
+  end
+
+  describe 'GET /api/v4/projects/:id/packages/pypi/files/*file_identifier' do
+    let_it_be(:package_name) { 'Dummy-Package' }
+    let_it_be(:package) { create(:pypi_package, project: project, name: package_name) }
+
+    let(:url) { "/projects/#{project.id}/packages/pypi/files/sample_project-1.0.0-py3-none-any.whl" }
+
+    subject { get api(url) }
+
+    context 'with packages features enabled' do
+      before do
+        stub_licensed_features(packages: true)
+      end
+
+      context 'with valid project' do
+        using RSpec::Parameterized::TableSyntax
+
+        where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
+          'PUBLIC'  | :developer  | true  | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | true  | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :developer  | true  | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | true  | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :developer  | false | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | false | true  | 'process PyPi api request' | :success
+          'PUBLIC'  | :developer  | false | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :guest      | false | false | 'process PyPi api request' | :success
+          'PUBLIC'  | :anonymous  | false | true  | 'process PyPi api request' | :success
+          'PRIVATE' | :developer  | true  | true  | 'process PyPi api request' | :success
+          'PRIVATE' | :guest      | true  | true  | 'process PyPi api request' | :forbidden
+          'PRIVATE' | :developer  | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | true  | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :developer  | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :guest      | false | true  | 'process PyPi api request' | :not_found
+          'PRIVATE' | :developer  | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :guest      | false | false | 'process PyPi api request' | :unauthorized
+          'PRIVATE' | :anonymous  | false | true  | 'process PyPi api request' | :unauthorized
+        end
+
+        with_them do
+          let(:token) { user_token ? personal_access_token.token : 'wrong' }
+          let(:headers) { user_role == :anonymous ? {} : build_basic_auth_header(user.username, token) }
+
+          subject { get api(url), headers: headers }
+
+          before do
+            project.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
+          end
+
+          it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+        end
+      end
+
+      it_behaves_like 'rejects PyPI access with unknown project id'
+    end
+
+    it_behaves_like 'rejects PyPI packages access with packages features disabled'
+  end
+end

ee/spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'process PyPi api request' do |user_type, status, add_member = true|
+  context "for user type #{user_type}" do
+    before do
+      project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    end
+
+    it_behaves_like 'returning response status', status
+  end
+end
+
+RSpec.shared_examples 'rejects PyPI access with unknown project id' do
+  context 'with an unknown project' do
+    let(:project) { OpenStruct.new(id: 1234567890) }
+
+    context 'as anonymous' do
+      it_behaves_like 'process PyPi api request', :anonymous, :unauthorized
+    end
+
+    context 'as authenticated user' do
+      subject { get api(url), headers: build_basic_auth_header(user.username, personal_access_token.token) }
+
+      it_behaves_like 'process PyPi api request', :anonymous, :not_found
+    end
+  end
+end
+
+RSpec.shared_examples 'rejects PyPI packages access with packages features disabled' do
+  context 'with packages features disabled' do
+    before do
+      stub_licensed_features(packages: false)
+    end
+
+    it_behaves_like 'process PyPi api request', :anonymous, :forbidden
+  end
+end