Merge branch 'debian_packages' into 'master'

Add Debian API skeleton

See merge request gitlab-org/gitlab!42670

changelogs/unreleased/debian_packages.yml

+---
+title: Add Debian API skeleton
+merge_request: 42670
+author: Mathieu Parent
+type: added

config/feature_flags/development/debian_packages.yml

+---
+name: debian_packages
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42670
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/5835
+group: group::package
+type: development
+default_enabled: false

db/migrate/20200916165232_add_debian_max_file_size_to_plan_limits.rb

+# frozen_string_literal: true
+
+class AddDebianMaxFileSizeToPlanLimits < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column :plan_limits, :debian_max_file_size, :bigint, default: 3.gigabytes, null: false
+  end
+end

db/schema_migrations/20200916165232

+18a3981a3becefe6700dd5fea87e8ba9478c0e83ddc80de1b3ee2ed77c221ce6
\ No newline at end of file

db/structure.sql

@@ -14365,7 +14365,8 @@ CREATE TABLE plan_limits (
     nuget_max_file_size bigint DEFAULT 524288000 NOT NULL,
     pypi_max_file_size bigint DEFAULT '3221225472'::bigint NOT NULL,
     generic_packages_max_file_size bigint DEFAULT '5368709120'::bigint NOT NULL,
-    golang_max_file_size bigint DEFAULT 104857600 NOT NULL
+    golang_max_file_size bigint DEFAULT 104857600 NOT NULL,
+    debian_max_file_size bigint DEFAULT '3221225472'::bigint NOT NULL
 );
 
 CREATE SEQUENCE plan_limits_id_seq

doc/administration/instance_limits.md

@@ -552,6 +552,9 @@ Plan.default.actual_limits.update!(maven_max_file_size: 100.megabytes)
 
 # For PyPI Packages
 Plan.default.actual_limits.update!(pypi_max_file_size: 100.megabytes)
+
+# For Debian Packages
+Plan.default.actual_limits.update!(debian_max_file_size: 100.megabytes)
 ```
 
 Set the limit to `0` to allow any file size.

lib/api/api.rb

@@ -196,6 +196,8 @@ module API
       mount ::API::ComposerPackages
       mount ::API::ConanProjectPackages
       mount ::API::ConanInstancePackages
+      mount ::API::DebianGroupPackages
+      mount ::API::DebianProjectPackages
       mount ::API::MavenPackages
       mount ::API::NpmPackages
       mount ::API::GenericPackages

lib/api/debian_group_packages.rb

+# frozen_string_literal: true
+
+module API
+  class DebianGroupPackages < Grape::API::Instance
+    params do
+      requires :id, type: String, desc: 'The ID of a group'
+    end
+
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      before do
+        not_found! unless ::Feature.enabled?(:debian_packages, user_group)
+
+        authorize_read_package!(user_group)
+      end
+
+      namespace ':id/-/packages/debian' do
+        include DebianPackageEndpoints
+      end
+    end
+  end
+end

lib/api/debian_package_endpoints.rb

+# frozen_string_literal: true
+
+module API
+  module DebianPackageEndpoints
+    extend ActiveSupport::Concern
+
+    DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze
+    COMPONENT_REGEX = %r{[a-z-]+}.freeze
+    ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze
+    LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze
+    PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX
+    DISTRIBUTION_REQUIREMENTS = {
+      distribution: DISTRIBUTION_REGEX
+    }.freeze
+    COMPONENT_ARCHITECTURE_REQUIREMENTS = {
+      component: COMPONENT_REGEX,
+      architecture: ARCHITECTURE_REGEX
+    }.freeze
+    COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = {
+      component: COMPONENT_REGEX,
+      letter: LETTER_REGEX,
+      source_package: PACKAGE_REGEX
+    }.freeze
+    FILE_NAME_REQUIREMENTS = {
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+
+    included do
+      helpers ::API::Helpers::PackagesHelpers
+      helpers ::API::Helpers::Packages::BasicAuthHelpers
+
+      format :txt
+
+      rescue_from ArgumentError do |e|
+        render_api_error!(e.message, 400)
+      end
+
+      rescue_from ActiveRecord::RecordInvalid do |e|
+        render_api_error!(e.message, 400)
+      end
+
+      before do
+        require_packages_enabled!
+      end
+
+      params do
+        requires :distribution, type: String, desc: 'The Debian Codename', file_path: true
+      end
+
+      namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
+        # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg
+        desc 'The Release file signature' do
+          detail 'This feature was introduced in GitLab 13.5'
+        end
+
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+        get 'Release.gpg' do
+          not_found!
+        end
+
+        # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release
+        desc 'The unsigned Release file' do
+          detail 'This feature was introduced in GitLab 13.5'
+        end
+
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+        get 'Release' do
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
+          'TODO Release'
+        end
+
+        # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease
+        desc 'The signed Release file' do
+          detail 'This feature was introduced in GitLab 13.5'
+        end
+
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+        get 'InRelease' do
+          not_found!
+        end
+
+        params do
+          requires :component, type: String, desc: 'The Debian Component'
+          requires :architecture, type: String, desc: 'The Debian Architecture'
+        end
+
+        namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
+          # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
+          desc 'The binary files index' do
+            detail 'This feature was introduced in GitLab 13.5'
+          end
+
+          route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+          get 'Packages' do
+            # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
+            'TODO Packages'
+          end
+        end
+      end
+
+      params do
+        requires :component, type: String, desc: 'The Debian Component'
+        requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
+        requires :source_package, type: String, desc: 'The Debian Source Package Name'
+      end
+
+      namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do
+        # GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name
+        params do
+          requires :file_name, type: String, desc: 'The Debian File Name'
+        end
+        desc 'The package' do
+          detail 'This feature was introduced in GitLab 13.5'
+        end
+
+        route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+        get ':file_name', requirements: FILE_NAME_REQUIREMENTS do
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
+          'TODO File'
+        end
+      end
+    end
+  end
+end

lib/api/debian_project_packages.rb

+# frozen_string_literal: true
+
+module API
+  class DebianProjectPackages < Grape::API::Instance
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      before do
+        not_found! unless ::Feature.enabled?(:debian_packages, user_project)
+
+        authorize_read_package!
+      end
+
+      namespace ':id/-/packages/debian' do
+        include DebianPackageEndpoints
+
+        params do
+          requires :file_name, type: String, desc: 'The file name'
+        end
+
+        namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do
+          # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name
+          params do
+            requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
+          end
+
+          route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+          put do
+            authorize_upload!(authorized_user_project)
+            bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size)
+
+            package_event('push_package')
+
+            created!
+          rescue ObjectStorage::RemoteStoreError => e
+            Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id })
+
+            forbidden!
+          end
+
+          # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize
+          route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+          post 'authorize' do
+            authorize_workhorse!(
+              subject: authorized_user_project,
+              has_length: false,
+              maximum_size: authorized_user_project.actual_limits.debian_max_file_size
+            )
+          end
+        end
+      end
+    end
+  end
+end

spec/fixtures/packages/debian/libsample0_1.2.3~alpha2-1_amd64.deb

+empty

spec/requests/api/debian_group_packages_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe API::DebianGroupPackages do
+  include HttpBasicAuthHelpers
+  include WorkhorseHelpers
+
+  include_context 'Debian repository shared context', :group do
+    describe 'GET groups/:id/-/packages/debian/dists/*distribution/Release.gpg' do
+      let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/Release.gpg" }
+
+      it_behaves_like 'Debian group repository GET endpoint', :not_found, nil
+    end
+
+    describe 'GET groups/:id/-/packages/debian/dists/*distribution/Release' do
+      let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/Release" }
+
+      it_behaves_like 'Debian group repository GET endpoint', :success, 'TODO Release'
+    end
+
+    describe 'GET groups/:id/-/packages/debian/dists/*distribution/InRelease' do
+      let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/InRelease" }
+
+      it_behaves_like 'Debian group repository GET endpoint', :not_found, nil
+    end
+
+    describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages' do
+      let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/#{component}/binary-#{architecture}/Packages" }
+
+      it_behaves_like 'Debian group repository GET endpoint', :success, 'TODO Packages'
+    end
+
+    describe 'GET groups/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name' do
+      let(:url) { "/groups/#{group.id}/-/packages/debian/pool/#{component}/#{letter}/#{source_package}/#{package_name}_#{package_version}_#{architecture}.deb" }
+
+      it_behaves_like 'Debian group repository GET endpoint', :success, 'TODO File'
+    end
+  end
+end

spec/requests/api/debian_project_packages_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe API::DebianProjectPackages do
+  include HttpBasicAuthHelpers
+  include WorkhorseHelpers
+
+  include_context 'Debian repository shared context', :project do
+    describe 'GET projects/:id/-/packages/debian/dists/*distribution/Release.gpg' do
+      let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/Release.gpg" }
+
+      it_behaves_like 'Debian project repository GET endpoint', :not_found, nil
+    end
+
+    describe 'GET projects/:id/-/packages/debian/dists/*distribution/Release' do
+      let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/Release" }
+
+      it_behaves_like 'Debian project repository GET endpoint', :success, 'TODO Release'
+    end
+
+    describe 'GET projects/:id/-/packages/debian/dists/*distribution/InRelease' do
+      let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/InRelease" }
+
+      it_behaves_like 'Debian project repository GET endpoint', :not_found, nil
+    end
+
+    describe 'GET projects/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages' do
+      let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/#{component}/binary-#{architecture}/Packages" }
+
+      it_behaves_like 'Debian project repository GET endpoint', :success, 'TODO Packages'
+    end
+
+    describe 'GET projects/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name' do
+      let(:url) { "/projects/#{project.id}/-/packages/debian/pool/#{component}/#{letter}/#{source_package}/#{package_name}_#{package_version}_#{architecture}.deb" }
+
+      it_behaves_like 'Debian project repository GET endpoint', :success, 'TODO File'
+    end
+
+    describe 'PUT projects/:id/-/packages/debian/incoming/:file_name' do
+      let(:method) { :put }
+      let(:url) { "/projects/#{project.id}/-/packages/debian/incoming/#{file_name}" }
+
+      it_behaves_like 'Debian project repository PUT endpoint', :created, nil
+    end
+  end
+end