Commit a6309aba authored by James Lopez's avatar James Lopez

Merge branch 'debian_packages' into 'master'

Add Debian API skeleton

See merge request gitlab-org/gitlab!42670
parents 03e62d80 74b20d3b
---
title: Add Debian API skeleton
merge_request: 42670
author: Mathieu Parent
type: added
---
name: debian_packages
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42670
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/5835
group: group::package
type: development
default_enabled: false
# frozen_string_literal: true
class AddDebianMaxFileSizeToPlanLimits < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
add_column :plan_limits, :debian_max_file_size, :bigint, default: 3.gigabytes, null: false
end
end
18a3981a3becefe6700dd5fea87e8ba9478c0e83ddc80de1b3ee2ed77c221ce6
\ No newline at end of file
......@@ -14365,7 +14365,8 @@ CREATE TABLE plan_limits (
nuget_max_file_size bigint DEFAULT 524288000 NOT NULL,
pypi_max_file_size bigint DEFAULT '3221225472'::bigint NOT NULL,
generic_packages_max_file_size bigint DEFAULT '5368709120'::bigint NOT NULL,
golang_max_file_size bigint DEFAULT 104857600 NOT NULL
golang_max_file_size bigint DEFAULT 104857600 NOT NULL,
debian_max_file_size bigint DEFAULT '3221225472'::bigint NOT NULL
);
CREATE SEQUENCE plan_limits_id_seq
......
......@@ -552,6 +552,9 @@ Plan.default.actual_limits.update!(maven_max_file_size: 100.megabytes)
# For PyPI Packages
Plan.default.actual_limits.update!(pypi_max_file_size: 100.megabytes)
# For Debian Packages
Plan.default.actual_limits.update!(debian_max_file_size: 100.megabytes)
```
Set the limit to `0` to allow any file size.
......@@ -196,6 +196,8 @@ module API
mount ::API::ComposerPackages
mount ::API::ConanProjectPackages
mount ::API::ConanInstancePackages
mount ::API::DebianGroupPackages
mount ::API::DebianProjectPackages
mount ::API::MavenPackages
mount ::API::NpmPackages
mount ::API::GenericPackages
......
# frozen_string_literal: true
module API
class DebianGroupPackages < Grape::API::Instance
params do
requires :id, type: String, desc: 'The ID of a group'
end
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
before do
not_found! unless ::Feature.enabled?(:debian_packages, user_group)
authorize_read_package!(user_group)
end
namespace ':id/-/packages/debian' do
include DebianPackageEndpoints
end
end
end
end
# frozen_string_literal: true
module API
module DebianPackageEndpoints
extend ActiveSupport::Concern
DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze
COMPONENT_REGEX = %r{[a-z-]+}.freeze
ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze
LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze
PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX
DISTRIBUTION_REQUIREMENTS = {
distribution: DISTRIBUTION_REGEX
}.freeze
COMPONENT_ARCHITECTURE_REQUIREMENTS = {
component: COMPONENT_REGEX,
architecture: ARCHITECTURE_REGEX
}.freeze
COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = {
component: COMPONENT_REGEX,
letter: LETTER_REGEX,
source_package: PACKAGE_REGEX
}.freeze
FILE_NAME_REQUIREMENTS = {
file_name: API::NO_SLASH_URL_PART_REGEX
}.freeze
included do
helpers ::API::Helpers::PackagesHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
format :txt
rescue_from ArgumentError do |e|
render_api_error!(e.message, 400)
end
rescue_from ActiveRecord::RecordInvalid do |e|
render_api_error!(e.message, 400)
end
before do
require_packages_enabled!
end
params do
requires :distribution, type: String, desc: 'The Debian Codename', file_path: true
end
namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
# GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg
desc 'The Release file signature' do
detail 'This feature was introduced in GitLab 13.5'
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'Release.gpg' do
not_found!
end
# GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release
desc 'The unsigned Release file' do
detail 'This feature was introduced in GitLab 13.5'
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'Release' do
# https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
'TODO Release'
end
# GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease
desc 'The signed Release file' do
detail 'This feature was introduced in GitLab 13.5'
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'InRelease' do
not_found!
end
params do
requires :component, type: String, desc: 'The Debian Component'
requires :architecture, type: String, desc: 'The Debian Architecture'
end
namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
# GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
desc 'The binary files index' do
detail 'This feature was introduced in GitLab 13.5'
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get 'Packages' do
# https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
'TODO Packages'
end
end
end
params do
requires :component, type: String, desc: 'The Debian Component'
requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
requires :source_package, type: String, desc: 'The Debian Source Package Name'
end
namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do
# GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name
params do
requires :file_name, type: String, desc: 'The Debian File Name'
end
desc 'The package' do
detail 'This feature was introduced in GitLab 13.5'
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get ':file_name', requirements: FILE_NAME_REQUIREMENTS do
# https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
'TODO File'
end
end
end
end
end
# frozen_string_literal: true
module API
class DebianProjectPackages < Grape::API::Instance
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
before do
not_found! unless ::Feature.enabled?(:debian_packages, user_project)
authorize_read_package!
end
namespace ':id/-/packages/debian' do
include DebianPackageEndpoints
params do
requires :file_name, type: String, desc: 'The file name'
end
namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do
# PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name
params do
requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
end
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
put do
authorize_upload!(authorized_user_project)
bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size)
package_event('push_package')
created!
rescue ObjectStorage::RemoteStoreError => e
Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id })
forbidden!
end
# PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize
route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
post 'authorize' do
authorize_workhorse!(
subject: authorized_user_project,
has_length: false,
maximum_size: authorized_user_project.actual_limits.debian_max_file_size
)
end
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::DebianGroupPackages do
include HttpBasicAuthHelpers
include WorkhorseHelpers
include_context 'Debian repository shared context', :group do
describe 'GET groups/:id/-/packages/debian/dists/*distribution/Release.gpg' do
let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/Release.gpg" }
it_behaves_like 'Debian group repository GET endpoint', :not_found, nil
end
describe 'GET groups/:id/-/packages/debian/dists/*distribution/Release' do
let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/Release" }
it_behaves_like 'Debian group repository GET endpoint', :success, 'TODO Release'
end
describe 'GET groups/:id/-/packages/debian/dists/*distribution/InRelease' do
let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/InRelease" }
it_behaves_like 'Debian group repository GET endpoint', :not_found, nil
end
describe 'GET groups/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages' do
let(:url) { "/groups/#{group.id}/-/packages/debian/dists/#{distribution}/#{component}/binary-#{architecture}/Packages" }
it_behaves_like 'Debian group repository GET endpoint', :success, 'TODO Packages'
end
describe 'GET groups/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name' do
let(:url) { "/groups/#{group.id}/-/packages/debian/pool/#{component}/#{letter}/#{source_package}/#{package_name}_#{package_version}_#{architecture}.deb" }
it_behaves_like 'Debian group repository GET endpoint', :success, 'TODO File'
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::DebianProjectPackages do
include HttpBasicAuthHelpers
include WorkhorseHelpers
include_context 'Debian repository shared context', :project do
describe 'GET projects/:id/-/packages/debian/dists/*distribution/Release.gpg' do
let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/Release.gpg" }
it_behaves_like 'Debian project repository GET endpoint', :not_found, nil
end
describe 'GET projects/:id/-/packages/debian/dists/*distribution/Release' do
let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/Release" }
it_behaves_like 'Debian project repository GET endpoint', :success, 'TODO Release'
end
describe 'GET projects/:id/-/packages/debian/dists/*distribution/InRelease' do
let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/InRelease" }
it_behaves_like 'Debian project repository GET endpoint', :not_found, nil
end
describe 'GET projects/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages' do
let(:url) { "/projects/#{project.id}/-/packages/debian/dists/#{distribution}/#{component}/binary-#{architecture}/Packages" }
it_behaves_like 'Debian project repository GET endpoint', :success, 'TODO Packages'
end
describe 'GET projects/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name' do
let(:url) { "/projects/#{project.id}/-/packages/debian/pool/#{component}/#{letter}/#{source_package}/#{package_name}_#{package_version}_#{architecture}.deb" }
it_behaves_like 'Debian project repository GET endpoint', :success, 'TODO File'
end
describe 'PUT projects/:id/-/packages/debian/incoming/:file_name' do
let(:method) { :put }
let(:url) { "/projects/#{project.id}/-/packages/debian/incoming/#{file_name}" }
it_behaves_like 'Debian project repository PUT endpoint', :created, nil
end
end
end
# frozen_string_literal: true
RSpec.shared_context 'Debian repository shared context' do |object_type|
before do
stub_feature_flags(debian_packages: true)
end
if object_type == :project
let(:project) { create(:project, :public) }
elsif object_type == :group
let(:group) { create(:group, :public) }
end
let(:user) { create(:user) }
let(:personal_access_token) { create(:personal_access_token, user: user) }
let(:distribution) { 'bullseye' }
let(:component) { 'main' }
let(:architecture) { 'amd64' }
let(:source_package) { 'sample' }
let(:letter) { source_package[0..2] == 'lib' ? source_package[0..3] : source_package[0] }
let(:package_name) { 'libsample0' }
let(:package_version) { '1.2.3~alpha2-1' }
let(:file_name) { "#{package_name}_#{package_version}_#{architecture}.deb" }
let(:method) { :get }
let(:workhorse_params) do
if method == :put
file_upload = fixture_file_upload("spec/fixtures/packages/debian/#{file_name}")
{ file: file_upload }
else
{}
end
end
let(:params) { workhorse_params }
let(:auth_headers) { {} }
let(:workhorse_headers) do
if method == :put
workhorse_token = JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256')
{ 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token }
else
{}
end
end
let(:headers) { auth_headers.merge(workhorse_headers) }
let(:send_rewritten_field) { true }
subject do
if method == :put
workhorse_finalize(
api(url),
method: method,
file_key: :file,
params: params,
headers: headers,
send_rewritten_field: send_rewritten_field
)
else
send method, api(url), headers: headers, params: params
end
end
end
RSpec.shared_context 'Debian repository auth headers' do |user_role, user_token, auth_method = :token|
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:auth_headers) do
if user_role == :anonymous
{}
elsif auth_method == :token
{ 'Private-Token' => token }
else
basic_auth_header(user.username, token)
end
end
end
RSpec.shared_context 'Debian repository project access' do |project_visibility_level, user_role, user_token, auth_method|
include_context 'Debian repository auth headers', user_role, user_token, auth_method do
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
end
end
end
RSpec.shared_examples 'Debian project repository GET request' do |user_role, add_member, status, body|
context "for user type #{user_role}" do
before do
project.send("add_#{user_role}", user) if add_member && user_role != :anonymous
end
and_body = body.nil? ? '' : ' and expected body'
it "returns #{status}#{and_body}" do
subject
expect(response).to have_gitlab_http_status(status)
unless body.nil?
expect(response.body).to eq(body)
end
end
end
end
RSpec.shared_examples 'Debian project repository PUT request' do |user_role, add_member, status, body|
context "for user type #{user_role}" do
before do
project.send("add_#{user_role}", user) if add_member && user_role != :anonymous
end
and_body = body.nil? ? '' : ' and expected body'
if status == :created
it 'creates package files' do
pending "Debian package creation not implemented"
expect { subject }
.to change { project.packages.debian.count }.by(1)
expect(response).to have_gitlab_http_status(status)
unless body.nil?
expect(response.body).to eq(body)
end
end
it_behaves_like 'a gitlab tracking event', described_class.name, 'push_package'
else
it "returns #{status}#{and_body}" do
subject
expect(response).to have_gitlab_http_status(status)
unless body.nil?
expect(response.body).to eq(body)
end
end
end
end
end
RSpec.shared_examples 'rejects Debian access with unknown project id' do
context 'with an unknown project' do
let(:project) { double(id: non_existing_record_id) }
context 'as anonymous' do
it_behaves_like 'Debian project repository GET request', :anonymous, true, :not_found, nil
end
context 'as authenticated user' do
subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
it_behaves_like 'Debian project repository GET request', :anonymous, true, :not_found, nil
end
end
end
RSpec.shared_examples 'Debian project repository GET endpoint' do |success_status, success_body|
context 'with valid project' do
using RSpec::Parameterized::TableSyntax
where(:project_visibility_level, :user_role, :member, :user_token, :expected_status, :expected_body) do
'PUBLIC' | :developer | true | true | success_status | success_body
'PUBLIC' | :guest | true | true | success_status | success_body
'PUBLIC' | :developer | true | false | success_status | success_body
'PUBLIC' | :guest | true | false | success_status | success_body
'PUBLIC' | :developer | false | true | success_status | success_body
'PUBLIC' | :guest | false | true | success_status | success_body
'PUBLIC' | :developer | false | false | success_status | success_body
'PUBLIC' | :guest | false | false | success_status | success_body
'PUBLIC' | :anonymous | false | true | success_status | success_body
'PRIVATE' | :developer | true | true | success_status | success_body
'PRIVATE' | :guest | true | true | :forbidden | nil
'PRIVATE' | :developer | true | false | :not_found | nil
'PRIVATE' | :guest | true | false | :not_found | nil
'PRIVATE' | :developer | false | true | :not_found | nil
'PRIVATE' | :guest | false | true | :not_found | nil
'PRIVATE' | :developer | false | false | :not_found | nil
'PRIVATE' | :guest | false | false | :not_found | nil
'PRIVATE' | :anonymous | false | true | :not_found | nil
end
with_them do
include_context 'Debian repository project access', params[:project_visibility_level], params[:user_role], params[:user_token], :basic do
it_behaves_like 'Debian project repository GET request', params[:user_role], params[:member], params[:expected_status], params[:expected_body]
end
end
end
it_behaves_like 'rejects Debian access with unknown project id'
end
RSpec.shared_examples 'Debian project repository PUT endpoint' do |success_status, success_body|
context 'with valid project' do
using RSpec::Parameterized::TableSyntax
where(:project_visibility_level, :user_role, :member, :user_token, :expected_status, :expected_body) do
'PUBLIC' | :developer | true | true | success_status | nil
'PUBLIC' | :guest | true | true | :forbidden | nil
'PUBLIC' | :developer | true | false | :unauthorized | nil
'PUBLIC' | :guest | true | false | :unauthorized | nil
'PUBLIC' | :developer | false | true | :forbidden | nil
'PUBLIC' | :guest | false | true | :forbidden | nil
'PUBLIC' | :developer | false | false | :unauthorized | nil
'PUBLIC' | :guest | false | false | :unauthorized | nil
'PUBLIC' | :anonymous | false | true | :unauthorized | nil
'PRIVATE' | :developer | true | true | success_status | nil
'PRIVATE' | :guest | true | true | :forbidden | nil
'PRIVATE' | :developer | true | false | :not_found | nil
'PRIVATE' | :guest | true | false | :not_found | nil
'PRIVATE' | :developer | false | true | :not_found | nil
'PRIVATE' | :guest | false | true | :not_found | nil
'PRIVATE' | :developer | false | false | :not_found | nil
'PRIVATE' | :guest | false | false | :not_found | nil
'PRIVATE' | :anonymous | false | true | :not_found | nil
end
with_them do
include_context 'Debian repository project access', params[:project_visibility_level], params[:user_role], params[:user_token], :basic do
it_behaves_like 'Debian project repository PUT request', params[:user_role], params[:member], params[:expected_status], params[:expected_body]
end
end
end
it_behaves_like 'rejects Debian access with unknown project id'
end
RSpec.shared_context 'Debian repository group access' do |group_visibility_level, user_role, user_token, auth_method|
include_context 'Debian repository auth headers', user_role, user_token, auth_method do
before do
group.update_column(:visibility_level, Gitlab::VisibilityLevel.const_get(group_visibility_level, false))
end
end
end
RSpec.shared_examples 'Debian group repository GET request' do |user_role, add_member, status, body|
context "for user type #{user_role}" do
before do
group.send("add_#{user_role}", user) if add_member && user_role != :anonymous
end
and_body = body.nil? ? '' : ' and expected body'
it "returns #{status}#{and_body}" do
subject
expect(response).to have_gitlab_http_status(status)
unless body.nil?
expect(response.body).to eq(body)
end
end
end
end
RSpec.shared_examples 'rejects Debian access with unknown group id' do
context 'with an unknown group' do
let(:group) { double(id: non_existing_record_id) }
context 'as anonymous' do
it_behaves_like 'Debian group repository GET request', :anonymous, true, :not_found, nil
end
context 'as authenticated user' do
subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
it_behaves_like 'Debian group repository GET request', :anonymous, true, :not_found, nil
end
end
end
RSpec.shared_examples 'Debian group repository GET endpoint' do |success_status, success_body|
context 'with valid group' do
using RSpec::Parameterized::TableSyntax
where(:group_visibility_level, :user_role, :member, :user_token, :expected_status, :expected_body) do
'PUBLIC' | :developer | true | true | success_status | success_body
'PUBLIC' | :guest | true | true | success_status | success_body
'PUBLIC' | :developer | true | false | success_status | success_body
'PUBLIC' | :guest | true | false | success_status | success_body
'PUBLIC' | :developer | false | true | success_status | success_body
'PUBLIC' | :guest | false | true | success_status | success_body
'PUBLIC' | :developer | false | false | success_status | success_body
'PUBLIC' | :guest | false | false | success_status | success_body
'PUBLIC' | :anonymous | false | true | success_status | success_body
'PRIVATE' | :developer | true | true | success_status | success_body
'PRIVATE' | :guest | true | true | :forbidden | nil
'PRIVATE' | :developer | true | false | :not_found | nil
'PRIVATE' | :guest | true | false | :not_found | nil
'PRIVATE' | :developer | false | true | :not_found | nil
'PRIVATE' | :guest | false | true | :not_found | nil
'PRIVATE' | :developer | false | false | :not_found | nil
'PRIVATE' | :guest | false | false | :not_found | nil
'PRIVATE' | :anonymous | false | true | :not_found | nil
end
with_them do
include_context 'Debian repository group access', params[:group_visibility_level], params[:user_role], params[:user_token], :basic do
it_behaves_like 'Debian group repository GET request', params[:user_role], params[:member], params[:expected_status], params[:expected_body]
end
end
end
it_behaves_like 'rejects Debian access with unknown group id'
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment