Skip to content

G
gitlab-ce
Commit 7ecd9ccb authored by GitLab Bot's avatar GitLab Bot
Browse files
Options

Automatic merge of gitlab-org/gitlab master

parents 051b027c 6c7acbdb
Hide whitespace changes
Inline Side-by-side
Showing with 391 additions and 88 deletions
GITALY_SERVER_VERSION
View file @ 7ecd9ccb
747d61c17a51f361cc883c9d4700e174219088a5
8cc6bb1ffb6830fa416c8d0e32f9edebf6573730
app/models/packages/helm.rb
View file @ 7ecd9ccb
......@@ -2,6 +2,8 @@
module Packages
module Helm
TEMPORARY_PACKAGE_NAME = 'Helm.Temporary.Package'
def self.table_name_prefix
'packages_helm_'
end
......
config/metrics/counts_all/20210625095025_package_events_i_package_helm_push_package.yml 0 → 100644
View file @ 7ecd9ccb
---
key_path: counts.package_events_i_package_helm_push_package
description: The total count of Helm packages that have been published.
product_section: ops
product_stage: package
product_group: group::package
product_category: package_registry
value_type: number
status: implemented
milestone: "14.1"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64814
time_frame: all
data_source: redis
data_category: Optional
distribution:
- ce
- ee
tier:
- free
- premium
- ultimate
doc/development/usage_ping/dictionary.md
View file @ 7ecd9ccb
......@@ -4346,6 +4346,18 @@ Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `counts.package_events_i_package_helm_push_package`
The total count of Helm packages that have been published.
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210625095025_package_events_i_package_helm_push_package.yml)
Group: `group::package`
Status: `implemented`
Tiers: `free`, `premium`, `ultimate`
### `counts.package_events_i_package_maven_delete_package`
A count of Maven packages that have been deleted
......
lib/api/helm_packages.rb
View file @ 7ecd9ccb
......@@ -10,6 +10,7 @@ module API
feature_category :package_registry
PACKAGE_FILENAME = 'package.tgz'
FILE_NAME_REQUIREMENTS = {
file_name: API::NO_SLASH_URL_PART_REGEX
}.freeze
......@@ -75,6 +76,55 @@ module API
present_carrierwave_file!(package_file.file)
end
desc 'Authorize a chart upload from workhorse' do
detail 'This feature was introduced in GitLab 14.0'
end
params do
requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex
end
post "api/:channel/charts/authorize" do
authorize_workhorse!(
subject: authorized_user_project,
has_length: false,
maximum_size: authorized_user_project.actual_limits.helm_max_file_size
)
end
desc 'Upload a chart' do
detail 'This feature was introduced in GitLab 14.0'
end
params do
requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex
requires :chart, type: ::API::Validations::Types::WorkhorseFile, desc: 'The chart file to be published (generated by Multipart middleware)'
end
post "api/:channel/charts" do
authorize_upload!(authorized_user_project)
bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:helm_max_file_size, params[:chart].size)
package = ::Packages::CreateTemporaryPackageService.new(
authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)
).execute(:helm, name: ::Packages::Helm::TEMPORARY_PACKAGE_NAME)
chart_params = {
file: params[:chart],
file_name: PACKAGE_FILENAME
}
chart_package_file = ::Packages::CreatePackageFileService.new(
package, chart_params.merge(build: current_authenticated_job)
).execute
track_package_event('push_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace)
::Packages::Helm::ExtractionWorker.perform_async(params[:channel], chart_package_file.id) # rubocop:disable CodeReuse/Worker
created!
rescue ObjectStorage::RemoteStoreError => e
Gitlab::ErrorTracking.track_exception(e, extra: { channel: params[:channel], project_id: authorized_user_project.id })
forbidden!
end
end
end
end
......
lib/gitlab/usage_data_counters/counter_events/package_events.yml
View file @ 7ecd9ccb
......@@ -22,6 +22,7 @@
- i_package_golang_pull_package
- i_package_golang_push_package
- i_package_helm_pull_package
- i_package_helm_push_package
- i_package_maven_delete_package
- i_package_maven_pull_package
- i_package_maven_push_package
......
spec/lib/gitlab/usage_data_counters/package_event_counter_spec.rb
View file @ 7ecd9ccb
......@@ -14,7 +14,7 @@ RSpec.describe Gitlab::UsageDataCounters::PackageEventCounter, :clean_gitlab_red
end
it 'includes the right events' do
expect(described_class::KNOWN_EVENTS.size).to eq 52
expect(described_class::KNOWN_EVENTS.size).to eq 53
end
described_class::KNOWN_EVENTS.each do |event|
......
spec/requests/api/helm_packages_spec.rb
View file @ 7ecd9ccb
......@@ -20,47 +20,160 @@ RSpec.describe API::HelmPackages do
describe 'GET /api/v4/projects/:id/packages/helm/:channel/charts/:file_name.tgz' do
let(:url) { "/projects/#{project.id}/packages/helm/#{package.package_files.first.helm_channel}/charts/#{package.name}-#{package.version}.tgz" }
subject { get api(url) }
subject { get api(url), headers: headers }
context 'with valid project' do
where(:visibility, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
:public | :developer | true | true | 'process helm download content request' | :success
:public | :guest | true | true | 'process helm download content request' | :success
:public | :developer | true | false | 'rejects helm packages access' | :unauthorized
:public | :guest | true | false | 'rejects helm packages access' | :unauthorized
:public | :developer | false | true | 'process helm download content request' | :success
:public | :guest | false | true | 'process helm download content request' | :success
:public | :developer | false | false | 'rejects helm packages access' | :unauthorized
:public | :guest | false | false | 'rejects helm packages access' | :unauthorized
:public | :anonymous | false | true | 'process helm download content request' | :success
:private | :developer | true | true | 'process helm download content request' | :success
:private | :guest | true | true | 'rejects helm packages access' | :forbidden
:private | :developer | true | false | 'rejects helm packages access' | :unauthorized
:private | :guest | true | false | 'rejects helm packages access' | :unauthorized
:private | :developer | false | true | 'rejects helm packages access' | :not_found
:private | :guest | false | true | 'rejects helm packages access' | :not_found
:private | :developer | false | false | 'rejects helm packages access' | :unauthorized
:private | :guest | false | false | 'rejects helm packages access' | :unauthorized
:private | :anonymous | false | true | 'rejects helm packages access' | :unauthorized
where(:visibility, :user_role, :shared_examples_name, :expected_status) do
:public | :guest | 'process helm download content request' | :success
:public | :not_a_member | 'process helm download content request' | :success
:public | :anonymous | 'process helm download content request' | :success
:private | :reporter | 'process helm download content request' | :success
:private | :guest | 'rejects helm packages access' | :forbidden
:private | :not_a_member | 'rejects helm packages access' | :not_found
:private | :anonymous | 'rejects helm packages access' | :unauthorized
end
with_them do
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
let(:snowplow_gitlab_standard_context) { { project: project, namespace: project.namespace } }
subject { get api(url), headers: headers }
before do
project.update!(visibility: visibility.to_s)
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
end
end
context 'when an invalid token is passed' do
let(:headers) { basic_auth_header(user.username, 'wrong') }
it_behaves_like 'returning response status', :unauthorized
end
it_behaves_like 'deploy token for package GET requests'
end
describe 'POST /api/v4/projects/:id/packages/helm/api/:channel/charts/authorize' do
include_context 'workhorse headers'
let(:channel) { 'stable' }
let(:url) { "/projects/#{project.id}/packages/helm/api/#{channel}/charts/authorize" }
let(:headers) { {} }
subject { post api(url), headers: headers }
context 'with valid project' do
where(:visibility_level, :user_role, :shared_examples_name, :expected_status) do
:public | :developer | 'process helm workhorse authorization' | :success
:public | :reporter | 'rejects helm packages access' | :forbidden
:public | :not_a_member | 'rejects helm packages access' | :forbidden
:public | :anonymous | 'rejects helm packages access' | :unauthorized
:private | :developer | 'process helm workhorse authorization' | :success
:private | :reporter | 'rejects helm packages access' | :forbidden
:private | :not_a_member | 'rejects helm packages access' | :not_found
:private | :anonymous | 'rejects helm packages access' | :unauthorized
end
with_them do
let(:user_headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
let(:headers) { user_headers.merge(workhorse_headers) }
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
end
end
context 'when an invalid token is passed' do
let(:headers) { basic_auth_header(user.username, 'wrong') }
it_behaves_like 'returning response status', :unauthorized
end
it_behaves_like 'deploy token for package uploads'
it_behaves_like 'job token for package uploads', authorize_endpoint: true, accept_invalid_username: true do
let_it_be(:job) { create(:ci_build, :running, user: user, project: project) }
end
it_behaves_like 'rejects helm access with unknown project id'
end
describe 'POST /api/v4/projects/:id/packages/helm/api/:channel/charts' do
include_context 'workhorse headers'
let_it_be(:file_name) { 'package.tgz' }
let(:channel) { 'stable' }
let(:url) { "/projects/#{project.id}/packages/helm/api/#{channel}/charts" }
let(:headers) { {} }
let(:params) { { chart: temp_file(file_name) } }
let(:file_key) { :chart }
let(:send_rewritten_field) { true }
let(:snowplow_gitlab_standard_context) { { project: project, namespace: project.namespace } }
subject do
workhorse_finalize(
api(url),
method: :post,
file_key: file_key,
params: params,
headers: headers,
send_rewritten_field: send_rewritten_field
)
end
context 'with valid project' do
where(:visibility_level, :user_role, :shared_examples_name, :expected_status) do
:public | :developer | 'process helm upload' | :created
:public | :reporter | 'rejects helm packages access' | :forbidden
:public | :not_a_member | 'rejects helm packages access' | :forbidden
:public | :anonymous | 'rejects helm packages access' | :unauthorized
:private | :developer | 'process helm upload' | :created
:private | :guest | 'rejects helm packages access' | :forbidden
:private | :not_a_member | 'rejects helm packages access' | :not_found
:private | :anonymous | 'rejects helm packages access' | :unauthorized
end
with_them do
let(:user_headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
let(:headers) { user_headers.merge(workhorse_headers) }
before do
project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
end
end
context 'when an invalid token is passed' do
let(:headers) { basic_auth_header(user.username, 'wrong') }
it_behaves_like 'returning response status', :unauthorized
end
it_behaves_like 'deploy token for package uploads'
it_behaves_like 'job token for package uploads', accept_invalid_username: true do
let_it_be(:job) { create(:ci_build, :running, user: user, project: project) }
end
it_behaves_like 'rejects helm access with unknown project id'
context 'file size above maximum limit' do
let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token).merge(workhorse_headers) }
before do
allow_next_instance_of(UploadedFile) do |uploaded_file|
allow(uploaded_file).to receive(:size).and_return(project.actual_limits.helm_max_file_size + 1)
end
end
it_behaves_like 'returning response status', :bad_request
end
end
end
spec/support/shared_examples/requests/api/helm_packages_shared_examples.rb
View file @ 7ecd9ccb
# frozen_string_literal: true
RSpec.shared_examples 'rejects helm packages access' do |user_type, status, add_member = true|
RSpec.shared_examples 'rejects helm packages access' do |user_type, status|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
project.send("add_#{user_type}", user) if user_type != :anonymous && user_type != :not_a_member
end
it_behaves_like 'returning response status', status
......@@ -18,17 +18,17 @@ RSpec.shared_examples 'rejects helm packages access' do |user_type, status, add_
end
end
RSpec.shared_examples 'process helm service index request' do |user_type, status, add_member = true|
RSpec.shared_examples 'process helm service index request' do |user_type, status|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
project.send("add_#{user_type}", user) if user_type != :anonymous && user_type != :not_a_member
end
it_behaves_like 'returning response status', status
it 'returns a valid YAML response', :aggregate_failures do
subject
expect(response).to have_gitlab_http_status(status)
expect(response.media_type).to eq('text/yaml')
expect(response.body).to start_with("---\napiVersion: v1\nentries:\n")
......@@ -49,19 +49,139 @@ RSpec.shared_examples 'process helm service index request' do |user_type, status
end
end
RSpec.shared_examples 'process helm download content request' do |user_type, status, add_member = true|
RSpec.shared_examples 'process helm workhorse authorization' do |user_type, status, test_bypass: false|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
project.send("add_#{user_type}", user) if user_type != :anonymous && user_type != :not_a_member
end
it_behaves_like 'returning response status', status
it 'has the proper status and content type' do
subject
expect(response).to have_gitlab_http_status(status)
expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
end
context 'with a request that bypassed gitlab-workhorse' do
let(:headers) do
basic_auth_header(user.username, personal_access_token.token)
.merge(workhorse_headers)
.tap { |h| h.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) }
end
before do
project.add_maintainer(user)
end
it_behaves_like 'returning response status', :forbidden
end
end
end
RSpec.shared_examples 'process helm upload' do |user_type, status|
shared_examples 'creates helm package files' do
it 'creates package files' do
expect(::Packages::Helm::ExtractionWorker).to receive(:perform_async).once
expect { subject }
.to change { project.packages.count }.by(1)
.and change { Packages::PackageFile.count }.by(1)
expect(response).to have_gitlab_http_status(status)
package_file = project.packages.last.package_files.reload.last
expect(package_file.file_name).to eq('package.tgz')
end
end
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if user_type != :anonymous && user_type != :not_a_member
end
context 'with object storage disabled' do
before do
stub_package_file_object_storage(enabled: false)
end
context 'without a file from workhorse' do
let(:send_rewritten_field) { false }
it_behaves_like 'returning response status', :bad_request
end
context 'with correct params' do
it_behaves_like 'package workhorse uploads'
it_behaves_like 'creates helm package files'
it_behaves_like 'a package tracking event', 'API::HelmPackages', 'push_package'
end
end
context 'with object storage enabled' do
let(:tmp_object) do
fog_connection.directories.new(key: 'packages').files.create( # rubocop:disable Rails/SaveBang
key: "tmp/uploads/#{file_name}",
body: 'content'
)
end
let(:fog_file) { fog_to_uploaded_file(tmp_object) }
let(:params) { { chart: fog_file, 'chart.remote_id' => file_name } }
context 'and direct upload enabled' do
let(:fog_connection) do
stub_package_file_object_storage(direct_upload: true)
end
it_behaves_like 'creates helm package files'
['123123', '../../123123'].each do |remote_id|
context "with invalid remote_id: #{remote_id}" do
let(:params) do
{
chart: fog_file,
'chart.remote_id' => remote_id
}
end
it_behaves_like 'returning response status', :forbidden
end
end
end
context 'and direct upload disabled' do
context 'and background upload disabled' do
let(:fog_connection) do
stub_package_file_object_storage(direct_upload: false, background_upload: false)
end
it_behaves_like 'creates helm package files'
end
context 'and background upload enabled' do
let(:fog_connection) do
stub_package_file_object_storage(direct_upload: false, background_upload: true)
end
it_behaves_like 'creates helm package files'
end
end
end
it_behaves_like 'background upload schedules a file migration'
end
end
RSpec.shared_examples 'process helm download content request' do |user_type, status|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if user_type != :anonymous && user_type != :not_a_member
end
it_behaves_like 'a package tracking event', 'API::HelmPackages', 'pull_package'
it 'returns a valid package archive' do
it 'returns expected status and a valid package archive' do
subject
expect(response).to have_gitlab_http_status(status)
expect(response.media_type).to eq('application/octet-stream')
end
end
......@@ -83,81 +203,61 @@ RSpec.shared_examples 'rejects helm access with unknown project id' do
end
end
RSpec.shared_examples 'handling helm chart index requests' do |anonymous_requests_example_name: 'process helm service index request', anonymous_requests_status: :success|
RSpec.shared_examples 'handling helm chart index requests' do
context 'with valid project' do
subject { get api(url), headers: headers }
using RSpec::Parameterized::TableSyntax
context 'personal token' do
where(:visibility, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
:public | :developer | true | true | 'process helm service index request' | :success
:public | :guest | true | true | 'process helm service index request' | :success
:public | :developer | true | false | 'rejects helm packages access' | :unauthorized
:public | :guest | true | false | 'rejects helm packages access' | :unauthorized
:public | :developer | false | true | 'process helm service index request' | :success
:public | :guest | false | true | 'process helm service index request' | :success
:public | :developer | false | false | 'rejects helm packages access' | :unauthorized
:public | :guest | false | false | 'rejects helm packages access' | :unauthorized
:public | :anonymous | false | true | anonymous_requests_example_name | anonymous_requests_status
:private | :developer | true | true | 'process helm service index request' | :success
:private | :guest | true | true | 'rejects helm packages access' | :forbidden
:private | :developer | true | false | 'rejects helm packages access' | :unauthorized
:private | :guest | true | false | 'rejects helm packages access' | :unauthorized
:private | :developer | false | true | 'rejects helm packages access' | :not_found
:private | :guest | false | true | 'rejects helm packages access' | :not_found
:private | :developer | false | false | 'rejects helm packages access' | :unauthorized
:private | :guest | false | false | 'rejects helm packages access' | :unauthorized
:private | :anonymous | false | true | 'rejects helm packages access' | :unauthorized
where(:visibility, :user_role, :shared_examples_name, :expected_status) do
:public | :guest | 'process helm service index request' | :success
:public | :not_a_member | 'process helm service index request' | :success
:public | :anonymous | 'process helm service index request' | :success
:private | :reporter | 'process helm service index request' | :success
:private | :guest | 'rejects helm packages access' | :forbidden
:private | :not_a_member | 'rejects helm packages access' | :not_found
:private | :anonymous | 'rejects helm packages access' | :unauthorized
end
with_them do
let(:token) { user_token ? personal_access_token.token : 'wrong' }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
subject { get api(url), headers: headers }
let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
before do
project.update!(visibility: visibility.to_s)
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
end
end
context 'when an invalid token is passed' do
let(:headers) { basic_auth_header(user.username, 'wrong') }
it_behaves_like 'returning response status', :unauthorized
end
context 'with job token' do
where(:visibility, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
:public | :developer | true | true | 'process helm service index request' | :success
:public | :guest | true | true | 'process helm service index request' | :success
:public | :developer | true | false | 'rejects helm packages access' | :unauthorized
:public | :guest | true | false | 'rejects helm packages access' | :unauthorized
:public | :developer | false | true | 'process helm service index request' | :success
:public | :guest | false | true | 'process helm service index request' | :success
:public | :developer | false | false | 'rejects helm packages access' | :unauthorized
:public | :guest | false | false | 'rejects helm packages access' | :unauthorized
:public | :anonymous | false | true | anonymous_requests_example_name | anonymous_requests_status
:private | :developer | true | true | 'process helm service index request' | :success
:private | :guest | true | true | 'rejects helm packages access' | :forbidden
:private | :developer | true | false | 'rejects helm packages access' | :unauthorized
:private | :guest | true | false | 'rejects helm packages access' | :unauthorized
:private | :developer | false | true | 'rejects helm packages access' | :not_found
:private | :guest | false | true | 'rejects helm packages access' | :not_found
:private | :developer | false | false | 'rejects helm packages access' | :unauthorized
:private | :guest | false | false | 'rejects helm packages access' | :unauthorized
:private | :anonymous | false | true | 'rejects helm packages access' | :unauthorized
where(:visibility, :user_role, :shared_examples_name, :expected_status) do
:public | :guest | 'process helm service index request' | :success
:public | :not_a_member | 'process helm service index request' | :success
:public | :anonymous | 'process helm service index request' | :success
:private | :reporter | 'process helm service index request' | :success
:private | :guest | 'rejects helm packages access' | :forbidden
:private | :not_a_member | 'rejects helm packages access' | :not_found
:private | :anonymous | 'rejects helm packages access' | :unauthorized
end
with_them do
let_it_be(:ci_build) { create(:ci_build, project: project, user: user, status: :running) }
let(:job) { user_token ? ci_build : double(token: 'wrong') }
let(:headers) { user_role == :anonymous ? {} : job_basic_auth_header(job) }
subject { get api(url), headers: headers }
let(:headers) { user_role == :anonymous ? {} : job_basic_auth_header(ci_build) }
before do
project.update!(visibility: visibility.to_s)
end
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
end
end
end
......
spec/support/shared_examples/requests/api/packages_shared_examples.rb
View file @ 7ecd9ccb
......@@ -100,7 +100,7 @@ RSpec.shared_examples 'job token for package GET requests' do
end
end
RSpec.shared_examples 'job token for package uploads' do |authorize_endpoint: false|
RSpec.shared_examples 'job token for package uploads' do |authorize_endpoint: false, accept_invalid_username: false|
context 'with job token headers' do
let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token).merge(workhorse_headers) }
......@@ -133,7 +133,11 @@ RSpec.shared_examples 'job token for package uploads' do |authorize_endpoint: fa
context 'invalid user' do
let(:headers) { basic_auth_header('foo', job.token).merge(workhorse_headers) }
it_behaves_like 'returning response status', :unauthorized
if accept_invalid_username
it_behaves_like 'returning response status', :success
else
it_behaves_like 'returning response status', :unauthorized
end
end
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7