Merge branch 'helm_upload_endpoint' into 'master'

Helm chart upload API endpoint

See merge request gitlab-org/gitlab!64814

app/models/packages/helm.rb

@@ -2,6 +2,8 @@
 
 module Packages
   module Helm
+    TEMPORARY_PACKAGE_NAME = 'Helm.Temporary.Package'
+
     def self.table_name_prefix
       'packages_helm_'
     end

config/metrics/counts_all/20210625095025_package_events_i_package_helm_push_package.yml

+---
+key_path: counts.package_events_i_package_helm_push_package
+description: The total count of Helm packages that have been published.
+product_section: ops
+product_stage: package
+product_group: group::package
+product_category: package_registry
+value_type: number
+status: implemented
+milestone: "14.1"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64814
+time_frame: all
+data_source: redis
+data_category: Optional
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

doc/development/usage_ping/dictionary.md

@@ -4346,6 +4346,18 @@ Status: `implemented`
 
 Tiers: `free`, `premium`, `ultimate`
 
+### `counts.package_events_i_package_helm_push_package`
+
+The total count of Helm packages that have been published.
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210625095025_package_events_i_package_helm_push_package.yml)
+
+Group: `group::package`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `counts.package_events_i_package_maven_delete_package`
 
 A count of Maven packages that have been deleted

lib/api/helm_packages.rb

@@ -10,6 +10,7 @@ module API
 
     feature_category :package_registry
 
+    PACKAGE_FILENAME = 'package.tgz'
     FILE_NAME_REQUIREMENTS = {
       file_name: API::NO_SLASH_URL_PART_REGEX
     }.freeze
@@ -75,6 +76,55 @@ module API
 
           present_carrierwave_file!(package_file.file)
         end
+
+        desc 'Authorize a chart upload from workhorse' do
+          detail 'This feature was introduced in GitLab 14.0'
+        end
+        params do
+          requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex
+        end
+        post "api/:channel/charts/authorize" do
+          authorize_workhorse!(
+            subject: authorized_user_project,
+            has_length: false,
+            maximum_size: authorized_user_project.actual_limits.helm_max_file_size
+          )
+        end
+
+        desc 'Upload a chart' do
+          detail 'This feature was introduced in GitLab 14.0'
+        end
+        params do
+          requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex
+          requires :chart, type: ::API::Validations::Types::WorkhorseFile, desc: 'The chart file to be published (generated by Multipart middleware)'
+        end
+        post "api/:channel/charts" do
+          authorize_upload!(authorized_user_project)
+          bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:helm_max_file_size, params[:chart].size)
+
+          package = ::Packages::CreateTemporaryPackageService.new(
+            authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)
+          ).execute(:helm, name: ::Packages::Helm::TEMPORARY_PACKAGE_NAME)
+
+          chart_params = {
+            file:      params[:chart],
+            file_name: PACKAGE_FILENAME
+          }
+
+          chart_package_file = ::Packages::CreatePackageFileService.new(
+            package, chart_params.merge(build: current_authenticated_job)
+          ).execute
+
+          track_package_event('push_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace)
+
+          ::Packages::Helm::ExtractionWorker.perform_async(params[:channel], chart_package_file.id) # rubocop:disable CodeReuse/Worker
+
+          created!
+        rescue ObjectStorage::RemoteStoreError => e
+          Gitlab::ErrorTracking.track_exception(e, extra: { channel: params[:channel], project_id: authorized_user_project.id })
+
+          forbidden!
+        end
       end
     end
   end

lib/gitlab/usage_data_counters/counter_events/package_events.yml

@@ -22,6 +22,7 @@
 - i_package_golang_pull_package
 - i_package_golang_push_package
 - i_package_helm_pull_package
+- i_package_helm_push_package
 - i_package_maven_delete_package
 - i_package_maven_pull_package
 - i_package_maven_push_package

spec/lib/gitlab/usage_data_counters/package_event_counter_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe Gitlab::UsageDataCounters::PackageEventCounter, :clean_gitlab_red
   end
 
   it 'includes the right events' do
-    expect(described_class::KNOWN_EVENTS.size).to eq 52
+    expect(described_class::KNOWN_EVENTS.size).to eq 53
   end
 
   described_class::KNOWN_EVENTS.each do |event|

spec/requests/api/helm_packages_spec.rb

@@ -20,47 +20,160 @@ RSpec.describe API::HelmPackages do
   describe 'GET /api/v4/projects/:id/packages/helm/:channel/charts/:file_name.tgz' do
     let(:url) { "/projects/#{project.id}/packages/helm/#{package.package_files.first.helm_channel}/charts/#{package.name}-#{package.version}.tgz" }
 
-    subject { get api(url) }
+    subject { get api(url), headers: headers }
 
     context 'with valid project' do
-      where(:visibility, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
-        :public  | :developer  | true  | true  | 'process helm download content request'   | :success
-        :public  | :guest      | true  | true  | 'process helm download content request'   | :success
-        :public  | :developer  | true  | false | 'rejects helm packages access'            | :unauthorized
-        :public  | :guest      | true  | false | 'rejects helm packages access'            | :unauthorized
-        :public  | :developer  | false | true  | 'process helm download content request'   | :success
-        :public  | :guest      | false | true  | 'process helm download content request'   | :success
-        :public  | :developer  | false | false | 'rejects helm packages access'            | :unauthorized
-        :public  | :guest      | false | false | 'rejects helm packages access'            | :unauthorized
-        :public  | :anonymous  | false | true  | 'process helm download content request'   | :success
-        :private | :developer  | true  | true  | 'process helm download content request'   | :success
-        :private | :guest      | true  | true  | 'rejects helm packages access'            | :forbidden
-        :private | :developer  | true  | false | 'rejects helm packages access'            | :unauthorized
-        :private | :guest      | true  | false | 'rejects helm packages access'            | :unauthorized
-        :private | :developer  | false | true  | 'rejects helm packages access'            | :not_found
-        :private | :guest      | false | true  | 'rejects helm packages access'            | :not_found
-        :private | :developer  | false | false | 'rejects helm packages access'            | :unauthorized
-        :private | :guest      | false | false | 'rejects helm packages access'            | :unauthorized
-        :private | :anonymous  | false | true  | 'rejects helm packages access'            | :unauthorized
+      where(:visibility, :user_role, :shared_examples_name, :expected_status) do
+        :public  | :guest        | 'process helm download content request'   | :success
+        :public  | :not_a_member | 'process helm download content request'   | :success
+        :public  | :anonymous    | 'process helm download content request'   | :success
+        :private | :reporter     | 'process helm download content request'   | :success
+        :private | :guest        | 'rejects helm packages access'            | :forbidden
+        :private | :not_a_member | 'rejects helm packages access'            | :not_found
+        :private | :anonymous    | 'rejects helm packages access'            | :unauthorized
       end
 
       with_them do
-        let(:token) { user_token ? personal_access_token.token : 'wrong' }
-        let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
+        let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
         let(:snowplow_gitlab_standard_context) { { project: project, namespace: project.namespace } }
 
-        subject { get api(url), headers: headers }
-
         before do
           project.update!(visibility: visibility.to_s)
         end
 
-        it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+        it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
+      end
     end
+
+    context 'when an invalid token is passed' do
+      let(:headers) { basic_auth_header(user.username, 'wrong') }
+
+      it_behaves_like 'returning response status', :unauthorized
     end
 
     it_behaves_like 'deploy token for package GET requests'
+  end
+
+  describe 'POST /api/v4/projects/:id/packages/helm/api/:channel/charts/authorize' do
+    include_context 'workhorse headers'
+
+    let(:channel) { 'stable' }
+    let(:url) { "/projects/#{project.id}/packages/helm/api/#{channel}/charts/authorize" }
+    let(:headers) { {} }
+
+    subject { post api(url), headers: headers }
+
+    context 'with valid project' do
+      where(:visibility_level, :user_role, :shared_examples_name, :expected_status) do
+        :public  | :developer    | 'process helm workhorse authorization' | :success
+        :public  | :reporter     | 'rejects helm packages access'         | :forbidden
+        :public  | :not_a_member | 'rejects helm packages access'         | :forbidden
+        :public  | :anonymous    | 'rejects helm packages access'         | :unauthorized
+        :private | :developer    | 'process helm workhorse authorization' | :success
+        :private | :reporter     | 'rejects helm packages access'         | :forbidden
+        :private | :not_a_member | 'rejects helm packages access'         | :not_found
+        :private | :anonymous    | 'rejects helm packages access'         | :unauthorized
+      end
+
+      with_them do
+        let(:user_headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
+        let(:headers) { user_headers.merge(workhorse_headers) }
+
+        before do
+          project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+        end
+
+        it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
+      end
+    end
+
+    context 'when an invalid token is passed' do
+      let(:headers) { basic_auth_header(user.username, 'wrong') }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+
+    it_behaves_like 'deploy token for package uploads'
+
+    it_behaves_like 'job token for package uploads', authorize_endpoint: true, accept_invalid_username: true do
+      let_it_be(:job) { create(:ci_build, :running, user: user, project: project) }
+    end
+
+    it_behaves_like 'rejects helm access with unknown project id'
+  end
+
+  describe 'POST /api/v4/projects/:id/packages/helm/api/:channel/charts' do
+    include_context 'workhorse headers'
+
+    let_it_be(:file_name) { 'package.tgz' }
+
+    let(:channel) { 'stable' }
+    let(:url) { "/projects/#{project.id}/packages/helm/api/#{channel}/charts" }
+    let(:headers) { {} }
+    let(:params) { { chart: temp_file(file_name) } }
+    let(:file_key) { :chart }
+    let(:send_rewritten_field) { true }
+    let(:snowplow_gitlab_standard_context) { { project: project, namespace: project.namespace } }
+
+    subject do
+      workhorse_finalize(
+        api(url),
+        method: :post,
+        file_key: file_key,
+        params: params,
+        headers: headers,
+        send_rewritten_field: send_rewritten_field
+      )
+    end
+
+    context 'with valid project' do
+      where(:visibility_level, :user_role, :shared_examples_name, :expected_status) do
+        :public  | :developer     | 'process helm upload'          | :created
+        :public  | :reporter      | 'rejects helm packages access' | :forbidden
+        :public  | :not_a_member  | 'rejects helm packages access' | :forbidden
+        :public  | :anonymous     | 'rejects helm packages access' | :unauthorized
+        :private | :developer     | 'process helm upload'          | :created
+        :private | :guest         | 'rejects helm packages access' | :forbidden
+        :private | :not_a_member  | 'rejects helm packages access' | :not_found
+        :private | :anonymous     | 'rejects helm packages access' | :unauthorized
+      end
+
+      with_them do
+        let(:user_headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, personal_access_token.token) }
+        let(:headers) { user_headers.merge(workhorse_headers) }
+
+        before do
+          project.update_column(:visibility_level, Gitlab::VisibilityLevel.level_value(visibility_level.to_s))
+        end
+
+        it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status]
+      end
+    end
+
+    context 'when an invalid token is passed' do
+      let(:headers) { basic_auth_header(user.username, 'wrong') }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+
+    it_behaves_like 'deploy token for package uploads'
+
+    it_behaves_like 'job token for package uploads', accept_invalid_username: true do
+      let_it_be(:job) { create(:ci_build, :running, user: user, project: project) }
+    end
 
     it_behaves_like 'rejects helm access with unknown project id'
+
+    context 'file size above maximum limit' do
+      let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token).merge(workhorse_headers) }
+
+      before do
+        allow_next_instance_of(UploadedFile) do |uploaded_file|
+          allow(uploaded_file).to receive(:size).and_return(project.actual_limits.helm_max_file_size + 1)
+        end
+      end
+
+      it_behaves_like 'returning response status', :bad_request
+    end
   end
 end

spec/support/shared_examples/requests/api/packages_shared_examples.rb

@@ -100,7 +100,7 @@ RSpec.shared_examples 'job token for package GET requests' do
   end
 end
 
-RSpec.shared_examples 'job token for package uploads' do |authorize_endpoint: false|
+RSpec.shared_examples 'job token for package uploads' do |authorize_endpoint: false, accept_invalid_username: false|
   context 'with job token headers' do
     let(:headers) { basic_auth_header(::Gitlab::Auth::CI_JOB_USER, job.token).merge(workhorse_headers) }
 
@@ -133,9 +133,13 @@ RSpec.shared_examples 'job token for package uploads' do |authorize_endpoint: fa
     context 'invalid user' do
       let(:headers) { basic_auth_header('foo', job.token).merge(workhorse_headers) }
 
+      if accept_invalid_username
+        it_behaves_like 'returning response status', :success
+      else
         it_behaves_like 'returning response status', :unauthorized
       end
     end
+  end
 end
 
 RSpec.shared_examples 'a package tracking event' do |category, action|