Commit fd225d48 authored by Stan Hu's avatar Stan Hu

Merge branch '326819-check-maven-path' into 'master'

Check maven path for maven file API endpoints [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!59241
parents b0ec3d8a 37a34673
......@@ -19,6 +19,7 @@ class Packages::Maven::Metadatum < ApplicationRecord
validate :maven_package_type
scope :for_package_ids, -> (package_ids) { where(package_id: package_ids) }
scope :with_path, ->(path) { where(path: path) }
scope :order_created, -> { reorder('created_at ASC') }
def self.pluck_app_name
......
---
title: Add index for the path column on the packages_maven_metadata table
merge_request: 59241
author:
type: performance
---
name: check_maven_path_first
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59241
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/327487
milestone: '13.11'
type: development
group: group::package
default_enabled: false
# frozen_string_literal: true
class AddIndexToPackagesMavenMetadataPath < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
INDEX_NAME = 'index_packages_maven_metadata_on_path'
def up
add_concurrent_index :packages_maven_metadata, :path, name: INDEX_NAME
end
def down
remove_concurrent_index :packages_maven_metadata, :path, name: INDEX_NAME
end
end
2da634fa920e3989d9b8e53ddc1ba005e5bc0f4701426e3841d90a42bd2e908f
\ No newline at end of file
......@@ -23360,6 +23360,8 @@ CREATE INDEX index_packages_events_on_package_id ON packages_events USING btree
CREATE INDEX index_packages_maven_metadata_on_package_id_and_path ON packages_maven_metadata USING btree (package_id, path);
CREATE INDEX index_packages_maven_metadata_on_path ON packages_maven_metadata USING btree (path);
CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_nuget_dependency_link_metadata USING btree (dependency_link_id);
CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_generic ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 7);
......@@ -23,6 +23,15 @@ module API
helpers ::API::Helpers::PackagesHelpers
helpers do
def path_exists?(path)
# return true when FF disabled so that processing the request is not stopped
return true unless Feature.enabled?(:check_maven_path_first)
return false if path.blank?
Packages::Maven::Metadatum.with_path(path)
.exists?
end
def extract_format(file_name)
name, _, format = file_name.rpartition('.')
......@@ -104,6 +113,9 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
# return a similar failure to authorize_read_package!(project)
forbidden! unless path_exists?(params[:path])
file_name, format = extract_format(params[:file_name])
# To avoid name collision we require project path and project package be the same.
......@@ -142,6 +154,9 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
# return a similar failure to group = find_group(params[:id])
not_found!('Group') unless path_exists?(params[:path])
file_name, format = extract_format(params[:file_name])
group = find_group(params[:id])
......@@ -181,6 +196,9 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
# return a similar failure to user_project
not_found!('Project') unless path_exists?(params[:path])
authorize_read_package!(user_project)
file_name, format = extract_format(params[:file_name])
......
......@@ -54,7 +54,7 @@ RSpec.describe Packages::Maven::Metadatum, type: :model do
let_it_be(:metadatum3) { create(:maven_metadatum, package: package) }
let_it_be(:metadatum4) { create(:maven_metadatum, package: package) }
subject { Packages::Maven::Metadatum.for_package_ids(package.id).order_created }
subject { described_class.for_package_ids(package.id).order_created }
it { is_expected.to eq([metadatum1, metadatum2, metadatum3, metadatum4]) }
end
......@@ -64,10 +64,20 @@ RSpec.describe Packages::Maven::Metadatum, type: :model do
let_it_be(:metadatum2) { create(:maven_metadatum, package: package, app_name: 'two') }
let_it_be(:metadatum3) { create(:maven_metadatum, package: package, app_name: 'three') }
subject { Packages::Maven::Metadatum.for_package_ids(package.id).pluck_app_name }
subject { described_class.for_package_ids(package.id).pluck_app_name }
it { is_expected.to match_array([metadatum1, metadatum2, metadatum3].map(&:app_name)) }
end
describe '.with_path' do
let_it_be(:metadatum1) { create(:maven_metadatum, package: package, path: 'one') }
let_it_be(:metadatum2) { create(:maven_metadatum, package: package, path: 'two') }
let_it_be(:metadatum3) { create(:maven_metadatum, package: package, path: 'three') }
subject { described_class.with_path('two') }
it { is_expected.to match_array([metadatum2]) }
end
end
end
end
......@@ -47,7 +47,21 @@ RSpec.describe API::MavenPackages do
end
end
shared_examples 'processing HEAD requests' do
shared_examples 'rejecting the request for non existing maven path' do |expected_status: :not_found|
before do
if Feature.enabled?(:check_maven_path_first)
expect(::Packages::Maven::PackageFinder).not_to receive(:new)
end
end
it 'rejects the request' do
subject
expect(response).to have_gitlab_http_status(expected_status)
end
end
shared_examples 'processing HEAD requests' do |instance_level: false|
subject { head api(url) }
before do
......@@ -92,6 +106,12 @@ RSpec.describe API::MavenPackages do
subject
end
context 'with a non existing maven path' do
let(:path) { 'foo/bar/1.2.3' }
it_behaves_like 'rejecting the request for non existing maven path', expected_status: instance_level ? :forbidden : :not_found
end
end
end
......@@ -99,9 +119,8 @@ RSpec.describe API::MavenPackages do
context 'successful download' do
subject do
download_file(
package_file.file_name,
{},
Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => deploy_token.token
file_name: package_file.file_name,
request_headers: { Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => deploy_token.token }
)
end
......@@ -126,7 +145,7 @@ RSpec.describe API::MavenPackages do
shared_examples 'downloads with a job token' do
context 'with a running job' do
it 'allows download with job token' do
download_file(package_file.file_name, job_token: job.token)
download_file(file_name: package_file.file_name, params: { job_token: job.token })
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
......@@ -139,7 +158,7 @@ RSpec.describe API::MavenPackages do
end
it 'returns unauthorized error' do
download_file(package_file.file_name, job_token: job.token)
download_file(file_name: package_file.file_name, params: { job_token: job.token })
expect(response).to have_gitlab_http_status(:unauthorized)
end
......@@ -149,7 +168,7 @@ RSpec.describe API::MavenPackages do
describe 'GET /api/v4/packages/maven/*path/:file_name' do
shared_examples 'handling all conditions' do
context 'a public project' do
subject { download_file(package_file.file_name) }
subject { download_file(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -161,12 +180,18 @@ RSpec.describe API::MavenPackages do
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
download_file(file_name: package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
context 'with a non existing maven path' do
subject { download_file(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path', expected_status: :forbidden
end
end
context 'internal project' do
......@@ -175,7 +200,7 @@ RSpec.describe API::MavenPackages do
project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
subject { download_file_with_token(package_file.file_name) }
subject { download_file_with_token(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -187,7 +212,7 @@ RSpec.describe API::MavenPackages do
end
it 'denies download when no private token' do
download_file(package_file.file_name)
download_file(file_name: package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
......@@ -195,10 +220,16 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path', expected_status: :forbidden
end
end
context 'private project' do
subject { download_file_with_token(package_file.file_name) }
subject { download_file_with_token(file_name: package_file.file_name) }
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
......@@ -222,7 +253,7 @@ RSpec.describe API::MavenPackages do
end
it 'denies download when no private token' do
download_file(package_file.file_name)
download_file(file_name: package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
......@@ -241,13 +272,18 @@ RSpec.describe API::MavenPackages do
unauthorized_deploy_token.update!(id: another_user.id)
download_file(
package_file.file_name,
{},
Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => unauthorized_deploy_token.token
file_name: package_file.file_name,
request_headers: { Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => unauthorized_deploy_token.token }
)
expect(response).to have_gitlab_http_status(:forbidden)
end
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path', expected_status: :forbidden
end
end
context 'project name is different from a package name' do
......@@ -256,7 +292,7 @@ RSpec.describe API::MavenPackages do
end
it 'rejects request' do
download_file(package_file.file_name)
download_file(file_name: package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
......@@ -279,26 +315,43 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'handling all conditions'
end
def download_file(file_name, params = {}, request_headers = headers)
get api("/packages/maven/#{maven_metadatum.path}/#{file_name}"), params: params, headers: request_headers
context 'with check_maven_path_first enabled' do
before do
stub_feature_flags(check_maven_path_first: true)
end
def download_file_with_token(file_name, params = {}, request_headers = headers_with_token)
download_file(file_name, params, request_headers)
it_behaves_like 'handling all conditions'
end
context 'with check_maven_path_first disabled' do
before do
stub_feature_flags(check_maven_path_first: false)
end
it_behaves_like 'handling all conditions'
end
def download_file(file_name:, params: {}, request_headers: headers, path: maven_metadatum.path)
get api("/packages/maven/#{path}/#{file_name}"), params: params, headers: request_headers
end
def download_file_with_token(file_name:, params: {}, request_headers: headers_with_token, path: maven_metadatum.path)
download_file(file_name: file_name, params: params, request_headers: request_headers, path: path)
end
end
describe 'HEAD /api/v4/packages/maven/*path/:file_name' do
let(:url) { "/packages/maven/#{package.maven_metadatum.path}/#{package_file.file_name}" }
let(:path) { package.maven_metadatum.path }
let(:url) { "/packages/maven/#{path}/#{package_file.file_name}" }
it_behaves_like 'processing HEAD requests'
it_behaves_like 'processing HEAD requests', instance_level: true
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
end
it_behaves_like 'processing HEAD requests'
it_behaves_like 'processing HEAD requests', instance_level: true
end
context 'with maven_packages_group_level_improvements disabled' do
......@@ -306,7 +359,23 @@ RSpec.describe API::MavenPackages do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it_behaves_like 'processing HEAD requests'
it_behaves_like 'processing HEAD requests', instance_level: true
end
context 'with check_maven_path_first enabled' do
before do
stub_feature_flags(check_maven_path_first: true)
end
it_behaves_like 'processing HEAD requests', instance_level: true
end
context 'with check_maven_path_first disabled' do
before do
stub_feature_flags(check_maven_path_first: false)
end
it_behaves_like 'processing HEAD requests', instance_level: true
end
end
......@@ -318,7 +387,7 @@ RSpec.describe API::MavenPackages do
shared_examples 'handling all conditions' do
context 'a public project' do
subject { download_file(package_file.file_name) }
subject { download_file(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -330,12 +399,18 @@ RSpec.describe API::MavenPackages do
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
download_file(file_name: package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
context 'with a non existing maven path' do
subject { download_file(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path'
end
end
context 'internal project' do
......@@ -344,7 +419,7 @@ RSpec.describe API::MavenPackages do
project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
subject { download_file_with_token(package_file.file_name) }
subject { download_file_with_token(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -356,7 +431,7 @@ RSpec.describe API::MavenPackages do
end
it 'denies download when no private token' do
download_file(package_file.file_name)
download_file(file_name: package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
......@@ -364,6 +439,12 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path'
end
end
context 'private project' do
......@@ -371,7 +452,7 @@ RSpec.describe API::MavenPackages do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
subject { download_file_with_token(package_file.file_name) }
subject { download_file_with_token(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -392,7 +473,7 @@ RSpec.describe API::MavenPackages do
end
it 'denies download when no private token' do
download_file(package_file.file_name)
download_file(file_name: package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
......@@ -401,8 +482,14 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'downloads with a deploy token'
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path'
end
context 'with group deploy token' do
subject { download_file_with_token(package_file.file_name, {}, group_deploy_token_headers) }
subject { download_file_with_token(file_name: package_file.file_name, request_headers: group_deploy_token_headers) }
it 'returns the file' do
subject
......@@ -419,13 +506,19 @@ RSpec.describe API::MavenPackages do
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3', request_headers: group_deploy_token_headers) }
it_behaves_like 'rejecting the request for non existing maven path'
end
end
context 'with a reporter from a subgroup accessing the root group' do
let_it_be(:root_group) { create(:group, :private) }
let_it_be(:group) { create(:group, :private, parent: root_group) }
subject { download_file_with_token(package_file.file_name, {}, headers_with_token, root_group.id) }
subject { download_file_with_token(file_name: package_file.file_name, request_headers: headers_with_token, group_id: root_group.id) }
before do
project.update!(namespace: group)
......@@ -438,6 +531,12 @@ RSpec.describe API::MavenPackages do
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3', request_headers: headers_with_token, group_id: root_group.id) }
it_behaves_like 'rejecting the request for non existing maven path'
end
end
end
......@@ -457,7 +556,7 @@ RSpec.describe API::MavenPackages do
let(:maven_metadatum) { package3.maven_metadatum }
subject { download_file_with_token(package_file3.file_name) }
subject { download_file_with_token(file_name: package_file3.file_name) }
before do
sub_group1.add_developer(user)
......@@ -511,17 +610,34 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'handling all conditions'
end
def download_file(file_name, params = {}, request_headers = headers, group_id = group.id)
get api("/groups/#{group_id}/-/packages/maven/#{maven_metadatum.path}/#{file_name}"), params: params, headers: request_headers
context 'with check_maven_path_first enabled' do
before do
stub_feature_flags(check_maven_path_first: true)
end
def download_file_with_token(file_name, params = {}, request_headers = headers_with_token, group_id = group.id)
download_file(file_name, params, request_headers, group_id)
it_behaves_like 'handling all conditions'
end
context 'with check_maven_path_first disabled' do
before do
stub_feature_flags(check_maven_path_first: false)
end
it_behaves_like 'handling all conditions'
end
def download_file(file_name:, params: {}, request_headers: headers, path: maven_metadatum.path, group_id: group.id)
get api("/groups/#{group_id}/-/packages/maven/#{path}/#{file_name}"), params: params, headers: request_headers
end
def download_file_with_token(file_name:, params: {}, request_headers: headers_with_token, path: maven_metadatum.path, group_id: group.id)
download_file(file_name: file_name, params: params, request_headers: request_headers, path: path, group_id: group_id)
end
end
describe 'HEAD /api/v4/groups/:id/-/packages/maven/*path/:file_name' do
let(:url) { "/groups/#{group.id}/-/packages/maven/#{package.maven_metadatum.path}/#{package_file.file_name}" }
let(:path) { package.maven_metadatum.path }
let(:url) { "/groups/#{group.id}/-/packages/maven/#{path}/#{package_file.file_name}" }
context 'with maven_packages_group_level_improvements enabled' do
before do
......@@ -538,12 +654,28 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'processing HEAD requests'
end
context 'with check_maven_path_first enabled' do
before do
stub_feature_flags(check_maven_path_first: true)
end
it_behaves_like 'processing HEAD requests'
end
context 'with check_maven_path_first disabled' do
before do
stub_feature_flags(check_maven_path_first: false)
end
it_behaves_like 'processing HEAD requests'
end
end
describe 'GET /api/v4/projects/:id/packages/maven/*path/:file_name' do
shared_examples 'handling all conditions' do
context 'a public project' do
subject { download_file(package_file.file_name) }
subject { download_file(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -555,12 +687,18 @@ RSpec.describe API::MavenPackages do
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
download_file(file_name: package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
context 'with a non existing maven path' do
subject { download_file(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path'
end
end
context 'private project' do
......@@ -568,7 +706,7 @@ RSpec.describe API::MavenPackages do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
subject { download_file_with_token(package_file.file_name) }
subject { download_file_with_token(file_name: package_file.file_name) }
it_behaves_like 'tracking the file download event'
......@@ -588,7 +726,7 @@ RSpec.describe API::MavenPackages do
end
it 'denies download when no private token' do
download_file(package_file.file_name)
download_file(file_name: package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
......@@ -596,6 +734,12 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
context 'with a non existing maven path' do
subject { download_file_with_token(file_name: package_file.file_name, path: 'foo/bar/1.2.3') }
it_behaves_like 'rejecting the request for non existing maven path'
end
end
end
......@@ -615,18 +759,35 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'handling all conditions'
end
def download_file(file_name, params = {}, request_headers = headers)
context 'with check_maven_path_first enabled' do
before do
stub_feature_flags(check_maven_path_first: true)
end
it_behaves_like 'handling all conditions'
end
context 'with check_maven_path_first disabled' do
before do
stub_feature_flags(check_maven_path_first: false)
end
it_behaves_like 'handling all conditions'
end
def download_file(file_name:, params: {}, request_headers: headers, path: maven_metadatum.path)
get api("/projects/#{project.id}/packages/maven/" \
"#{maven_metadatum.path}/#{file_name}"), params: params, headers: request_headers
"#{path}/#{file_name}"), params: params, headers: request_headers
end
def download_file_with_token(file_name, params = {}, request_headers = headers_with_token)
download_file(file_name, params, request_headers)
def download_file_with_token(file_name:, params: {}, request_headers: headers_with_token, path: maven_metadatum.path)
download_file(file_name: file_name, params: params, request_headers: request_headers, path: path)
end
end
describe 'HEAD /api/v4/projects/:id/packages/maven/*path/:file_name' do
let(:url) { "/projects/#{project.id}/packages/maven/#{package.maven_metadatum.path}/#{package_file.file_name}" }
let(:path) { package.maven_metadatum.path }
let(:url) { "/projects/#{project.id}/packages/maven/#{path}/#{package_file.file_name}" }
context 'with maven_packages_group_level_improvements enabled' do
before do
......@@ -643,11 +804,27 @@ RSpec.describe API::MavenPackages do
it_behaves_like 'processing HEAD requests'
end
context 'with check_maven_path_first enabled' do
before do
stub_feature_flags(check_maven_path_first: true)
end
it_behaves_like 'processing HEAD requests'
end
context 'with check_maven_path_first disabled' do
before do
stub_feature_flags(check_maven_path_first: false)
end
it_behaves_like 'processing HEAD requests'
end
end
describe 'PUT /api/v4/projects/:id/packages/maven/*path/:file_name/authorize' do
it 'rejects a malicious request' do
put api("/projects/#{project.id}/packages/maven/com/example/my-app/#{version}/%2e%2e%2F.ssh%2Fauthorized_keys/authorize"), params: {}, headers: headers_with_token
put api("/projects/#{project.id}/packages/maven/com/example/my-app/#{version}/%2e%2e%2F.ssh%2Fauthorized_keys/authorize"), headers: headers_with_token
expect(response).to have_gitlab_http_status(:bad_request)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment