Commit d5e63a6d authored by Ethan Reesor's avatar Ethan Reesor

Clean up Go proxy implementation

- Various fixes and improvements
- Hide proxy behind a feature flag, due to performance issues: #218083
- Add a feature flag for testing to disable strict go.mod validation
- Document why case en/decoding is necessary
- Refactor pseudo-version processing
  - Move logic from VersionFinder to ModuleHelpers
  - Document reasoning for matching and validation
- Replace BasicAuthHelper and custom method with route setting
- Use serach_files_by_name instead of tree to improve performance
- Use correct content type for zip: closes #214876
parent 4b568ff0
......@@ -56,7 +56,6 @@ Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/213770) for de
### Fetch modules from private projects
NOTE: **Note:**
`go` does not support transmitting credentials over insecure connections. The
steps below work only if GitLab is configured for HTTPS.
......
......@@ -5,29 +5,31 @@ module Packages
class ModuleFinder
include ::API::Helpers::Packages::Go::ModuleHelpers
GITLAB_GO_URL = (Settings.build_gitlab_go_url + '/').freeze
attr_reader :project, :module_name
def initialize(project, module_name)
module_name = CGI.unescape(module_name)
module_name = Pathname.new(module_name).cleanpath.to_s
@project = project
@module_name = module_name
end
# rubocop: disable CodeReuse/ActiveRecord
def execute
return if @module_name.blank? || !@module_name.start_with?(GITLAB_GO_URL)
return if @module_name.blank? || !@module_name.start_with?(gitlab_go_url)
module_path = @module_name[GITLAB_GO_URL.length..].split('/')
module_path = @module_name[gitlab_go_url.length..].split('/')
project_path = project.full_path.split('/')
return unless module_path.take(project_path.length) == project_path
module_project_path = module_path.shift(project_path.length)
return unless module_project_path == project_path
Packages::GoModule.new(@project, @module_name, module_path.join('/'))
end
private
Packages::GoModule.new(@project, @module_name, module_path.drop(project_path.length).join('/'))
def gitlab_go_url
@gitlab_go_url ||= Settings.build_gitlab_go_url + '/'
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
# frozen_string_literal: true
class Packages::GoModule
include Gitlab::Utils::StrongMemoize
attr_reader :project, :name, :path
def initialize(project, name, path)
......@@ -10,7 +12,7 @@ class Packages::GoModule
end
def versions
@versions ||= Packages::Go::VersionFinder.new(self).execute
strong_memoize(:versions) { Packages::Go::VersionFinder.new(self).execute }
end
def version_by(ref: nil, commit: nil)
......@@ -40,6 +42,10 @@ class Packages::GoModule
end
def gomod_valid?(gomod)
if Feature.enabled?(:go_proxy_disable_gomod_validation, @project)
return gomod&.start_with?("module ")
end
gomod&.split("\n", 2)&.first == "module #{@name}"
end
......@@ -47,8 +53,8 @@ class Packages::GoModule
def version_by_name(name)
# avoid a Gitaly call if possible
if defined?(@versions)
v = @versions.find { |v| v.name == ref }
if strong_memoized?(:versions)
v = versions.find { |v| v.name == ref }
return v if v
end
......@@ -60,8 +66,8 @@ class Packages::GoModule
def version_by_ref(ref)
# reuse existing versions
if defined?(@versions)
v = @versions.find { |v| v.ref == ref }
if strong_memoized?(:versions)
v = versions.find { |v| v.ref == ref }
return v if v
end
......
# frozen_string_literal: true
class Packages::GoModuleVersion
include Gitlab::Utils::StrongMemoize
include ::API::Helpers::Packages::Go::ModuleHelpers
VALID_TYPES = %i[ref commit pseudo].freeze
......@@ -42,14 +43,15 @@ class Packages::GoModuleVersion
end
def gomod
@gomod ||=
if defined?(@blobs)
strong_memoize(:gomod) do
if strong_memoized?(:blobs)
blob_at(@mod.path + '/go.mod')
elsif @mod.path.empty?
@mod.project.repository.blob_at(@commit.sha, 'go.mod')&.data
else
@mod.project.repository.blob_at(@commit.sha, @mod.path + '/go.mod')&.data
end
end
end
def archive
......@@ -57,12 +59,26 @@ class Packages::GoModuleVersion
Zip::OutputStream.write_buffer do |zip|
files.each do |file|
zip.put_next_entry "#{full_name}/#{file.path[suffix_len...]}"
zip.write blob_at(file.path)
zip.put_next_entry "#{full_name}/#{file[suffix_len...]}"
zip.write blob_at(file)
end
end
end
def files
strong_memoize(:files) do
ls_tree.filter { |e| !excluded.any? { |n| e.start_with? n } }
end
end
def excluded
strong_memoize(:excluded) do
ls_tree
.filter { |f| f.end_with?('/go.mod') && f != @mod.path + '/go.mod' }
.map { |f| f[0..-7] }
end
end
def valid?
@mod.path_valid?(major) && @mod.gomod_valid?(gomod)
end
......@@ -78,15 +94,19 @@ class Packages::GoModuleVersion
end
def blobs
@blobs ||= @mod.project.repository.batch_blobs(files.map { |x| [@commit.sha, x.path] })
strong_memoize(:blobs) { @mod.project.repository.batch_blobs(files.map { |x| [@commit.sha, x] }) }
end
def files
return @files if defined?(@files)
def ls_tree
strong_memoize(:ls_tree) do
path =
if @mod.path.empty?
'.'
else
@mod.path
end
sha = @commit.sha
tree = @mod.project.repository.tree(sha, @mod.path, recursive: true).entries.filter { |e| e.file? }
nested = tree.filter { |e| e.name == 'go.mod' && !(@mod.path == '' && e.path == 'go.mod' || e.path == @mod.path + '/go.mod') }.map { |e| e.path[0..-7] }
@files = tree.filter { |e| !nested.any? { |n| e.path.start_with? n } }
@mod.project.repository.gitaly_repository_client.search_files_by_name(@commit.sha, path)
end
end
end
# frozen_string_literal: true
class Packages::SemVer
# basic semver, but bounded (^expr$)
PATTERN = /\A(v?)(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([-.a-z0-9]+))?(?:\+([-.a-z0-9]+))?\z/i.freeze
attr_accessor :major, :minor, :patch, :prerelease, :build
def initialize(major = 0, minor = 0, patch = 0, prerelease = nil, build = nil, prefixed: false)
......@@ -37,11 +34,11 @@ class Packages::SemVer
end
def self.match(str, prefixed: false)
m = PATTERN.match(str)
return unless m
return if prefixed == m[1].empty?
return unless str&.start_with?('v') == prefixed
str = str[1..] if prefixed
m
Gitlab::Regex.semver_regex.match(str)
end
def self.match?(str, prefixed: false)
......@@ -52,6 +49,6 @@ class Packages::SemVer
m = match str, prefixed: prefixed
return unless m
new(m[2].to_i, m[3].to_i, m[4].to_i, m[5], m[6], prefixed: prefixed)
new(m[1].to_i, m[2].to_i, m[3].to_i, m[4], m[5], prefixed: prefixed)
end
end
# frozen_string_literal: true
module API
class GoProxy < Grape::API
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
helpers ::API::Helpers::PackagesHelpers
helpers ::API::Helpers::Packages::Go::ModuleHelpers
# basic semver, except case encoded (A => !a)
......@@ -13,27 +12,27 @@ module API
before { require_packages_enabled! }
helpers do
# support personal access tokens for HTTP Basic in addition to the usual methods
def find_personal_access_token
pa = find_personal_access_token_from_http_basic_auth
return pa if pa
# copied from Gitlab::Auth::AuthFinders
token =
current_request.params[::Gitlab::Auth::AuthFinders::PRIVATE_TOKEN_PARAM].presence ||
current_request.env[::Gitlab::Auth::AuthFinders::PRIVATE_TOKEN_HEADER].presence ||
parsed_oauth_token
return unless token
# Expiration, revocation and scopes are verified in `validate_access_token!`
PersonalAccessToken.find_by_token(token) || raise(::Gitlab::Auth::UnauthorizedError)
def find_project!(id)
# based on API::Helpers::Packages::BasicAuthHelpers#authorized_project_find!
project = find_project(id)
return project if project && can?(current_user, :read_project, project)
if current_user
not_found!('Project')
else
unauthorized!
end
end
def find_module
not_found! unless Feature.enabled?(:go_proxy, user_project)
module_name = case_decode params[:module_name]
bad_request!('Module Name') if module_name.blank?
mod = ::Packages::Go::ModuleFinder.new(authorized_user_project, module_name).execute
mod = ::Packages::Go::ModuleFinder.new(user_project, module_name).execute
not_found! unless mod
......@@ -55,13 +54,13 @@ module API
params do
requires :id, type: String, desc: 'The ID of a project'
requires :module_name, type: String, desc: 'Module name'
requires :module_name, type: String, desc: 'Module name', coerce_with: ->(val) { CGI.unescape(val) }
end
route_setting :authentication, job_token_allowed: true
route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
before do
authorize_read_package!(authorized_user_project)
authorize_packages_feature!(authorized_user_project)
authorize_read_package!
authorize_packages_feature!
end
namespace ':id/packages/go/*module_name/@v' do
......@@ -110,10 +109,10 @@ module API
get ':module_version.zip', requirements: MODULE_VERSION_REQUIREMENTS do
ver = find_version
# TODO: Content-Type should be application/zip, see #214876
content_type 'application/zip'
env['api.format'] = :binary
header['Content-Disposition'] = ActionDispatch::Http::ContentDisposition.format(disposition: 'attachment', filename: ver.name + '.zip')
header['Content-Transfer-Encoding'] = 'binary'
content_type 'text/plain'
status :ok
body ver.archive.string
end
......
......@@ -6,10 +6,24 @@ module API
module Go
module ModuleHelpers
def case_encode(str)
# Converts "github.com/Azure" to "github.com/!azure"
#
# From `go help goproxy`:
#
# > To avoid problems when serving from case-sensitive file systems,
# > the <module> and <version> elements are case-encoded, replacing
# > every uppercase letter with an exclamation mark followed by the
# > corresponding lower-case letter: github.com/Azure encodes as
# > github.com/!azure.
str.gsub(/A-Z/) { |s| "!#{s.downcase}"}
end
def case_decode(str)
# Converts "github.com/!azure" to "github.com/Azure"
#
# See #case_encode
str.gsub(/![[:alpha:]]/) { |s| s[1..].upcase }
end
......@@ -29,7 +43,7 @@ module API
pre = version.prerelease
# valid pseudo-versions are
# Valid pseudo-versions are:
# vX.0.0-yyyymmddhhmmss-sha1337beef0, when no earlier tagged commit exists for X
# vX.Y.Z-pre.0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z-pre
# vX.Y.(Z+1)-0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z
......@@ -41,7 +55,41 @@ module API
pre = pre[m[0].length..]
end
/\A\d{14}-[A-Za-z0-9]+\z/.freeze.match? pre
# This pattern is intentionally more forgiving than the patterns
# above. Correctness is verified by #pseudo_version_commit.
/\A\d{14}-\h+\z/.freeze.match? pre
end
def pseudo_version_commit(project, semver)
# Per Go's implementation of pseudo-versions, a tag should be
# considered a pseudo-version if it matches one of the patterns
# listed in #pseudo_version?, regardless of the content of the
# timestamp or the length of the SHA fragment. However, an error
# should be returned if the timestamp is not correct or if the SHA
# fragment is not exactly 12 characters long. See also Go's
# implementation of:
#
# - [*codeRepo.validatePseudoVersion](https://github.com/golang/go/blob/daf70d6c1688a1ba1699c933b3c3f04d6f2f73d9/src/cmd/go/internal/modfetch/coderepo.go#L530)
# - [Pseudo-version parsing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/pseudo.go)
# - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go)
# Go ignores anything before '.' or after the second '-', so we will do the same
timestamp, sha = semver.prerelease.split('-').last 2
timestamp = timestamp.split('.').last
commit = project.repository.commit_by(oid: sha)
# Error messages are based on the responses of proxy.golang.org
# Verify that the SHA fragment references a commit
raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit
# Require the SHA fragment to be 12 characters long
raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12
# Require the timestamp to match that of the commit
raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp
commit
end
def parse_semver(str)
......
......@@ -75,21 +75,27 @@ FactoryBot.define do
transient do
name { nil }
message { 'Add module' }
end
service do
port = ::Gitlab.config.gitlab.port
host = ::Gitlab.config.gitlab.host
domain = case port when 80, 443 then host else "#{host}:#{port}" end
url = "#{domain}/#{project.path_with_namespace}"
if name.nil?
path = ''
else
url += '/' + name
path = name + '/'
url do
v = "#{::Gitlab.config.gitlab.host}/#{project.path_with_namespace}"
if name
v + '/' + name
else
v
end
end
path do
if name
name + '/'
else
''
end
end
end
service do
Files::MultiService.new(
project,
project.owner,
......
......@@ -58,13 +58,6 @@ describe Packages::Go::ModuleFinder do
end
end
context 'with a URL encoded relative path component' do
it_behaves_like 'an invalid path' do
let(:module_name) { base_url(project) + '/%2E%2E%2Fxyz' }
let(:expected_name) { base_url(project.namespace) + '/xyz' }
end
end
context 'with many relative path components' do
it_behaves_like 'an invalid path' do
let(:module_name) { base_url(project) + ('/..' * 10) + '/xyz' }
......
......@@ -13,11 +13,16 @@ describe Packages::Go::VersionFinder do
create :go_module_commit, :module, project: project, tag: 'v1.0.1'
create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg'
create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod'
create :go_module_commit, :module, project: project, tag: 'v1.0.4', name: 'bad-mod', url: 'example.com/go-lib'
create :go_module_commit, :files, project: project, tag: 'c1', files: { 'y.go' => "package a\n" }
create :go_module_commit, :module, project: project, tag: 'c2', name: 'v2'
create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" }
end
before do
stub_feature_flags(go_proxy_disable_gomod_validation: false)
end
shared_examples '#execute' do |*expected|
it "returns #{expected.empty? ? 'nothing' : expected.join(', ')}" do
actual = finder.execute.map { |x| x.name }
......@@ -35,7 +40,7 @@ describe Packages::Go::VersionFinder do
context 'for the root module' do
let(:mod) { create :go_module, project: project }
it_behaves_like '#execute', 'v1.0.1', 'v1.0.2', 'v1.0.3'
it_behaves_like '#execute', 'v1.0.1', 'v1.0.2', 'v1.0.3', 'v1.0.4'
end
context 'for the package' do
......@@ -47,7 +52,7 @@ describe Packages::Go::VersionFinder do
context 'for the submodule' do
let(:mod) { create :go_module, project: project, path: 'mod' }
it_behaves_like '#execute', 'v1.0.3'
it_behaves_like '#execute', 'v1.0.3', 'v1.0.4'
end
context 'for the root module v2' do
......@@ -55,6 +60,22 @@ describe Packages::Go::VersionFinder do
it_behaves_like '#execute', 'v2.0.0'
end
context 'for the bad module' do
let(:mod) { create :go_module, project: project, path: 'bad-mod' }
context 'with gomod checking enabled' do
it_behaves_like '#execute'
end
context 'with gomod checking disabled' do
before do
stub_feature_flags(go_proxy_disable_gomod_validation: true)
end
it_behaves_like '#execute', 'v1.0.4'
end
end
end
describe '#find' do
......
......@@ -3,6 +3,10 @@
require 'spec_helper'
describe Packages::GoModule, type: :model do
before do
stub_feature_flags(go_proxy_disable_gomod_validation: false)
end
describe '#path_valid?' do
context 'with root path' do
let_it_be(:package) { create(:go_module) }
......
......@@ -19,7 +19,7 @@ describe Packages::GoModuleVersion, type: :model do
shared_examples '#files' do |desc, *entries|
it "returns #{desc}" do
actual = version.files.map { |x| x.path }.to_set
actual = version.files.map { |x| x }.to_set
expect(actual).to eq(entries.to_set)
end
end
......
......@@ -29,7 +29,9 @@ describe API::GoProxy do
before do
project.add_developer(user)
stub_licensed_features(packages: true)
stub_feature_flags(go_proxy_disable_gomod_validation: false)
modules
end
......@@ -54,7 +56,7 @@ describe API::GoProxy do
end
end
shared_examples 'a missing module version list resource' do |*versions, path: ''|
shared_examples 'a missing module version list resource' do |path: ''|
let(:module_name) { "#{base}#{path}" }
let(:resource) { "list" }
......@@ -168,6 +170,18 @@ describe API::GoProxy do
context 'for the root module v2' do
it_behaves_like 'a module version list resource', 'v2.0.0', path: '/v2'
end
context 'with a URL encoded relative path component' do
it_behaves_like 'a missing module version list resource', path: '/%2E%2E%2Fxyz'
end
context 'with the feature disabled' do
before do
stub_feature_flags(go_proxy: false)
end
it_behaves_like 'a missing module version list resource'
end
end
describe 'GET /projects/:id/packages/go/*module_name/@v/:module_version.info' do
......@@ -356,26 +370,31 @@ describe API::GoProxy do
it 'returns ok with an oauth token' do
get_resource(oauth_access_token: oauth)
expect(response).to have_gitlab_http_status(:ok)
end
it 'returns ok with a job token' do
get_resource(oauth_access_token: job)
expect(response).to have_gitlab_http_status(:ok)
end
it 'returns ok with a personal access token' do
get_resource(personal_access_token: pa_token)
expect(response).to have_gitlab_http_status(:ok)
end
it 'returns ok with a personal access token and basic authentication' do
get_resource(headers: build_basic_auth_header(user.username, pa_token.token))
expect(response).to have_gitlab_http_status(:ok)
end
it 'returns unauthorized with no authentication' do
get_resource
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
......@@ -393,6 +412,7 @@ describe API::GoProxy do
it 'returns ok with no authentication' do
get_resource
expect(response).to have_gitlab_http_status(:ok)
end
end
......@@ -406,26 +426,31 @@ describe API::GoProxy do
describe 'GET /projects/:id/packages/go/*module_name/@v/list' do
it 'returns not found with a user' do
get_resource(user)
expect(response).to have_gitlab_http_status(:not_found)
end
it 'returns not found with an oauth token' do
get_resource(oauth_access_token: oauth)
expect(response).to have_gitlab_http_status(:not_found)
end
it 'returns not found with a job token' do
get_resource(oauth_access_token: job)
expect(response).to have_gitlab_http_status(:not_found)
end
it 'returns not found with a personal access token' do
get_resource(personal_access_token: pa_token)
expect(response).to have_gitlab_http_status(:not_found)
end
it 'returns unauthorized with no authentication' do
get_resource
expect(response).to have_gitlab_http_status(:unauthorized)
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment