Commit f3b5cbc9 authored by David Fernandez's avatar David Fernandez

Merge branch '220628-go-proxy-packages' into 'master'

Create Packages from Go module versions

See merge request gitlab-org/gitlab!34558
parents a239b168 d42d5244
# frozen_string_literal: true
module Packages
module Go
class PackageFinder
delegate :exists?, to: :candidates
def initialize(project, module_name, module_version)
@project = project
@module_name = module_name
@module_version = module_version
end
def execute
candidates.first
end
private
def candidates
@project
.packages
.golang
.with_name(@module_name)
.with_version(@module_version)
end
end
end
end
...@@ -23,7 +23,8 @@ module Packages ...@@ -23,7 +23,8 @@ module Packages
when String when String
if pseudo_version? target if pseudo_version? target
semver = parse_semver(target) semver = parse_semver(target)
commit = pseudo_version_commit(@mod.project, semver) version = parse_pseudo_version(semver)
commit = validate_pseudo_version(@mod.project, version)
Packages::Go::ModuleVersion.new(@mod, :pseudo, commit, name: target, semver: semver) Packages::Go::ModuleVersion.new(@mod, :pseudo, commit, name: target, semver: semver)
else else
@mod.version_by(ref: target) @mod.version_by(ref: target)
......
...@@ -4,6 +4,7 @@ module Packages ...@@ -4,6 +4,7 @@ module Packages
module Go module Go
class ModuleVersion class ModuleVersion
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
include Gitlab::Golang
VALID_TYPES = %i[ref commit pseudo].freeze VALID_TYPES = %i[ref commit pseudo].freeze
...@@ -81,6 +82,9 @@ module Packages ...@@ -81,6 +82,9 @@ module Packages
end end
def valid? def valid?
# assume the module version is valid if a corresponding Package exists
return true if ::Packages::Go::PackageFinder.new(mod.project, mod.name, name).exists?
@mod.path_valid?(major) && @mod.gomod_valid?(gomod) @mod.path_valid?(major) && @mod.gomod_valid?(gomod)
end end
......
# frozen_string_literal: true
module Packages
module Go
class CreatePackageService < BaseService
GoZipSizeError = Class.new(StandardError)
attr_accessor :version
def initialize(project, user = nil, version:)
super(project, user)
@version = version
end
def execute
# check for existing package to avoid SQL errors due to the index
package = ::Packages::Go::PackageFinder.new(version.mod.project, version.mod.name, version.name).execute
return package if package
# this can be expensive, so do it outside the transaction
files = {}
files[:mod] = prepare_file(version, :mod, version.gomod)
files[:zip] = prepare_file(version, :zip, version.archive.string)
ActiveRecord::Base.transaction do
# create new package and files
package = create_package
files.each { |type, (file, digests)| create_file(package, type, file, digests) }
package
end
end
private
def prepare_file(version, type, content)
file = CarrierWaveStringFile.new(content)
raise GoZipSizeError, "#{version.mod.name}@#{version.name}.#{type} exceeds size limit" if file.size > project.actual_limits.golang_max_file_size
digests = {
md5: Digest::MD5.hexdigest(content),
sha1: Digest::SHA1.hexdigest(content),
sha256: Digest::SHA256.hexdigest(content)
}
[file, digests]
end
def create_package
version.mod.project.packages.create!(
name: version.mod.name,
version: version.name,
package_type: :golang,
created_at: version.commit.committed_date
)
end
def create_file(package, type, file, digests)
CreatePackageFileService.new(package,
file: file,
size: file.size,
file_name: "#{version.name}.#{type}",
file_md5: digests[:md5],
file_sha1: digests[:sha1],
file_sha256: digests[:sha256]
).execute
end
end
end
end
# frozen_string_literal: true
module Packages
module Go
class SyncPackagesService < BaseService
include Gitlab::Golang
def initialize(project, ref, path = '')
super(project)
@ref = ref
@path = path
raise ArgumentError, 'project is required' unless project
raise ArgumentError, 'ref is required' unless ref
raise ArgumentError, "ref #{ref} not found" unless project.repository.find_tag(ref) || project.repository.find_branch(ref)
end
def execute_async
Packages::Go::SyncPackagesWorker.perform_async(project.id, @ref, @path)
end
end
end
end
...@@ -1083,6 +1083,14 @@ ...@@ -1083,6 +1083,14 @@
:weight: 1 :weight: 1
:idempotent: :idempotent:
:tags: [] :tags: []
- :name: package_repositories:packages_go_sync_packages
:feature_category: :package_registry
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: package_repositories:packages_maven_metadata_sync - :name: package_repositories:packages_maven_metadata_sync
:feature_category: :package_registry :feature_category: :package_registry
:has_external_dependencies: :has_external_dependencies:
......
# frozen_string_literal: true
module Packages
module Go
class SyncPackagesWorker
include ApplicationWorker
include Gitlab::Golang
queue_namespace :package_repositories
feature_category :package_registry
deduplicate :until_executing
idempotent!
def perform(project_id, ref_name, path)
project = Project.find_by_id(project_id)
return unless project && project.repository.find_tag(ref_name)
module_name = go_path(project, path)
mod = Packages::Go::ModuleFinder.new(project, module_name).execute
return unless mod
ver = Packages::Go::VersionFinder.new(mod).find(ref_name)
return unless ver
Packages::Go::CreatePackageService.new(project, nil, version: ver).execute
rescue ::Packages::Go::CreatePackageService::GoZipSizeError => ex
Gitlab::ErrorTracking.log_exception(ex)
end
end
end
end
---
title: Add Go Packages as a cache for the Go proxy
merge_request: 34558
author: Ethan Reesor (@firelizzard)
type: added
# frozen_string_literal: true
class AddUniqueIndexForGolangPackages < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_packages_on_project_id_name_version_unique_when_golang'
PACKAGE_TYPE_GOLANG = 8
disable_ddl_transaction!
def up
add_concurrent_index :packages_packages, [:project_id, :name, :version], unique: true, where: "package_type = #{PACKAGE_TYPE_GOLANG}", name: INDEX_NAME
end
def down
remove_concurrent_index_by_name(:packages_packages, INDEX_NAME)
end
end
f4c81be1168dc8dc3eaadbc9b0d46cfd5aefa0b9e4d61fa8276bbc4f59216da8
\ No newline at end of file
...@@ -23294,6 +23294,8 @@ CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_ ...@@ -23294,6 +23294,8 @@ CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_
CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_generic ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 7); CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_generic ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 7);
CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_golang ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 8);
CREATE INDEX index_packages_package_file_build_infos_on_package_file_id ON packages_package_file_build_infos USING btree (package_file_id); CREATE INDEX index_packages_package_file_build_infos_on_package_file_id ON packages_package_file_build_infos USING btree (package_file_id);
CREATE INDEX index_packages_package_file_build_infos_on_pipeline_id ON packages_package_file_build_infos USING btree (pipeline_id); CREATE INDEX index_packages_package_file_build_infos_on_pipeline_id ON packages_package_file_build_infos USING btree (pipeline_id);
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
module Gitlab module Gitlab
module Golang module Golang
PseudoVersion = Struct.new(:semver, :timestamp, :commit_id)
extend self extend self
def local_module_prefix def local_module_prefix
...@@ -37,11 +39,11 @@ module Gitlab ...@@ -37,11 +39,11 @@ module Gitlab
end end
# This pattern is intentionally more forgiving than the patterns # This pattern is intentionally more forgiving than the patterns
# above. Correctness is verified by #pseudo_version_commit. # above. Correctness is verified by #validate_pseudo_version.
/\A\d{14}-\h+\z/.freeze.match? pre /\A\d{14}-\h+\z/.freeze.match? pre
end end
def pseudo_version_commit(project, semver) def parse_pseudo_version(semver)
# Per Go's implementation of pseudo-versions, a tag should be # Per Go's implementation of pseudo-versions, a tag should be
# considered a pseudo-version if it matches one of the patterns # considered a pseudo-version if it matches one of the patterns
# listed in #pseudo_version?, regardless of the content of the # listed in #pseudo_version?, regardless of the content of the
...@@ -55,9 +57,14 @@ module Gitlab ...@@ -55,9 +57,14 @@ module Gitlab
# - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go) # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go)
# Go ignores anything before '.' or after the second '-', so we will do the same # Go ignores anything before '.' or after the second '-', so we will do the same
timestamp, sha = semver.prerelease.split('-').last 2 timestamp, commit_id = semver.prerelease.split('-').last 2
timestamp = timestamp.split('.').last timestamp = timestamp.split('.').last
commit = project.repository.commit_by(oid: sha)
PseudoVersion.new(semver, timestamp, commit_id)
end
def validate_pseudo_version(project, version, commit = nil)
commit ||= project.repository.commit_by(oid: version.commit_id)
# Error messages are based on the responses of proxy.golang.org # Error messages are based on the responses of proxy.golang.org
...@@ -65,10 +72,10 @@ module Gitlab ...@@ -65,10 +72,10 @@ module Gitlab
raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit
# Require the SHA fragment to be 12 characters long # Require the SHA fragment to be 12 characters long
raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12 raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12
# Require the timestamp to match that of the commit # Require the timestamp to match that of the commit
raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp
commit commit
end end
...@@ -77,6 +84,14 @@ module Gitlab ...@@ -77,6 +84,14 @@ module Gitlab
Packages::SemVer.parse(str, prefixed: true) Packages::SemVer.parse(str, prefixed: true)
end end
def go_path(project, path = nil)
if path.blank?
"#{local_module_prefix}/#{project.full_path}"
else
"#{local_module_prefix}/#{project.full_path}/#{path}"
end
end
def pkg_go_dev_url(name, version = nil) def pkg_go_dev_url(name, version = nil)
if version if version
"https://pkg.go.dev/#{name}@#{version}" "https://pkg.go.dev/#{name}@#{version}"
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Go::PackageFinder do
include_context 'basic Go module'
let_it_be(:mod) { create :go_module, project: project }
let_it_be(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.1' }
let_it_be(:package) { create :golang_package, project: project, name: mod.name, version: 'v1.0.1' }
let(:finder) { described_class.new(project, mod_name, version_name) }
describe '#exists?' do
subject { finder.exists? }
context 'with a valid name and version' do
let(:mod_name) { mod.name }
let(:version_name) { version.name }
it 'executes SELECT 1' do
expect { subject }.to exceed_query_limit(0).for_query(/^SELECT 1/)
end
it { is_expected.to eq(true) }
end
context 'with an invalid name' do
let(:mod_name) { 'foo/bar' }
let(:version_name) { 'baz' }
it { is_expected.to eq(false) }
end
context 'with an invalid version' do
let(:mod_name) { mod.name }
let(:version_name) { 'baz' }
it { is_expected.to eq(false) }
end
end
describe '#execute' do
subject { finder.execute }
context 'with a valid name and version' do
let(:mod_name) { mod.name }
let(:version_name) { version.name }
it 'executes a single query' do
expect { subject }.not_to exceed_query_limit(1)
end
it { is_expected.to eq(package) }
end
context 'with an invalid name' do
let(:mod_name) { 'foo/bar' }
let(:version_name) { 'baz' }
it { is_expected.to eq(nil) }
end
context 'with an invalid version' do
let(:mod_name) { mod.name }
let(:version_name) { 'baz' }
it { is_expected.to eq(nil) }
end
end
end
...@@ -3,19 +3,9 @@ ...@@ -3,19 +3,9 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Packages::Go::ModuleVersion, type: :model do RSpec.describe Packages::Go::ModuleVersion, type: :model do
let_it_be(:user) { create :user } include_context 'basic Go module'
let_it_be(:project) { create :project_empty_repo, creator: user, path: 'my-go-lib' }
let_it_be(:mod) { create :go_module, project: project }
before :all do let_it_be(:mod) { create :go_module, project: project }
create :go_module_commit, :files, project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' }
create :go_module_commit, :module, project: project, tag: 'v1.0.1'
create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg'
create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod'
create :go_module_commit, :files, project: project, files: { 'y.go' => "package a\n" }
create :go_module_commit, :module, project: project, name: 'v2'
create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" }
end
shared_examples '#files' do |desc, *entries| shared_examples '#files' do |desc, *entries|
it "returns #{desc}" do it "returns #{desc}" do
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Go::CreatePackageService do
let_it_be(:project) { create :project_empty_repo, path: 'my-go-lib' }
let_it_be(:mod) { create :go_module, project: project }
before :all do
create :go_module_commit, :module, project: project, tag: 'v1.0.0'
end
shared_examples 'creates a package' do |files:|
it "returns a valid package with #{files ? files.to_s : 'no'} file(s)" do
expect(subject).to be_valid
expect(subject.name).to eq(version.mod.name)
expect(subject.version).to eq(version.name)
expect(subject.package_type).to eq('golang')
expect(subject.created_at).to eq(version.commit.committed_date)
expect(subject.package_files.count).to eq(files)
end
end
shared_examples 'creates a package file' do |type|
it "returns a package with a #{type} file" do
file_name = "#{version.name}.#{type}"
expect(subject.package_files.map { |f| f.file_name }).to include(file_name)
file = subject.package_files.with_file_name(file_name).first
expect(file).not_to be_nil
expect(file.file).not_to be_nil
expect(file.size).to eq(file.file.size)
expect(file.file_name).to eq(file_name)
expect(file.file_md5).not_to be_nil
expect(file.file_sha1).not_to be_nil
expect(file.file_sha256).not_to be_nil
end
end
describe '#execute' do
subject { described_class.new(project, nil, version: version).execute }
let(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.0' }
context 'with no existing package' do
it_behaves_like 'creates a package', files: 2
it_behaves_like 'creates a package file', :mod
it_behaves_like 'creates a package file', :zip
it 'creates a new package' do
expect { subject }
.to change { project.packages.count }.by(1)
.and change { Packages::PackageFile.count }.by(2)
end
end
context 'with an existing package' do
before do
described_class.new(project, version: version).execute
end
it_behaves_like 'creates a package', files: 2
it_behaves_like 'creates a package file', :mod
it_behaves_like 'creates a package file', :zip
it 'does not create a package or files' do
expect { subject }
.to not_change { project.packages.count }
.and not_change { Packages::PackageFile.count }
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Go::SyncPackagesService do
include_context 'basic Go module'
let(:params) { { info: true, mod: true, zip: true } }
describe '#execute_async' do
it 'schedules a package refresh' do
expect(::Packages::Go::SyncPackagesWorker).to receive(:perform_async).once
described_class.new(project, 'master').execute_async
end
end
describe '#initialize' do
context 'without a project' do
it 'raises an error' do
expect { described_class.new(nil, 'master') }
.to raise_error(ArgumentError, 'project is required')
end
end
context 'without a ref' do
it 'raises an error' do
expect { described_class.new(project, nil) }
.to raise_error(ArgumentError, 'ref is required')
end
end
context 'with an invalid ref' do
it 'raises an error' do
expect { described_class.new(project, 'not-a-ref') }
.to raise_error(ArgumentError)
end
end
end
end
# frozen_string_literal: true
RSpec.shared_context 'basic Go module' do
let_it_be(:user) { create :user }
let_it_be(:project) { create :project_empty_repo, creator: user, path: 'my-go-lib' }
let_it_be(:commit_v1_0_0) { create :go_module_commit, :files, project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' } }
let_it_be(:commit_v1_0_1) { create :go_module_commit, :module, project: project, tag: 'v1.0.1' }
let_it_be(:commit_v1_0_2) { create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg' }
let_it_be(:commit_v1_0_3) { create :go_module_commit, :module, project: project, tag: 'v1.0.3', name: 'mod' }
let_it_be(:commit_file_y) { create :go_module_commit, :files, project: project, files: { 'y.go' => "package a\n" } }
let_it_be(:commit_mod_v2) { create :go_module_commit, :module, project: project, name: 'v2' }
let_it_be(:commit_v2_0_0) { create :go_module_commit, :files, project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" } }
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Packages::Go::SyncPackagesWorker, type: :worker do
include_context 'basic Go module'
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PUBLIC)
end
def perform(ref_name, path)
described_class.new.perform(project.id, ref_name, path)
end
def validate_package(package, mod, ver)
expect(package).not_to be_nil
expect(package.name).to eq(mod.name)
expect(package.version).to eq(ver.name)
expect(package.package_type).to eq('golang')
expect(package.created_at).to eq(ver.commit.committed_date)
expect(package.package_files.count).to eq(2)
end
shared_examples 'it creates a package' do |path, version, exists: false|
subject { perform(version, path) }
it "returns a package for example.com/project#{path.empty? ? '' : '/' + path}@#{version}" do
expect { subject }
.to change { project.packages.count }.by(exists ? 0 : 1)
.and change { Packages::PackageFile.count }.by(exists ? 0 : 2)
mod = create :go_module, project: project, path: path
ver = create :go_module_version, :tagged, mod: mod, name: version
validate_package(subject, mod, ver)
end
end
describe '#perform' do
context 'with no existing packages' do
it_behaves_like 'it creates a package', '', 'v1.0.1'
it_behaves_like 'it creates a package', '', 'v1.0.2'
it_behaves_like 'it creates a package', '', 'v1.0.3'
it_behaves_like 'it creates a package', 'mod', 'v1.0.3'
it_behaves_like 'it creates a package', 'v2', 'v2.0.0'
end
context 'with existing packages' do
before do
mod = create :go_module, project: project
ver = create :go_module_version, :tagged, mod: mod, name: 'v1.0.1'
Packages::Go::CreatePackageService.new(project, nil, version: ver).execute
end
it_behaves_like 'it creates a package', '', 'v1.0.1', exists: true
it_behaves_like 'it creates a package', '', 'v1.0.2'
it_behaves_like 'it creates a package', '', 'v1.0.3'
it_behaves_like 'it creates a package', 'mod', 'v1.0.3'
it_behaves_like 'it creates a package', 'v2', 'v2.0.0'
end
context 'with a package that exceeds project limits' do
before do
Plan.default.actual_limits.update!({ 'golang_max_file_size': 1 })
end
it 'logs an exception' do
expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(::Packages::Go::CreatePackageService::GoZipSizeError))
perform('v2.0.0', 'v2')
end
end
where(:path, :version) do
[
['', 'v1.0.1'],
['', 'v1.0.2'],
['', 'v1.0.3'],
['mod', 'v1.0.3'],
['v2', 'v2.0.0']
]
end
with_them do
it_behaves_like 'an idempotent worker' do
let(:job_args) { [project.id, version, path] }
it 'creates a package' do
expect { subject }
.to change { project.packages.count }.by(1)
.and change { Packages::PackageFile.count }.by(2)
mod = create :go_module, project: project, path: path
ver = create :go_module_version, :tagged, mod: mod, name: version
package = ::Packages::Go::PackageFinder.new(project, mod.name, ver.name).execute
validate_package(package, mod, ver)
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment