Merge branch '220628-go-proxy-packages' into 'master'

Create Packages from Go module versions

See merge request gitlab-org/gitlab!34558

app/finders/packages/go/package_finder.rb

+# frozen_string_literal: true
+
+module Packages
+  module Go
+    class PackageFinder
+      delegate :exists?, to: :candidates
+
+      def initialize(project, module_name, module_version)
+        @project = project
+        @module_name = module_name
+        @module_version = module_version
+      end
+
+      def execute
+        candidates.first
+      end
+
+      private
+
+      def candidates
+        @project
+          .packages
+          .golang
+          .with_name(@module_name)
+          .with_version(@module_version)
+      end
+    end
+  end
+end

app/finders/packages/go/version_finder.rb

@@ -23,7 +23,8 @@ module Packages
         when String
           if pseudo_version? target
             semver = parse_semver(target)
-            commit = pseudo_version_commit(@mod.project, semver)
+            version = parse_pseudo_version(semver)
+            commit = validate_pseudo_version(@mod.project, version)
             Packages::Go::ModuleVersion.new(@mod, :pseudo, commit, name: target, semver: semver)
           else
             @mod.version_by(ref: target)

app/models/packages/go/module_version.rb

@@ -4,6 +4,7 @@ module Packages
   module Go
     class ModuleVersion
       include Gitlab::Utils::StrongMemoize
+      include Gitlab::Golang
 
       VALID_TYPES = %i[ref commit pseudo].freeze
 
@@ -81,6 +82,9 @@ module Packages
       end
 
       def valid?
+        # assume the module version is valid if a corresponding Package exists
+        return true if ::Packages::Go::PackageFinder.new(mod.project, mod.name, name).exists?
+
         @mod.path_valid?(major) && @mod.gomod_valid?(gomod)
       end
 

app/services/packages/go/create_package_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Go
+    class CreatePackageService < BaseService
+      GoZipSizeError = Class.new(StandardError)
+
+      attr_accessor :version
+
+      def initialize(project, user = nil, version:)
+        super(project, user)
+
+        @version = version
+      end
+
+      def execute
+        # check for existing package to avoid SQL errors due to the index
+        package = ::Packages::Go::PackageFinder.new(version.mod.project, version.mod.name, version.name).execute
+        return package if package
+
+        # this can be expensive, so do it outside the transaction
+        files = {}
+        files[:mod] = prepare_file(version, :mod, version.gomod)
+        files[:zip] = prepare_file(version, :zip, version.archive.string)
+
+        ActiveRecord::Base.transaction do
+          # create new package and files
+          package = create_package
+          files.each { |type, (file, digests)| create_file(package, type, file, digests) }
+          package
+        end
+      end
+
+      private
+
+      def prepare_file(version, type, content)
+        file = CarrierWaveStringFile.new(content)
+        raise GoZipSizeError, "#{version.mod.name}@#{version.name}.#{type} exceeds size limit" if file.size > project.actual_limits.golang_max_file_size
+
+        digests = {
+          md5: Digest::MD5.hexdigest(content),
+          sha1: Digest::SHA1.hexdigest(content),
+          sha256: Digest::SHA256.hexdigest(content)
+        }
+
+        [file, digests]
+      end
+
+      def create_package
+        version.mod.project.packages.create!(
+          name: version.mod.name,
+          version: version.name,
+          package_type: :golang,
+          created_at: version.commit.committed_date
+        )
+      end
+
+      def create_file(package, type, file, digests)
+        CreatePackageFileService.new(package,
+          file: file,
+          size: file.size,
+          file_name: "#{version.name}.#{type}",
+          file_md5: digests[:md5],
+          file_sha1: digests[:sha1],
+          file_sha256: digests[:sha256]
+        ).execute
+      end
+    end
+  end
+end

app/services/packages/go/sync_packages_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Go
+    class SyncPackagesService < BaseService
+      include Gitlab::Golang
+
+      def initialize(project, ref, path = '')
+        super(project)
+
+        @ref = ref
+        @path = path
+
+        raise ArgumentError, 'project is required' unless project
+        raise ArgumentError, 'ref is required' unless ref
+        raise ArgumentError, "ref #{ref} not found" unless project.repository.find_tag(ref) || project.repository.find_branch(ref)
+      end
+
+      def execute_async
+        Packages::Go::SyncPackagesWorker.perform_async(project.id, @ref, @path)
+      end
+    end
+  end
+end

app/workers/all_queues.yml

@@ -1083,6 +1083,14 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: package_repositories:packages_go_sync_packages
+  :feature_category: :package_registry
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: package_repositories:packages_maven_metadata_sync
   :feature_category: :package_registry
   :has_external_dependencies:

app/workers/packages/go/sync_packages_worker.rb

+# frozen_string_literal: true
+
+module Packages
+  module Go
+    class SyncPackagesWorker
+      include ApplicationWorker
+      include Gitlab::Golang
+
+      queue_namespace :package_repositories
+      feature_category :package_registry
+
+      deduplicate :until_executing
+      idempotent!
+
+      def perform(project_id, ref_name, path)
+        project = Project.find_by_id(project_id)
+        return unless project && project.repository.find_tag(ref_name)
+
+        module_name = go_path(project, path)
+        mod = Packages::Go::ModuleFinder.new(project, module_name).execute
+        return unless mod
+
+        ver = Packages::Go::VersionFinder.new(mod).find(ref_name)
+        return unless ver
+
+        Packages::Go::CreatePackageService.new(project, nil, version: ver).execute
+
+      rescue ::Packages::Go::CreatePackageService::GoZipSizeError => ex
+        Gitlab::ErrorTracking.log_exception(ex)
+      end
+    end
+  end
+end

changelogs/unreleased/220628-go-proxy-packages.yml

+---
+title: Add Go Packages as a cache for the Go proxy
+merge_request: 34558
+author: Ethan Reesor (@firelizzard)
+type: added

db/migrate/20210106061254_add_unique_index_for_golang_packages.rb

+# frozen_string_literal: true
+
+class AddUniqueIndexForGolangPackages < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INDEX_NAME = 'index_packages_on_project_id_name_version_unique_when_golang'
+  PACKAGE_TYPE_GOLANG = 8
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :packages_packages, [:project_id, :name, :version], unique: true, where: "package_type = #{PACKAGE_TYPE_GOLANG}", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name(:packages_packages, INDEX_NAME)
+  end
+end

db/schema_migrations/20210106061254

+f4c81be1168dc8dc3eaadbc9b0d46cfd5aefa0b9e4d61fa8276bbc4f59216da8
\ No newline at end of file

db/structure.sql

@@ -23294,6 +23294,8 @@ CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_
 
 CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_generic ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 7);
 
+CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_golang ON packages_packages USING btree (project_id, name, version) WHERE (package_type = 8);
+
 CREATE INDEX index_packages_package_file_build_infos_on_package_file_id ON packages_package_file_build_infos USING btree (package_file_id);
 
 CREATE INDEX index_packages_package_file_build_infos_on_pipeline_id ON packages_package_file_build_infos USING btree (pipeline_id);

lib/gitlab/golang.rb

@@ -2,6 +2,8 @@
 
 module Gitlab
   module Golang
+    PseudoVersion = Struct.new(:semver, :timestamp, :commit_id)
+
     extend self
 
     def local_module_prefix
@@ -37,11 +39,11 @@ module Gitlab
       end
 
       # This pattern is intentionally more forgiving than the patterns
-      # above. Correctness is verified by #pseudo_version_commit.
+      # above. Correctness is verified by #validate_pseudo_version.
       /\A\d{14}-\h+\z/.freeze.match? pre
     end
 
-    def pseudo_version_commit(project, semver)
+    def parse_pseudo_version(semver)
       # Per Go's implementation of pseudo-versions, a tag should be
       # considered a pseudo-version if it matches one of the patterns
       # listed in #pseudo_version?, regardless of the content of the
@@ -55,9 +57,14 @@ module Gitlab
       # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go)
 
       # Go ignores anything before '.' or after the second '-', so we will do the same
-      timestamp, sha = semver.prerelease.split('-').last 2
+      timestamp, commit_id = semver.prerelease.split('-').last 2
       timestamp = timestamp.split('.').last
-      commit = project.repository.commit_by(oid: sha)
+
+      PseudoVersion.new(semver, timestamp, commit_id)
+    end
+
+    def validate_pseudo_version(project, version, commit = nil)
+      commit ||= project.repository.commit_by(oid: version.commit_id)
 
       # Error messages are based on the responses of proxy.golang.org
 
@@ -65,10 +72,10 @@ module Gitlab
       raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit
 
       # Require the SHA fragment to be 12 characters long
-      raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12
+      raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12
 
       # Require the timestamp to match that of the commit
-      raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp
+      raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp
 
       commit
     end
@@ -77,6 +84,14 @@ module Gitlab
       Packages::SemVer.parse(str, prefixed: true)
     end
 
+    def go_path(project, path = nil)
+      if path.blank?
+        "#{local_module_prefix}/#{project.full_path}"
+      else
+        "#{local_module_prefix}/#{project.full_path}/#{path}"
+      end
+    end
+
     def pkg_go_dev_url(name, version = nil)
       if version
         "https://pkg.go.dev/#{name}@#{version}"

spec/finders/packages/go/package_finder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Go::PackageFinder do
+  include_context 'basic Go module'
+
+  let_it_be(:mod) { create :go_module, project: project }
+  let_it_be(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.1' }
+  let_it_be(:package) { create :golang_package, project: project, name: mod.name, version: 'v1.0.1' }
+
+  let(:finder) { described_class.new(project, mod_name, version_name) }
+
+  describe '#exists?' do
+    subject { finder.exists? }
+
+    context 'with a valid name and version' do
+      let(:mod_name) { mod.name }
+      let(:version_name) { version.name }
+
+      it 'executes SELECT 1' do
+        expect { subject }.to exceed_query_limit(0).for_query(/^SELECT 1/)
+      end
+
+      it { is_expected.to eq(true) }
+    end
+
+    context 'with an invalid name' do
+      let(:mod_name) { 'foo/bar' }
+      let(:version_name) { 'baz' }
+
+      it { is_expected.to eq(false) }
+    end
+
+    context 'with an invalid version' do
+      let(:mod_name) { mod.name }
+      let(:version_name) { 'baz' }
+
+      it { is_expected.to eq(false) }
+    end
+  end
+
+  describe '#execute' do
+    subject { finder.execute }
+
+    context 'with a valid name and version' do
+      let(:mod_name) { mod.name }
+      let(:version_name) { version.name }
+
+      it 'executes a single query' do
+        expect { subject }.not_to exceed_query_limit(1)
+      end
+
+      it { is_expected.to eq(package) }
+    end
+
+    context 'with an invalid name' do
+      let(:mod_name) { 'foo/bar' }
+      let(:version_name) { 'baz' }
+
+      it { is_expected.to eq(nil) }
+    end
+
+    context 'with an invalid version' do
+      let(:mod_name) { mod.name }
+      let(:version_name) { 'baz' }
+
+      it { is_expected.to eq(nil) }
+    end
+  end
+end

spec/models/packages/go/module_version_spec.rb

@@ -3,19 +3,9 @@
 require 'spec_helper'
 
 RSpec.describe Packages::Go::ModuleVersion, type: :model do
-  let_it_be(:user) { create :user }
-  let_it_be(:project) { create :project_empty_repo, creator: user, path: 'my-go-lib' }
-  let_it_be(:mod) { create :go_module, project: project }
+  include_context 'basic Go module'
 
-  before :all do
-    create :go_module_commit, :files,   project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' }
-    create :go_module_commit, :module,  project: project, tag: 'v1.0.1'
-    create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg'
-    create :go_module_commit, :module,  project: project, tag: 'v1.0.3', name: 'mod'
-    create :go_module_commit, :files,   project: project,                files: { 'y.go' => "package a\n" }
-    create :go_module_commit, :module,  project: project,                name: 'v2'
-    create :go_module_commit, :files,   project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" }
-  end
+  let_it_be(:mod) { create :go_module, project: project }
 
   shared_examples '#files' do |desc, *entries|
     it "returns #{desc}" do

spec/services/packages/go/create_package_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Go::CreatePackageService do
+  let_it_be(:project) { create :project_empty_repo, path: 'my-go-lib' }
+  let_it_be(:mod) { create :go_module, project: project }
+
+  before :all do
+    create :go_module_commit, :module, project: project, tag: 'v1.0.0'
+  end
+
+  shared_examples 'creates a package' do |files:|
+    it "returns a valid package with #{files ? files.to_s : 'no'} file(s)" do
+      expect(subject).to be_valid
+      expect(subject.name).to eq(version.mod.name)
+      expect(subject.version).to eq(version.name)
+      expect(subject.package_type).to eq('golang')
+      expect(subject.created_at).to eq(version.commit.committed_date)
+      expect(subject.package_files.count).to eq(files)
+    end
+  end
+
+  shared_examples 'creates a package file' do |type|
+    it "returns a package with a #{type} file" do
+      file_name = "#{version.name}.#{type}"
+      expect(subject.package_files.map { |f| f.file_name }).to include(file_name)
+
+      file = subject.package_files.with_file_name(file_name).first
+      expect(file).not_to be_nil
+      expect(file.file).not_to be_nil
+      expect(file.size).to eq(file.file.size)
+      expect(file.file_name).to eq(file_name)
+      expect(file.file_md5).not_to be_nil
+      expect(file.file_sha1).not_to be_nil
+      expect(file.file_sha256).not_to be_nil
+    end
+  end
+
+  describe '#execute' do
+    subject { described_class.new(project, nil, version: version).execute }
+
+    let(:version) { create :go_module_version, :tagged, mod: mod, name: 'v1.0.0' }
+
+    context 'with no existing package' do
+      it_behaves_like 'creates a package', files: 2
+      it_behaves_like 'creates a package file', :mod
+      it_behaves_like 'creates a package file', :zip
+
+      it 'creates a new package' do
+        expect { subject }
+          .to change { project.packages.count }.by(1)
+          .and change { Packages::PackageFile.count }.by(2)
+      end
+    end
+
+    context 'with an existing package' do
+      before do
+        described_class.new(project, version: version).execute
+      end
+
+      it_behaves_like 'creates a package', files: 2
+      it_behaves_like 'creates a package file', :mod
+      it_behaves_like 'creates a package file', :zip
+
+      it 'does not create a package or files' do
+        expect { subject }
+          .to not_change { project.packages.count }
+          .and not_change { Packages::PackageFile.count }
+      end
+    end
+  end
+end

spec/services/packages/go/sync_packages_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Go::SyncPackagesService do
+  include_context 'basic Go module'
+
+  let(:params) { { info: true, mod: true, zip: true } }
+
+  describe '#execute_async' do
+    it 'schedules a package refresh' do
+      expect(::Packages::Go::SyncPackagesWorker).to receive(:perform_async).once
+
+      described_class.new(project, 'master').execute_async
+    end
+  end
+
+  describe '#initialize' do
+    context 'without a project' do
+      it 'raises an error' do
+        expect { described_class.new(nil, 'master') }
+          .to raise_error(ArgumentError, 'project is required')
+      end
+    end
+
+    context 'without a ref' do
+      it 'raises an error' do
+        expect { described_class.new(project, nil) }
+          .to raise_error(ArgumentError, 'ref is required')
+      end
+    end
+
+    context 'with an invalid ref' do
+      it 'raises an error' do
+        expect { described_class.new(project, 'not-a-ref') }
+        .to raise_error(ArgumentError)
+      end
+    end
+  end
+end

spec/support/shared_contexts/requests/api/go_modules_shared_context.rb

+# frozen_string_literal: true
+
+RSpec.shared_context 'basic Go module' do
+  let_it_be(:user) { create :user }
+  let_it_be(:project) { create :project_empty_repo, creator: user, path: 'my-go-lib' }
+
+  let_it_be(:commit_v1_0_0) { create :go_module_commit, :files,   project: project, tag: 'v1.0.0', files: { 'README.md' => 'Hi' }        }
+  let_it_be(:commit_v1_0_1) { create :go_module_commit, :module,  project: project, tag: 'v1.0.1'                                        }
+  let_it_be(:commit_v1_0_2) { create :go_module_commit, :package, project: project, tag: 'v1.0.2', path: 'pkg'                           }
+  let_it_be(:commit_v1_0_3) { create :go_module_commit, :module,  project: project, tag: 'v1.0.3', name: 'mod'                           }
+  let_it_be(:commit_file_y) { create :go_module_commit, :files,   project: project,                files: { 'y.go' => "package a\n" }    }
+  let_it_be(:commit_mod_v2) { create :go_module_commit, :module,  project: project,                name: 'v2'                            }
+  let_it_be(:commit_v2_0_0) { create :go_module_commit, :files,   project: project, tag: 'v2.0.0', files: { 'v2/x.go' => "package a\n" } }
+end

spec/workers/packages/go/sync_packages_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Go::SyncPackagesWorker, type: :worker do
+  include_context 'basic Go module'
+
+  before do
+    project.update!(visibility_level: Gitlab::VisibilityLevel::PUBLIC)
+  end
+
+  def perform(ref_name, path)
+    described_class.new.perform(project.id, ref_name, path)
+  end
+
+  def validate_package(package, mod, ver)
+    expect(package).not_to be_nil
+    expect(package.name).to eq(mod.name)
+    expect(package.version).to eq(ver.name)
+    expect(package.package_type).to eq('golang')
+    expect(package.created_at).to eq(ver.commit.committed_date)
+    expect(package.package_files.count).to eq(2)
+  end
+
+  shared_examples 'it creates a package' do |path, version, exists: false|
+    subject { perform(version, path) }
+
+    it "returns a package for example.com/project#{path.empty? ? '' : '/' + path}@#{version}" do
+      expect { subject }
+        .to change { project.packages.count }.by(exists ? 0 : 1)
+        .and change { Packages::PackageFile.count }.by(exists ? 0 : 2)
+
+      mod = create :go_module, project: project, path: path
+      ver = create :go_module_version, :tagged, mod: mod, name: version
+      validate_package(subject, mod, ver)
+    end
+  end
+
+  describe '#perform' do
+    context 'with no existing packages' do
+      it_behaves_like 'it creates a package', '', 'v1.0.1'
+      it_behaves_like 'it creates a package', '', 'v1.0.2'
+      it_behaves_like 'it creates a package', '', 'v1.0.3'
+      it_behaves_like 'it creates a package', 'mod', 'v1.0.3'
+      it_behaves_like 'it creates a package', 'v2', 'v2.0.0'
+    end
+
+    context 'with existing packages' do
+      before do
+        mod = create :go_module, project: project
+        ver = create :go_module_version, :tagged, mod: mod, name: 'v1.0.1'
+        Packages::Go::CreatePackageService.new(project, nil, version: ver).execute
+      end
+
+      it_behaves_like 'it creates a package', '', 'v1.0.1', exists: true
+      it_behaves_like 'it creates a package', '', 'v1.0.2'
+      it_behaves_like 'it creates a package', '', 'v1.0.3'
+      it_behaves_like 'it creates a package', 'mod', 'v1.0.3'
+      it_behaves_like 'it creates a package', 'v2', 'v2.0.0'
+    end
+
+    context 'with a package that exceeds project limits' do
+      before do
+        Plan.default.actual_limits.update!({ 'golang_max_file_size': 1 })
+      end
+
+      it 'logs an exception' do
+        expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(::Packages::Go::CreatePackageService::GoZipSizeError))
+
+        perform('v2.0.0', 'v2')
+      end
+    end
+
+    where(:path, :version) do
+      [
+        ['', 'v1.0.1'],
+        ['', 'v1.0.2'],
+        ['', 'v1.0.3'],
+        ['mod', 'v1.0.3'],
+        ['v2', 'v2.0.0']
+      ]
+    end
+
+    with_them do
+      it_behaves_like 'an idempotent worker' do
+        let(:job_args) { [project.id, version, path] }
+
+        it 'creates a package' do
+          expect { subject }
+            .to change { project.packages.count }.by(1)
+            .and change { Packages::PackageFile.count }.by(2)
+
+          mod = create :go_module, project: project, path: path
+          ver = create :go_module_version, :tagged, mod: mod, name: version
+          package = ::Packages::Go::PackageFinder.new(project, mod.name, ver.name).execute
+          validate_package(package, mod, ver)
+        end
+      end
+    end
+  end
+end