Merge branch 'debian_udeb_src' into 'master'

Generate Debian component files for udeb and source files

See merge request gitlab-org/gitlab!65188

app/models/packages/package_file.rb

@@ -5,7 +5,7 @@ class Packages::PackageFile < ApplicationRecord
 
   delegate :project, :project_id, to: :package
   delegate :conan_file_type, to: :conan_file_metadatum
-  delegate :file_type, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
+  delegate :file_type, :dsc?, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
   delegate :channel, :metadata, to: :helm_file_metadatum, prefix: :helm
 
   belongs_to :package
@@ -33,6 +33,8 @@ class Packages::PackageFile < ApplicationRecord
   scope :with_file_name_like, ->(file_name) { where(arel_table[:file_name].matches(file_name)) }
   scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
   scope :with_format, ->(format) { where(::Packages::PackageFile.arel_table[:file_name].matches("%.#{format}")) }
+
+  scope :preload_package, -> { preload(:package) }
   scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
   scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
   scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) }

app/services/packages/debian/generate_distribution_service.rb

@@ -12,7 +12,7 @@ module Packages
       DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
 
       # From https://salsa.debian.org/ftp-team/dak/-/blob/991aaa27a7f7aa773bb9c0cf2d516e383d9cffa0/setup/core-init.d/080_metadatakeys#L9
-      BINARIES_METADATA = %w(
+      METADATA_KEYS = %w(
         Package
         Source
         Binary
@@ -89,15 +89,18 @@ module Packages
         @distribution.components.ordered_by_name.each do |component|
           @distribution.architectures.ordered_by_name.each do |architecture|
             generate_component_file(component, :packages, architecture, :deb)
+            generate_component_file(component, :di_packages, architecture, :udeb)
           end
+          generate_component_file(component, :source, nil, :dsc)
         end
       end
 
       def generate_component_file(component, component_file_type, architecture, package_file_type)
         paragraphs = @distribution.package_files
+                                  .preload_package
                                   .preload_debian_file_metadata
                                   .with_debian_component_name(component.name)
-                                  .with_debian_architecture_name(architecture.name)
+                                  .with_debian_architecture_name(architecture&.name)
                                   .with_debian_file_type(package_file_type)
                                   .find_each
                                   .map(&method(:package_stanza_from_fields))
@@ -106,21 +109,49 @@ module Packages
 
       def package_stanza_from_fields(package_file)
         [
-          BINARIES_METADATA.map do |metadata_key|
-            rfc822_field(metadata_key, package_file.debian_fields[metadata_key])
+          METADATA_KEYS.map do |metadata_key|
+            metadata_name = metadata_key
+            metadata_value = package_file.debian_fields[metadata_key]
+
+            if package_file.debian_dsc?
+              metadata_name = 'Package' if metadata_key == 'Source'
+              checksum = case metadata_key
+                         when 'Files' then package_file.file_md5
+                         when 'Checksums-Sha256' then package_file.file_sha256
+                         when 'Checksums-Sha1' then package_file.file_sha1
+                         end
+
+              if checksum
+                metadata_value = "\n#{checksum} #{package_file.size} #{package_file.file_name}#{metadata_value}"
+              end
+            end
+
+            rfc822_field(metadata_name, metadata_value)
           end,
           rfc822_field('Section', package_file.debian_fields['Section'] || 'misc'),
           rfc822_field('Priority', package_file.debian_fields['Priority'] || 'extra'),
-          rfc822_field('Filename', package_filename(package_file)),
-          rfc822_field('Size', package_file.size),
-          rfc822_field('MD5sum', package_file.file_md5),
-          rfc822_field('SHA256', package_file.file_sha256)
+          package_file_extra_fields(package_file)
         ].flatten.compact.join('')
       end
 
-      def package_filename(package_file)
+      def package_file_extra_fields(package_file)
+        if package_file.debian_dsc?
+          [
+            rfc822_field('Directory', package_dirname(package_file))
+          ]
+        else
+          [
+            rfc822_field('Filename', "#{package_dirname(package_file)}/#{package_file.file_name}"),
+            rfc822_field('Size', package_file.size),
+            rfc822_field('MD5sum', package_file.file_md5),
+            rfc822_field('SHA256', package_file.file_sha256)
+          ]
+        end
+      end
+
+      def package_dirname(package_file)
         letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
-        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}/#{package_file.file_name}"
+        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}"
       end
 
       def pool_prefix(package_file)
@@ -206,7 +237,8 @@ module Packages
         return unless condition
         return if value.blank?
 
-        "#{name}: #{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
+        value = " #{value}" unless value[0] == "\n"
+        "#{name}:#{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
       end
 
       def valid_until_field

spec/factories/packages/debian/file_metadatum.rb

@@ -26,7 +26,27 @@ FactoryBot.define do
       file_type { 'dsc' }
       component { 'main' }
       architecture { nil }
-      fields { { 'a': 'b' } }
+      fields do
+        {
+          'Format' => '3.0 (native)',
+          'Source' => package_file.package.name,
+          'Binary' => 'sample-dev, libsample0, sample-udeb',
+          'Architecture' => 'any',
+          'Version': package_file.package.version,
+          'Maintainer' => "#{FFaker::Name.name} <#{FFaker::Internet.email}>",
+          'Homepage' => FFaker::Internet.http_url,
+          'Standards-Version' => '4.5.0',
+          'Build-Depends' => 'debhelper-compat (= 13)',
+          'Package-List' => <<~EOF.rstrip,
+           libsample0 deb libs optional arch=any',
+           sample-dev deb libdevel optional arch=any',
+           sample-udeb udeb libs optional arch=any',
+          EOF
+          'Checksums-Sha1' => "\nc5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz",
+          'Checksums-Sha256' => "\n40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz",
+          'Files' => "\nd5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz"
+        }
+      end
     end
 
     trait(:deb) do

spec/services/packages/debian/generate_distribution_service_spec.rb

@@ -6,6 +6,9 @@ RSpec.describe Packages::Debian::GenerateDistributionService do
   describe '#execute' do
     subject { described_class.new(distribution).execute }
 
+    let(:subject2) { described_class.new(distribution).execute }
+    let(:subject3) { described_class.new(distribution).execute }
+
     include_context 'with published Debian package'
 
     [:project, :group].each do |container_type|

spec/workers/packages/debian/generate_distribution_worker_spec.rb

@@ -9,6 +9,9 @@ RSpec.describe Packages::Debian::GenerateDistributionWorker, type: :worker do
 
     subject { described_class.new.perform(container_type, distribution_id) }
 
+    let(:subject2) { described_class.new.perform(container_type, distribution_id) }
+    let(:subject3) { described_class.new.perform(container_type, distribution_id) }
+
     include_context 'with published Debian package'
 
     [:project, :group].each do |container_type|