Generate Debian component files for udeb and source files

Changelog: added

app/models/packages/package_file.rb

@@ -5,7 +5,7 @@ class Packages::PackageFile < ApplicationRecord
 
   delegate :project, :project_id, to: :package
   delegate :conan_file_type, to: :conan_file_metadatum
-  delegate :file_type, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
+  delegate :file_type, :dsc?, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian
   delegate :channel, :metadata, to: :helm_file_metadatum, prefix: :helm
 
   belongs_to :package
@@ -33,6 +33,8 @@ class Packages::PackageFile < ApplicationRecord
   scope :with_file_name_like, ->(file_name) { where(arel_table[:file_name].matches(file_name)) }
   scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
   scope :with_format, ->(format) { where(::Packages::PackageFile.arel_table[:file_name].matches("%.#{format}")) }
+
+  scope :preload_package, -> { preload(:package) }
   scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
   scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
   scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) }

app/services/packages/debian/generate_distribution_service.rb

@@ -12,7 +12,7 @@ module Packages
       DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
 
       # From https://salsa.debian.org/ftp-team/dak/-/blob/991aaa27a7f7aa773bb9c0cf2d516e383d9cffa0/setup/core-init.d/080_metadatakeys#L9
-      BINARIES_METADATA = %w(
+      METADATA_KEYS = %w(
         Package
         Source
         Binary
@@ -89,15 +89,18 @@ module Packages
         @distribution.components.ordered_by_name.each do |component|
           @distribution.architectures.ordered_by_name.each do |architecture|
             generate_component_file(component, :packages, architecture, :deb)
+            generate_component_file(component, :di_packages, architecture, :udeb)
           end
+          generate_component_file(component, :source, nil, :dsc)
         end
       end
 
       def generate_component_file(component, component_file_type, architecture, package_file_type)
         paragraphs = @distribution.package_files
+                                  .preload_package
                                   .preload_debian_file_metadata
                                   .with_debian_component_name(component.name)
-                                  .with_debian_architecture_name(architecture.name)
+                                  .with_debian_architecture_name(architecture&.name)
                                   .with_debian_file_type(package_file_type)
                                   .find_each
                                   .map(&method(:package_stanza_from_fields))
@@ -106,21 +109,49 @@ module Packages
 
       def package_stanza_from_fields(package_file)
         [
-          BINARIES_METADATA.map do |metadata_key|
-            rfc822_field(metadata_key, package_file.debian_fields[metadata_key])
+          METADATA_KEYS.map do |metadata_key|
+            metadata_name = metadata_key
+            metadata_value = package_file.debian_fields[metadata_key]
+
+            if package_file.debian_dsc?
+              metadata_name = 'Package' if metadata_key == 'Source'
+              checksum = case metadata_key
+                         when 'Files' then package_file.file_md5
+                         when 'Checksums-Sha256' then package_file.file_sha256
+                         when 'Checksums-Sha1' then package_file.file_sha1
+                         end
+
+              if checksum
+                metadata_value = "\n#{checksum} #{package_file.size} #{package_file.file_name}#{metadata_value}"
+              end
+            end
+
+            rfc822_field(metadata_name, metadata_value)
           end,
           rfc822_field('Section', package_file.debian_fields['Section'] || 'misc'),
           rfc822_field('Priority', package_file.debian_fields['Priority'] || 'extra'),
-          rfc822_field('Filename', package_filename(package_file)),
-          rfc822_field('Size', package_file.size),
-          rfc822_field('MD5sum', package_file.file_md5),
-          rfc822_field('SHA256', package_file.file_sha256)
+          package_file_extra_fields(package_file)
         ].flatten.compact.join('')
       end
 
-      def package_filename(package_file)
+      def package_file_extra_fields(package_file)
+        if package_file.debian_dsc?
+          [
+            rfc822_field('Directory', package_dirname(package_file))
+          ]
+        else
+          [
+            rfc822_field('Filename', "#{package_dirname(package_file)}/#{package_file.file_name}"),
+            rfc822_field('Size', package_file.size),
+            rfc822_field('MD5sum', package_file.file_md5),
+            rfc822_field('SHA256', package_file.file_sha256)
+          ]
+        end
+      end
+
+      def package_dirname(package_file)
         letter = package_file.package.name.start_with?('lib') ? package_file.package.name[0..3] : package_file.package.name[0]
-        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}/#{package_file.file_name}"
+        "#{pool_prefix(package_file)}/#{letter}/#{package_file.package.name}/#{package_file.package.version}"
       end
 
       def pool_prefix(package_file)
@@ -206,7 +237,8 @@ module Packages
         return unless condition
         return if value.blank?
 
-        "#{name}: #{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
+        value = " #{value}" unless value[0] == "\n"
+        "#{name}:#{value.to_s.gsub("\n\n", "\n.\n").gsub("\n", "\n ")}\n"
       end
 
       def valid_until_field

spec/factories/packages/debian/file_metadatum.rb

@@ -26,7 +26,27 @@ FactoryBot.define do
       file_type { 'dsc' }
       component { 'main' }
       architecture { nil }
-      fields { { 'a': 'b' } }
+      fields do
+        {
+          'Format' => '3.0 (native)',
+          'Source' => package_file.package.name,
+          'Binary' => 'sample-dev, libsample0, sample-udeb',
+          'Architecture' => 'any',
+          'Version': package_file.package.version,
+          'Maintainer' => "#{FFaker::Name.name} <#{FFaker::Internet.email}>",
+          'Homepage' => FFaker::Internet.http_url,
+          'Standards-Version' => '4.5.0',
+          'Build-Depends' => 'debhelper-compat (= 13)',
+          'Package-List' => <<~EOF.rstrip,
+           libsample0 deb libs optional arch=any',
+           sample-dev deb libdevel optional arch=any',
+           sample-udeb udeb libs optional arch=any',
+          EOF
+          'Checksums-Sha1' => "\nc5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz",
+          'Checksums-Sha256' => "\n40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz",
+          'Files' => "\nd5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz"
+        }
+      end
     end
 
     trait(:deb) do

spec/services/packages/debian/generate_distribution_service_spec.rb

@@ -6,6 +6,9 @@ RSpec.describe Packages::Debian::GenerateDistributionService do
   describe '#execute' do
     subject { described_class.new(distribution).execute }
 
+    let(:subject2) { described_class.new(distribution).execute }
+    let(:subject3) { described_class.new(distribution).execute }
+
     include_context 'with published Debian package'
 
     [:project, :group].each do |container_type|

spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb

@@ -57,10 +57,13 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
       travel_to(current_time) do
         expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
 
+        components_count = 2
+        architectures_count = 3
+
         initial_count = 6
         destroyed_count = 2
-        # updated_count = 1
-        created_count = 5
+        updated_count = 1
+        created_count = components_count * (architectures_count * 2 + 1) - updated_count
 
         expect { subject }
           .to not_change { Packages::Package.count }
@@ -69,7 +72,7 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
           .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
           .and change { component_file1.reload.updated_at }.to(current_time.round)
 
-        debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
+        package_files = package.package_files.order(id: :asc).preload_debian_file_metadata.to_a
         pool_prefix = 'pool/unstable'
         pool_prefix += "/#{project.id}" if container_type == :group
         pool_prefix += "/p/#{package.name}/#{package.version}"
@@ -78,26 +81,26 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
         Source: #{package.name}
         Version: #{package.version}
         Installed-Size: 7
-        Maintainer: #{debs[0].debian_fields['Maintainer']}
+        Maintainer: #{package_files[2].debian_fields['Maintainer']}
         Architecture: amd64
         Description: Some mostly empty lib
          Used in GitLab tests.
          .
          Testing another paragraph.
         Multi-Arch: same
-        Homepage: #{debs[0].debian_fields['Homepage']}
+        Homepage: #{package_files[2].debian_fields['Homepage']}
         Section: libs
         Priority: optional
         Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
         Size: 409600
-        MD5sum: #{debs[0].file_md5}
-        SHA256: #{debs[0].file_sha256}
+        MD5sum: #{package_files[2].file_md5}
+        SHA256: #{package_files[2].file_sha256}
 
         Package: sample-dev
         Source: #{package.name} (#{package.version})
         Version: 1.2.3~binary
         Installed-Size: 7
-        Maintainer: #{debs[1].debian_fields['Maintainer']}
+        Maintainer: #{package_files[3].debian_fields['Maintainer']}
         Architecture: amd64
         Depends: libsample0 (= 1.2.3~binary)
         Description: Some mostly empty development files
@@ -105,23 +108,68 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
          .
          Testing another paragraph.
         Multi-Arch: same
-        Homepage: #{debs[1].debian_fields['Homepage']}
+        Homepage: #{package_files[3].debian_fields['Homepage']}
         Section: libdevel
         Priority: optional
         Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
         Size: 409600
-        MD5sum: #{debs[1].file_md5}
-        SHA256: #{debs[1].file_sha256}
+        MD5sum: #{package_files[3].file_md5}
+        SHA256: #{package_files[3].file_sha256}
+        EOF
+
+        expected_main_amd64_di_content = <<~EOF
+        Section: misc
+        Priority: extra
+        Filename: #{pool_prefix}/sample-udeb_1.2.3~alpha2_amd64.udeb
+        Size: 409600
+        MD5sum: #{package_files[4].file_md5}
+        SHA256: #{package_files[4].file_sha256}
+        EOF
+
+        expected_main_source_content = <<~EOF
+        Package: #{package.name}
+        Binary: sample-dev, libsample0, sample-udeb
+        Version: #{package.version}
+        Maintainer: #{package_files[1].debian_fields['Maintainer']}
+        Build-Depends: debhelper-compat (= 13)
+        Architecture: any
+        Standards-Version: 4.5.0
+        Format: 3.0 (native)
+        Files:
+         #{package_files[1].file_md5} #{package_files[1].size} #{package_files[1].file_name}
+         d5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz
+        Checksums-Sha256:
+         #{package_files[1].file_sha256} #{package_files[1].size} #{package_files[1].file_name}
+         40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz
+        Checksums-Sha1:
+         #{package_files[1].file_sha1} #{package_files[1].size} #{package_files[1].file_name}
+         c5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz
+        Homepage: #{package_files[1].debian_fields['Homepage']}
+        Section: misc
+        Priority: extra
+        Directory: #{pool_prefix}
         EOF
 
         check_component_file(current_time.round, 'main', :packages, 'all', nil)
         check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
         check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
 
+        check_component_file(current_time.round, 'main', :di_packages, 'all', nil)
+        check_component_file(current_time.round, 'main', :di_packages, 'amd64', expected_main_amd64_di_content)
+        check_component_file(current_time.round, 'main', :di_packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'main', :source, nil, expected_main_source_content)
+
         check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
         check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
         check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
 
+        check_component_file(current_time.round, 'contrib', :di_packages, 'all', nil)
+        check_component_file(current_time.round, 'contrib', :di_packages, 'amd64', nil)
+        check_component_file(current_time.round, 'contrib', :di_packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'contrib', :source, nil, nil)
+
         main_amd64_size = expected_main_amd64_content.length
         main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
         main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
@@ -130,6 +178,14 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
         contrib_all_md5sum = component_file1.file_md5
         contrib_all_sha256 = component_file1.file_sha256
 
+        main_amd64_di_size = expected_main_amd64_di_content.length
+        main_amd64_di_md5sum = Digest::MD5.hexdigest(expected_main_amd64_di_content)
+        main_amd64_di_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_di_content)
+
+        main_source_size = expected_main_source_content.length
+        main_source_md5sum = Digest::MD5.hexdigest(expected_main_source_content)
+        main_source_sha256 = Digest::SHA256.hexdigest(expected_main_source_content)
+
         expected_release_content = <<~EOF
         Codename: unstable
         Date: Sat, 25 Jan 2020 15:17:18 +0000
@@ -138,22 +194,44 @@ RSpec.shared_examples 'Generate Debian Distribution and component files' do
         Components: contrib main
         MD5Sum:
          #{contrib_all_md5sum}        #{contrib_all_size} contrib/binary-all/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/debian-installer/binary-all/Packages
          d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/debian-installer/binary-amd64/Packages
          d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/debian-installer/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/source/Source
          d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/debian-installer/binary-all/Packages
          #{main_amd64_md5sum}     #{main_amd64_size} main/binary-amd64/Packages
+         #{main_amd64_di_md5sum}      #{main_amd64_di_size} main/debian-installer/binary-amd64/Packages
          d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/debian-installer/binary-arm64/Packages
+         #{main_source_md5sum}      #{main_source_size} main/source/Source
         SHA256:
          #{contrib_all_sha256}        #{contrib_all_size} contrib/binary-all/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/debian-installer/binary-all/Packages
          e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/debian-installer/binary-amd64/Packages
          e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/debian-installer/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/source/Source
          e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/debian-installer/binary-all/Packages
          #{main_amd64_sha256}     #{main_amd64_size} main/binary-amd64/Packages
+         #{main_amd64_di_sha256}      #{main_amd64_di_size} main/debian-installer/binary-amd64/Packages
          e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/debian-installer/binary-arm64/Packages
+         #{main_source_sha256}      #{main_source_size} main/source/Source
         EOF
 
         check_release_files(expected_release_content)
       end
+
+      create_list(:debian_package, 10, project: project, published_in: project_distribution)
+      control_count = ActiveRecord::QueryRecorder.new { subject2 }.count
+
+      create_list(:debian_package, 10, project: project, published_in: project_distribution)
+      expect { subject3 }.not_to exceed_query_limit(control_count)
     end
   end
 

spec/workers/packages/debian/generate_distribution_worker_spec.rb

@@ -9,6 +9,9 @@ RSpec.describe Packages::Debian::GenerateDistributionWorker, type: :worker do
 
     subject { described_class.new.perform(container_type, distribution_id) }
 
+    let(:subject2) { described_class.new.perform(container_type, distribution_id) }
+    let(:subject3) { described_class.new.perform(container_type, distribution_id) }
+
     include_context 'with published Debian package'
 
     [:project, :group].each do |container_type|