Merge branch 'generate_distribution_worker' into 'master'

Add Packages::Debian::GenerateDistributionWorker

See merge request gitlab-org/gitlab!61526

app/models/concerns/packages/debian/component_file.rb

@@ -50,6 +50,8 @@ module Packages
 
         scope :with_file_type, ->(file_type) { where(file_type: file_type) }
 
+        scope :with_architecture, ->(architecture) { where(architecture: architecture) }
+
         scope :with_architecture_name, ->(architecture_name) do
           left_outer_joins(:architecture)
             .where("packages_debian_#{container_type}_architectures" => { name: architecture_name })
@@ -60,7 +62,7 @@ module Packages
 
         scope :preload_distribution, -> { includes(component: :distribution) }
 
-        scope :created_before, ->(reference) { where("#{table_name}.created_at < ?", reference) }
+        scope :updated_before, ->(reference) { where("#{table_name}.updated_at < ?", reference) }
 
         mount_file_store_uploader Packages::Debian::ComponentFileUploader
 

app/services/packages/debian/generate_distribution_service.rb

@@ -6,6 +6,8 @@ module Packages
       include Gitlab::Utils::StrongMemoize
       include ExclusiveLeaseGuard
 
+      ONE_HOUR = 1.hour.freeze
+
       # used by ExclusiveLeaseGuard
       DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
 
@@ -62,7 +64,7 @@ module Packages
 
       def initialize(distribution)
         @distribution = distribution
-        @last_generated_at = nil
+        @oldest_kept_generated_at = nil
         @md5sum = []
         @sha256 = []
       end
@@ -70,7 +72,10 @@ module Packages
       def execute
         try_obtain_lease do
           @distribution.transaction do
-            @last_generated_at = @distribution.component_files.maximum(:created_at)
+            # We consider `apt-get update` can take at most one hour
+            # We keep all generations younger than one hour
+            # and the previous generation
+            @oldest_kept_generated_at = @distribution.component_files.updated_before(release_date - ONE_HOUR).maximum(:updated_at)
             generate_component_files
             generate_release
             destroy_old_component_files
@@ -96,7 +101,7 @@ module Packages
                                   .with_debian_file_type(package_file_type)
                                   .find_each
                                   .map(&method(:package_stanza_from_fields))
-        create_component_file(component, component_file_type, architecture, package_file_type, paragraphs.join("\n"))
+        reuse_or_create_component_file(component, component_file_type, architecture, paragraphs.join("\n"))
       end
 
       def package_stanza_from_fields(package_file)
@@ -127,17 +132,32 @@ module Packages
         end
       end
 
-      def create_component_file(component, component_file_type, architecture, package_file_type, content)
-        component_file = component.files.create!(
-          file_type: component_file_type,
-          architecture: architecture,
-          compression_type: nil,
-          file: CarrierWaveStringFile.new(content),
-          file_md5: Digest::MD5.hexdigest(content),
-          file_sha256: Digest::SHA256.hexdigest(content)
-        )
-        @md5sum.append(" #{component_file.file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
-        @sha256.append(" #{component_file.file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
+      def reuse_or_create_component_file(component, component_file_type, architecture, content)
+        file_md5 = Digest::MD5.hexdigest(content)
+        file_sha256 = Digest::SHA256.hexdigest(content)
+        component_file = component.files
+                                  .with_file_type(component_file_type)
+                                  .with_architecture(architecture)
+                                  .with_compression_type(nil)
+                                  .with_file_sha256(file_sha256)
+                                  .last
+
+        if component_file
+          component_file.touch(time: release_date)
+        else
+          component_file = component.files.create!(
+            updated_at: release_date,
+            file_type: component_file_type,
+            architecture: architecture,
+            compression_type: nil,
+            file: CarrierWaveStringFile.new(content),
+            file_md5: file_md5,
+            file_sha256: file_sha256
+          )
+        end
+
+        @md5sum.append(" #{file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
+        @sha256.append(" #{file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
       end
 
       def generate_release
@@ -187,10 +207,9 @@ module Packages
       end
 
       def destroy_old_component_files
-        # Only keep the last generation and one hour before
-        return if @last_generated_at.nil?
+        return if @oldest_kept_generated_at.nil?
 
-        @distribution.component_files.created_before(@last_generated_at - 1.hour).destroy_all # rubocop:disable Cop/DestroyAll
+        @distribution.component_files.updated_before(@oldest_kept_generated_at).destroy_all # rubocop:disable Cop/DestroyAll
       end
 
       # used by ExclusiveLeaseGuard

app/services/packages/debian/process_changes_service.rb

@@ -25,6 +25,8 @@ module Packages
             update_files_metadata
             update_changes_metadata
           end
+
+          ::Packages::Debian::GenerateDistributionWorker.perform_async(:project, package.debian_distribution.id)
         end
       end
 

app/workers/all_queues.yml

@@ -1326,6 +1326,15 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: package_repositories:packages_debian_generate_distribution
+  :worker_name: Packages::Debian::GenerateDistributionWorker
+  :feature_category: :package_registry
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: package_repositories:packages_debian_process_changes
   :worker_name: Packages::Debian::ProcessChangesWorker
   :feature_category: :package_registry

app/workers/packages/debian/generate_distribution_worker.rb

+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class GenerateDistributionWorker # rubocop:disable Scalability/IdempotentWorker
+      include ApplicationWorker
+      include Gitlab::Utils::StrongMemoize
+
+      # The worker is idempotent, by reusing component files with the same file_sha256.
+      #
+      # See GenerateDistributionService#find_or_create_component_file
+      deduplicate :until_executed
+      idempotent!
+
+      queue_namespace :package_repositories
+      feature_category :package_registry
+
+      loggable_arguments 0
+
+      def perform(container_type, distribution_id)
+        @container_type = container_type
+        @distribution_id = distribution_id
+
+        return unless distribution
+
+        ::Packages::Debian::GenerateDistributionService.new(distribution).execute
+      end
+
+      private
+
+      def container_class
+        return ::Packages::Debian::GroupDistribution if @container_type == :group
+
+        ::Packages::Debian::ProjectDistribution
+      end
+
+      def distribution
+        strong_memoize(:distribution) do
+          container_class.find_by_id(@distribution_id)
+        end
+      end
+    end
+  end
+end

spec/factories/packages.rb

@@ -75,7 +75,7 @@ FactoryBot.define do
           create :debian_package_file, :source, evaluator.file_metadatum_trait, package: package
           create :debian_package_file, :dsc, evaluator.file_metadatum_trait, package: package
           create :debian_package_file, :deb, evaluator.file_metadatum_trait, package: package
-          create :debian_package_file, :deb2, evaluator.file_metadatum_trait, package: package
+          create :debian_package_file, :deb_dev, evaluator.file_metadatum_trait, package: package
           create :debian_package_file, :udeb, evaluator.file_metadatum_trait, package: package
           create :debian_package_file, :buildinfo, evaluator.file_metadatum_trait, package: package
           create :debian_package_file, :changes, evaluator.file_metadatum_trait, package: package

spec/factories/packages/debian/component_file.rb

@@ -2,6 +2,10 @@
 
 FactoryBot.define do
   factory :debian_project_component_file, class: 'Packages::Debian::ProjectComponentFile' do
+    transient do
+      file_fixture { 'spec/fixtures/packages/debian/distribution/Packages' }
+    end
+
     component { association(:debian_project_component) }
     architecture { association(:debian_project_architecture, distribution: component.distribution) }
 
@@ -13,7 +17,7 @@ FactoryBot.define do
     file_type { :packages }
 
     after(:build) do |component_file, evaluator|
-      component_file.file = fixture_file_upload('spec/fixtures/packages/debian/distribution/Packages')
+      component_file.file = fixture_file_upload(evaluator.file_fixture) if evaluator.file_fixture.present?
     end
 
     file_md5 { '12345abcde' }

spec/factories/packages/debian/file_metadatum.rb

@@ -33,7 +33,53 @@ FactoryBot.define do
       file_type { 'deb' }
       component { 'main' }
       architecture { 'amd64' }
-      fields { { 'a': 'b' } }
+      fields do
+        {
+        'Package' => 'libsample0',
+        'Source' => package_file.package.name,
+        'Version' => package_file.package.version,
+        'Architecture' => 'amd64',
+        'Maintainer' => "#{FFaker::Name.name} <#{FFaker::Internet.email}>",
+        'Installed-Size' => '7',
+        'Section' => 'libs',
+        'Priority' => 'optional',
+        'Multi-Arch' => 'same',
+        'Homepage' => FFaker::Internet.http_url,
+        'Description' => <<~EOF.rstrip
+        Some mostly empty lib
+        Used in GitLab tests.
+
+        Testing another paragraph.
+        EOF
+        }
+      end
+    end
+
+    trait(:deb_dev) do
+      file_type { 'deb' }
+      component { 'main' }
+      architecture { 'amd64' }
+      fields do
+        {
+          'Package' => 'sample-dev',
+          'Source' => "#{package_file.package.name} (#{package_file.package.version})",
+          'Version' => '1.2.3~binary',
+          'Architecture' => 'amd64',
+          'Maintainer' => "#{FFaker::Name.name} <#{FFaker::Internet.email}>",
+          'Installed-Size' => '7',
+          'Depends' => 'libsample0 (= 1.2.3~binary)',
+          'Section' => 'libdevel',
+          'Priority' => 'optional',
+          'Multi-Arch' => 'same',
+          'Homepage' => FFaker::Internet.http_url,
+          'Description' => <<~EOF.rstrip
+          Some mostly empty development files
+          Used in GitLab tests.
+
+          Testing another paragraph.
+          EOF
+        }
+      end
     end
 
     trait(:udeb) do

spec/factories/packages/package_file.rb

@@ -125,9 +125,9 @@ FactoryBot.define do
 
       trait(:source) do
         file_name { 'sample_1.2.3~alpha2.tar.xz' }
-        file_md5 { 'd79b34f58f61ff4ad696d9bd0b8daa68' }
-        file_sha1 { '5f8bba5574eb01ac3b1f5e2988e8c29307788236' }
-        file_sha256 { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362' }
+        file_md5 { 'd5ca476e4229d135a88f9c729c7606c9' }
+        file_sha1 { 'c5cfc111ea924842a89a06d5673f07dfd07de8ca' }
+        file_sha256 { '40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da' }
 
         transient do
           file_metadatum_trait { :source }
@@ -136,9 +136,9 @@ FactoryBot.define do
 
       trait(:dsc) do
         file_name { 'sample_1.2.3~alpha2.dsc' }
-        file_md5 { '3b0817804f669e16cdefac583ad88f0e' }
-        file_sha1 { '32ecbd674f0bfd310df68484d87752490685a8d6' }
-        file_sha256 { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba' }
+        file_md5 { 'ceccb6bb3e45ce6550b24234d4023e0f' }
+        file_sha1 { '375ba20ea1789e1e90d469c3454ce49a431d0442' }
+        file_sha256 { '81fc156ba937cdb6215362cc4bf6b8dc47be9b4253ba0f1a4ab10c7ea0c4c4e5' }
 
         transient do
           file_metadatum_trait { :dsc }
@@ -156,14 +156,14 @@ FactoryBot.define do
         end
       end
 
-      trait(:deb2) do
+      trait(:deb_dev) do
         file_name { 'sample-dev_1.2.3~binary_amd64.deb' }
-        file_md5 { 'd2afbd28e4d74430d22f9504e18bfdf5' }
-        file_sha1 { 'f81e4f66c8c6bb899653a3340c157965ee69634f' }
-        file_sha256 { '9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d' }
+        file_md5 { '5fafc04dcae1525e1367b15413e5a5c7' }
+        file_sha1 { 'fcd5220b1501ec150ccf37f06e4da919a8612be4' }
+        file_sha256 { 'b8aa8b73a14bc1e0012d4c5309770f5160a8ea7f9dfe6f45222ea6b8a3c35325' }
 
         transient do
-          file_metadatum_trait { :deb }
+          file_metadatum_trait { :deb_dev }
         end
       end
 
@@ -180,9 +180,9 @@ FactoryBot.define do
 
       trait(:buildinfo) do
         file_name { 'sample_1.2.3~alpha2_amd64.buildinfo' }
-        file_md5 { '4e085dd67c120ca967ec314f65770a42' }
-        file_sha1 { '0d47e899f3cc67a2253a4629456ff927e0db5c60' }
-        file_sha256 { 'f9900d3c94e94b329232668dcbef3dba2d96c07147b15b6dc0533452e4dd8a43' }
+        file_md5 { '12a5ac4f16ad75f8741327ac23b4c0d7' }
+        file_sha1 { '661f7507efa6fdd3763c95581d0baadb978b7ef5' }
+        file_sha256 { 'd0c169e9caa5b303a914b27b5adf69768fe6687d4925905b7d0cd9c0f9d4e56c' }
 
         transient do
           file_metadatum_trait { :buildinfo }

spec/fixtures/packages/debian/sample/debian/control

@@ -14,7 +14,7 @@ Section: libdevel
 Architecture: any
 Multi-Arch: same
 Depends: libsample0 (= ${binary:Version}), ${misc:Depends}
-Description: Some mostly empty developpement files
+Description: Some mostly empty development files
  Used in GitLab tests.
  .
  Testing another paragraph.

spec/fixtures/packages/debian/sample_1.2.3~alpha2.dsc

@@ -12,8 +12,8 @@ Package-List:
  sample-dev deb libdevel optional arch=any
  sample-udeb udeb libs optional arch=any
 Checksums-Sha1:
- 5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz
+ c5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz
 Checksums-Sha256:
- b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz
+ 40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz
 Files:
- d79b34f58f61ff4ad696d9bd0b8daa68 864 sample_1.2.3~alpha2.tar.xz
+ d5ca476e4229d135a88f9c729c7606c9 864 sample_1.2.3~alpha2.tar.xz

spec/fixtures/packages/debian/sample_1.2.3~alpha2_amd64.buildinfo

@@ -4,175 +4,182 @@ Binary: libsample0 sample-dev sample-udeb
 Architecture: amd64 source
 Version: 1.2.3~alpha2
 Checksums-Md5:
- 3b0817804f669e16cdefac583ad88f0e 671 sample_1.2.3~alpha2.dsc
+ ceccb6bb3e45ce6550b24234d4023e0f 671 sample_1.2.3~alpha2.dsc
  fb0842b21adc44207996296fe14439dd 1124 libsample0_1.2.3~alpha2_amd64.deb
- d2afbd28e4d74430d22f9504e18bfdf5 1164 sample-dev_1.2.3~binary_amd64.deb
+ 5fafc04dcae1525e1367b15413e5a5c7 1164 sample-dev_1.2.3~binary_amd64.deb
  72b1dd7d98229e2fb0355feda1d3a165 736 sample-udeb_1.2.3~alpha2_amd64.udeb
 Checksums-Sha1:
- 32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc
+ 375ba20ea1789e1e90d469c3454ce49a431d0442 671 sample_1.2.3~alpha2.dsc
  5248b95600e85bfe7f63c0dfce330a75f5777366 1124 libsample0_1.2.3~alpha2_amd64.deb
- f81e4f66c8c6bb899653a3340c157965ee69634f 1164 sample-dev_1.2.3~binary_amd64.deb
+ fcd5220b1501ec150ccf37f06e4da919a8612be4 1164 sample-dev_1.2.3~binary_amd64.deb
  e42e8f2fe04ed1bb73b44a187674480d0e49dcba 736 sample-udeb_1.2.3~alpha2_amd64.udeb
 Checksums-Sha256:
- 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc
+ 81fc156ba937cdb6215362cc4bf6b8dc47be9b4253ba0f1a4ab10c7ea0c4c4e5 671 sample_1.2.3~alpha2.dsc
  1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1 1124 libsample0_1.2.3~alpha2_amd64.deb
- 9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d 1164 sample-dev_1.2.3~binary_amd64.deb
+ b8aa8b73a14bc1e0012d4c5309770f5160a8ea7f9dfe6f45222ea6b8a3c35325 1164 sample-dev_1.2.3~binary_amd64.deb
  2b0c152b3ab4cc07663350424de972c2b7621d69fe6df2e0b94308a191e4632f 736 sample-udeb_1.2.3~alpha2_amd64.udeb
 Build-Origin: Debian
 Build-Architecture: amd64
-Build-Date: Thu, 08 Oct 2020 15:15:24 +0200
+Build-Date: Fri, 14 May 2021 16:51:32 +0200
 Build-Tainted-By:
- merged-usr-via-symlinks
+ merged-usr-via-aliased-dirs
  usr-local-has-includes
  usr-local-has-libraries
  usr-local-has-programs
 Installed-Build-Depends:
- autoconf (= 2.69-11.1),
- automake (= 1:1.16.2-4),
- autopoint (= 0.19.8.1-10),
- autotools-dev (= 20180224.1),
+ autoconf (= 2.69-14),
+ automake (= 1:1.16.3-2),
+ autopoint (= 0.21-4),
+ autotools-dev (= 20180224.1+nmu1),
  base-files (= 11),
- base-passwd (= 3.5.47),
- bash (= 5.0-7),
- binutils (= 2.35.1-1),
- binutils-common (= 2.35.1-1),
- binutils-x86-64-linux-gnu (= 2.35.1-1),
- bsdextrautils (= 2.36-3+b1),
- bsdmainutils (= 12.1.7),
- bsdutils (= 1:2.36-3+b1),
- build-essential (= 12.8),
+ base-passwd (= 3.5.49),
+ bash (= 5.1-2+b1),
+ binutils (= 2.35.2-2),
+ binutils-common (= 2.35.2-2),
+ binutils-x86-64-linux-gnu (= 2.35.2-2),
+ bsdextrautils (= 2.36.1-7),
+ bsdmainutils (= 12.1.7+nmu3),
+ bsdutils (= 1:2.36.1-7),
+ build-essential (= 12.9),
  bzip2 (= 1.0.8-4),
- calendar (= 12.1.7),
  coreutils (= 8.32-4+b1),
- cpp (= 4:10.2.0-1),
- cpp-10 (= 10.2.0-9),
- cpp-9 (= 9.3.0-18),
- dash (= 0.5.10.2-7),
- debconf (= 1.5.74),
- debhelper (= 13.2.1),
+ cpp (= 4:10.2.1-1),
+ cpp-10 (= 10.2.1-6),
+ cpp-9 (= 9.3.0-22),
+ dash (= 0.5.11+git20200708+dd9ef66-5),
+ debconf (= 1.5.75),
+ debhelper (= 13.3.4),
  debianutils (= 4.11.2),
- dh-autoreconf (= 19),
- dh-strip-nondeterminism (= 1.9.0-1),
- diffutils (= 1:3.7-3),
- dpkg (= 1.20.5),
- dpkg-dev (= 1.20.5),
- dwz (= 0.13-5),
- file (= 1:5.38-5),
- findutils (= 4.7.0-1),
- g++ (= 4:10.2.0-1),
- g++-10 (= 10.2.0-9),
- gcc (= 4:10.2.0-1),
- gcc-10 (= 10.2.0-9),
- gcc-10-base (= 10.2.0-9),
- gcc-9 (= 9.3.0-18),
- gcc-9-base (= 9.3.0-18),
- gettext (= 0.19.8.1-10),
- gettext-base (= 0.19.8.1-10),
- grep (= 3.4-1),
- groff-base (= 1.22.4-5),
- gzip (= 1.10-2),
+ dh-autoreconf (= 20),
+ dh-strip-nondeterminism (= 1.11.0-1),
+ diffutils (= 1:3.7-5),
+ dpkg (= 1.20.9),
+ dpkg-dev (= 1.20.9),
+ dwz (= 0.13+20210201-1),
+ file (= 1:5.39-3),
+ findutils (= 4.8.0-1),
+ g++ (= 4:10.2.1-1),
+ g++-10 (= 10.2.1-6),
+ gcc (= 4:10.2.1-1),
+ gcc-10 (= 10.2.1-6),
+ gcc-10-base (= 10.2.1-6),
+ gcc-9 (= 9.3.0-22),
+ gcc-9-base (= 9.3.0-22),
+ gettext (= 0.21-4),
+ gettext-base (= 0.21-4),
+ grep (= 3.6-1),
+ groff-base (= 1.22.4-6),
+ gzip (= 1.10-4),
  hostname (= 3.23),
- init-system-helpers (= 1.58),
+ init-system-helpers (= 1.60),
  intltool-debian (= 0.35.0+20060710.5),
- libacl1 (= 2.2.53-8),
+ libacl1 (= 2.2.53-10),
  libarchive-zip-perl (= 1.68-1),
- libasan5 (= 9.3.0-18),
- libasan6 (= 10.2.0-9),
- libatomic1 (= 10.2.0-9),
- libattr1 (= 1:2.4.48-5),
- libaudit-common (= 1:2.8.5-3),
- libaudit1 (= 1:2.8.5-3+b1),
- libbinutils (= 2.35.1-1),
- libblkid1 (= 2.36-3+b1),
- libbsd0 (= 0.10.0-1),
+ libasan5 (= 9.3.0-22),
+ libasan6 (= 10.2.1-6),
+ libatomic1 (= 10.2.1-6),
+ libattr1 (= 1:2.4.48-6),
+ libaudit-common (= 1:3.0-2),
+ libaudit1 (= 1:3.0-2),
+ libbinutils (= 2.35.2-2),
+ libblkid1 (= 2.36.1-7),
  libbz2-1.0 (= 1.0.8-4),
- libc-bin (= 2.31-3),
- libc-dev-bin (= 2.31-3),
- libc6 (= 2.31-3),
- libc6-dev (= 2.31-3),
- libcap-ng0 (= 0.7.9-2.2),
- libcc1-0 (= 10.2.0-9),
- libcroco3 (= 0.6.13-1),
- libcrypt-dev (= 1:4.4.17-1),
- libcrypt1 (= 1:4.4.17-1),
- libctf-nobfd0 (= 2.35.1-1),
- libctf0 (= 2.35.1-1),
- libdb5.3 (= 5.3.28+dfsg1-0.6),
- libdebconfclient0 (= 0.254),
- libdebhelper-perl (= 13.2.1),
- libdpkg-perl (= 1.20.5),
- libelf1 (= 0.181-1),
- libffi7 (= 3.3-4),
- libfile-stripnondeterminism-perl (= 1.9.0-1),
- libgcc-10-dev (= 10.2.0-9),
- libgcc-9-dev (= 9.3.0-18),
- libgcc-s1 (= 10.2.0-9),
- libgcrypt20 (= 1.8.6-2),
- libgdbm-compat4 (= 1.18.1-5.1),
- libgdbm6 (= 1.18.1-5.1),
- libglib2.0-0 (= 2.66.0-2),
- libgmp10 (= 2:6.2.0+dfsg-6),
- libgomp1 (= 10.2.0-9),
+ libc-bin (= 2.31-11),
+ libc-dev-bin (= 2.31-11),
+ libc6 (= 2.31-11),
+ libc6-dev (= 2.31-11),
+ libcap-ng0 (= 0.7.9-2.2+b1),
+ libcc1-0 (= 10.2.1-6),
+ libcom-err2 (= 1.46.2-1),
+ libcrypt-dev (= 1:4.4.18-2),
+ libcrypt1 (= 1:4.4.18-2),
+ libctf-nobfd0 (= 2.35.2-2),
+ libctf0 (= 2.35.2-2),
+ libdb5.3 (= 5.3.28+dfsg1-0.8),
+ libdebconfclient0 (= 0.257),
+ libdebhelper-perl (= 13.3.4),
+ libdpkg-perl (= 1.20.9),
+ libelf1 (= 0.183-1),
+ libfile-stripnondeterminism-perl (= 1.11.0-1),
+ libgcc-10-dev (= 10.2.1-6),
+ libgcc-9-dev (= 9.3.0-22),
+ libgcc-s1 (= 10.2.1-6),
+ libgcrypt20 (= 1.8.7-3),
+ libgdbm-compat4 (= 1.19-2),
+ libgdbm6 (= 1.19-2),
+ libgmp10 (= 2:6.2.1+dfsg-1),
+ libgomp1 (= 10.2.1-6),
  libgpg-error0 (= 1.38-2),
- libicu67 (= 67.1-4),
- libisl22 (= 0.22.1-1),
- libitm1 (= 10.2.0-9),
- liblsan0 (= 10.2.0-9),
- liblz4-1 (= 1.9.2-2),
- liblzma5 (= 5.2.4-1+b1),
- libmagic-mgc (= 1:5.38-5),
- libmagic1 (= 1:5.38-5),
- libmount1 (= 2.36-3+b1),
+ libgssapi-krb5-2 (= 1.18.3-5),
+ libicu67 (= 67.1-6),
+ libisl23 (= 0.23-1),
+ libitm1 (= 10.2.1-6),
+ libk5crypto3 (= 1.18.3-5),
+ libkeyutils1 (= 1.6.1-2),
+ libkrb5-3 (= 1.18.3-5),
+ libkrb5support0 (= 1.18.3-5),
+ liblsan0 (= 10.2.1-6),
+ liblz4-1 (= 1.9.3-1),
+ liblzma5 (= 5.2.5-2),
+ libmagic-mgc (= 1:5.39-3),
+ libmagic1 (= 1:5.39-3),
+ libmount1 (= 2.36.1-7),
  libmpc3 (= 1.2.0-1),
  libmpfr6 (= 4.1.0-3),
- libpam-modules (= 1.3.1-5),
- libpam-modules-bin (= 1.3.1-5),
- libpam-runtime (= 1.3.1-5),
- libpam0g (= 1.3.1-5),
- libpcre2-8-0 (= 10.34-7),
+ libnsl-dev (= 1.3.0-2),
+ libnsl2 (= 1.3.0-2),
+ libpam-modules (= 1.4.0-7),
+ libpam-modules-bin (= 1.4.0-7),
+ libpam-runtime (= 1.4.0-7),
+ libpam0g (= 1.4.0-7),
+ libpcre2-8-0 (= 10.36-2),
  libpcre3 (= 2:8.39-13),
- libperl5.30 (= 5.30.3-4),
+ libperl5.32 (= 5.32.1-4),
  libpipeline1 (= 1.5.3-1),
- libquadmath0 (= 10.2.0-9),
- libseccomp2 (= 2.4.4-1),
- libselinux1 (= 3.1-2),
- libsigsegv2 (= 2.12-2),
- libsmartcols1 (= 2.36-3+b1),
- libstdc++-10-dev (= 10.2.0-9),
- libstdc++6 (= 10.2.0-9),
+ libquadmath0 (= 10.2.1-6),
+ libseccomp2 (= 2.5.1-1),
+ libselinux1 (= 3.1-3),
+ libsigsegv2 (= 2.13-1),
+ libsmartcols1 (= 2.36.1-7),
+ libssl1.1 (= 1.1.1k-1),
+ libstdc++-10-dev (= 10.2.1-6),
+ libstdc++6 (= 10.2.1-6),
  libsub-override-perl (= 0.09-2),
- libsystemd0 (= 246.6-1),
- libtinfo6 (= 6.2+20200918-1),
- libtool (= 2.4.6-14),
- libtsan0 (= 10.2.0-9),
- libubsan1 (= 10.2.0-9),
+ libsystemd0 (= 247.3-5),
+ libtinfo6 (= 6.2+20201114-2),
+ libtirpc-common (= 1.3.1-1),
+ libtirpc-dev (= 1.3.1-1),
+ libtirpc3 (= 1.3.1-1),
+ libtool (= 2.4.6-15),
+ libtsan0 (= 10.2.1-6),
+ libubsan1 (= 10.2.1-6),
  libuchardet0 (= 0.0.7-1),
- libudev1 (= 246.6-1),
+ libudev1 (= 247.3-5),
  libunistring2 (= 0.9.10-4),
- libuuid1 (= 2.36-3+b1),
- libxml2 (= 2.9.10+dfsg-6),
- libzstd1 (= 1.4.5+dfsg-4),
- linux-libc-dev (= 5.8.10-1),
+ libuuid1 (= 2.36.1-7),
+ libxml2 (= 2.9.10+dfsg-6.3+b1),
+ libzstd1 (= 1.4.8+dfsg-2.1),
+ linux-libc-dev (= 5.10.28-1),
  login (= 1:4.8.1-1),
  lsb-base (= 11.1.0),
- m4 (= 1.4.18-4),
- make (= 4.3-4),
- man-db (= 2.9.3-2),
+ m4 (= 1.4.18-5),
+ make (= 4.3-4.1),
+ man-db (= 2.9.4-2),
  mawk (= 1.3.4.20200120-2),
- ncal (= 12.1.7),
- ncurses-base (= 6.2+20200918-1),
- ncurses-bin (= 6.2+20200918-1),
- patch (= 2.7.6-6),
- perl (= 5.30.3-4),
- perl-base (= 5.30.3-4),
- perl-modules-5.30 (= 5.30.3-4),
- po-debconf (= 1.0.21),
+ ncal (= 12.1.7+nmu3),
+ ncurses-base (= 6.2+20201114-2),
+ ncurses-bin (= 6.2+20201114-2),
+ patch (= 2.7.6-7),
+ perl (= 5.32.1-4),
+ perl-base (= 5.32.1-4),
+ perl-modules-5.32 (= 5.32.1-4),
+ po-debconf (= 1.0.21+nmu1),
  sed (= 4.7-1),
- sensible-utils (= 0.0.12+nmu1),
- sysvinit-utils (= 2.96-5),
- tar (= 1.30+dfsg-7),
- util-linux (= 2.36-3+b1),
- xz-utils (= 5.2.4-1+b1),
+ sensible-utils (= 0.0.14),
+ sysvinit-utils (= 2.96-7),
+ tar (= 1.34+dfsg-1),
+ util-linux (= 2.36.1-7),
+ xz-utils (= 5.2.5-2),
  zlib1g (= 1:1.2.11.dfsg-2)
 Environment:
  DEB_BUILD_OPTIONS="parallel=8"

spec/fixtures/packages/debian/sample_1.2.3~alpha2_amd64.changes

@@ -10,30 +10,30 @@ Maintainer: John Doe <john.doe@example.com>
 Changed-By: John Doe <john.doe@example.com>
 Description:
  libsample0 - Some mostly empty lib
- sample-dev - Some mostly empty developpement files
+ sample-dev - Some mostly empty development files
  sample-udeb - Some mostly empty udeb (udeb)
 Changes:
  sample (1.2.3~alpha2) unstable; urgency=medium
  .
    * Initial release
 Checksums-Sha1:
- 32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc
- 5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz
+ 375ba20ea1789e1e90d469c3454ce49a431d0442 671 sample_1.2.3~alpha2.dsc
+ c5cfc111ea924842a89a06d5673f07dfd07de8ca 864 sample_1.2.3~alpha2.tar.xz
  5248b95600e85bfe7f63c0dfce330a75f5777366 1124 libsample0_1.2.3~alpha2_amd64.deb
- f81e4f66c8c6bb899653a3340c157965ee69634f 1164 sample-dev_1.2.3~binary_amd64.deb
+ fcd5220b1501ec150ccf37f06e4da919a8612be4 1164 sample-dev_1.2.3~binary_amd64.deb
  e42e8f2fe04ed1bb73b44a187674480d0e49dcba 736 sample-udeb_1.2.3~alpha2_amd64.udeb
- 0d47e899f3cc67a2253a4629456ff927e0db5c60 5280 sample_1.2.3~alpha2_amd64.buildinfo
+ 661f7507efa6fdd3763c95581d0baadb978b7ef5 5507 sample_1.2.3~alpha2_amd64.buildinfo
 Checksums-Sha256:
- 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc
- b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz
+ 81fc156ba937cdb6215362cc4bf6b8dc47be9b4253ba0f1a4ab10c7ea0c4c4e5 671 sample_1.2.3~alpha2.dsc
+ 40e4682bb24a73251ccd7c7798c0094a649091e5625d6a14bcec9b4e7174f3da 864 sample_1.2.3~alpha2.tar.xz
  1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1 1124 libsample0_1.2.3~alpha2_amd64.deb
- 9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d 1164 sample-dev_1.2.3~binary_amd64.deb
+ b8aa8b73a14bc1e0012d4c5309770f5160a8ea7f9dfe6f45222ea6b8a3c35325 1164 sample-dev_1.2.3~binary_amd64.deb
  2b0c152b3ab4cc07663350424de972c2b7621d69fe6df2e0b94308a191e4632f 736 sample-udeb_1.2.3~alpha2_amd64.udeb
- f9900d3c94e94b329232668dcbef3dba2d96c07147b15b6dc0533452e4dd8a43 5280 sample_1.2.3~alpha2_amd64.buildinfo
+ d0c169e9caa5b303a914b27b5adf69768fe6687d4925905b7d0cd9c0f9d4e56c 5507 sample_1.2.3~alpha2_amd64.buildinfo
 Files:
- 3b0817804f669e16cdefac583ad88f0e 671 libs optional sample_1.2.3~alpha2.dsc
- d79b34f58f61ff4ad696d9bd0b8daa68 864 libs optional sample_1.2.3~alpha2.tar.xz
+ ceccb6bb3e45ce6550b24234d4023e0f 671 libs optional sample_1.2.3~alpha2.dsc
+ d5ca476e4229d135a88f9c729c7606c9 864 libs optional sample_1.2.3~alpha2.tar.xz
  fb0842b21adc44207996296fe14439dd 1124 libs optional libsample0_1.2.3~alpha2_amd64.deb
- d2afbd28e4d74430d22f9504e18bfdf5 1164 libdevel optional sample-dev_1.2.3~binary_amd64.deb
+ 5fafc04dcae1525e1367b15413e5a5c7 1164 libdevel optional sample-dev_1.2.3~binary_amd64.deb
  72b1dd7d98229e2fb0355feda1d3a165 736 libs optional sample-udeb_1.2.3~alpha2_amd64.udeb
- 4e085dd67c120ca967ec314f65770a42 5280 libs optional sample_1.2.3~alpha2_amd64.buildinfo
+ 12a5ac4f16ad75f8741327ac23b4c0d7 5507 libs optional sample_1.2.3~alpha2_amd64.buildinfo

spec/models/packages/debian/file_entry_spec.rb

@@ -7,11 +7,11 @@ RSpec.describe Packages::Debian::FileEntry, type: :model do
 
   let(:filename) { 'sample_1.2.3~alpha2.dsc' }
   let(:size) { 671 }
-  let(:md5sum) { '3b0817804f669e16cdefac583ad88f0e' }
+  let(:md5sum) { package_file.file_md5 }
   let(:section) { 'libs' }
   let(:priority) { 'optional' }
-  let(:sha1sum) { '32ecbd674f0bfd310df68484d87752490685a8d6' }
-  let(:sha256sum) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba' }
+  let(:sha1sum) { package_file.file_sha1 }
+  let(:sha256sum) { package_file.file_sha256 }
 
   let(:file_entry) do
     described_class.new(
@@ -42,7 +42,7 @@ RSpec.describe Packages::Debian::FileEntry, type: :model do
 
     describe '#md5sum' do
       it { is_expected.to validate_presence_of(:md5sum) }
-      it { is_expected.not_to allow_value('12345678901234567890123456789012').for(:md5sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 3b0817804f669e16cdefac583ad88f0e != 12345678901234567890123456789012') }
+      it { is_expected.not_to allow_value('12345678901234567890123456789012').for(:md5sum).with_message("mismatch for sample_1.2.3~alpha2.dsc: #{package_file.file_md5} != 12345678901234567890123456789012") }
     end
 
     describe '#section' do
@@ -55,12 +55,12 @@ RSpec.describe Packages::Debian::FileEntry, type: :model do
 
     describe '#sha1sum' do
       it { is_expected.to validate_presence_of(:sha1sum) }
-      it { is_expected.not_to allow_value('1234567890123456789012345678901234567890').for(:sha1sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 32ecbd674f0bfd310df68484d87752490685a8d6 != 1234567890123456789012345678901234567890') }
+      it { is_expected.not_to allow_value('1234567890123456789012345678901234567890').for(:sha1sum).with_message("mismatch for sample_1.2.3~alpha2.dsc: #{package_file.file_sha1} != 1234567890123456789012345678901234567890") }
     end
 
     describe '#sha256sum' do
       it { is_expected.to validate_presence_of(:sha256sum) }
-      it { is_expected.not_to allow_value('1234567890123456789012345678901234567890123456789012345678901234').for(:sha256sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba != 1234567890123456789012345678901234567890123456789012345678901234') }
+      it { is_expected.not_to allow_value('1234567890123456789012345678901234567890123456789012345678901234').for(:sha256sum).with_message("mismatch for sample_1.2.3~alpha2.dsc: #{package_file.file_sha256} != 1234567890123456789012345678901234567890123456789012345678901234") }
     end
 
     describe '#package_file' do

spec/services/packages/debian/extract_changes_metadata_service_spec.rb

@@ -6,8 +6,10 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
     let_it_be(:distribution) { create(:debian_project_distribution, codename: 'unstable') }
     let_it_be(:incoming) { create(:debian_incoming, project: distribution.project) }
 
-    let(:package_file) { incoming.package_files.last }
-    let(:service) { described_class.new(package_file) }
+    let(:source_file) { incoming.package_files.first }
+    let(:dsc_file) { incoming.package_files.second }
+    let(:changes_file) { incoming.package_files.last }
+    let(:service) { described_class.new(changes_file) }
 
     subject { service.execute }
 
@@ -23,7 +25,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
     end
 
     context 'with invalid package file' do
-      let(:package_file) { incoming.package_files.first }
+      let(:changes_file) { incoming.package_files.first }
 
       it 'raise ArgumentError', :aggregate_failures do
         expect { subject }.to raise_error(described_class::ExtractionError, "is not a changes file")
@@ -31,14 +33,14 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
     end
 
     context 'with invalid metadata' do
-      let(:md5_dsc) { '3b0817804f669e16cdefac583ad88f0e 671 libs optional sample_1.2.3~alpha2.dsc' }
-      let(:md5_source) { 'd79b34f58f61ff4ad696d9bd0b8daa68 864 libs optional sample_1.2.3~alpha2.tar.xz' }
+      let(:md5_dsc) { "#{dsc_file.file_md5} 671 libs optional sample_1.2.3~alpha2.dsc" }
+      let(:md5_source) { "#{source_file.file_md5} 864 libs optional sample_1.2.3~alpha2.tar.xz" }
       let(:md5s) { "#{md5_dsc}\n#{md5_source}" }
-      let(:sha1_dsc) { '32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc' }
-      let(:sha1_source) { '5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz' }
+      let(:sha1_dsc) { "#{dsc_file.file_sha1} 671 sample_1.2.3~alpha2.dsc" }
+      let(:sha1_source) { "#{source_file.file_sha1} 864 sample_1.2.3~alpha2.tar.xz" }
       let(:sha1s) { "#{sha1_dsc}\n#{sha1_source}" }
-      let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc' }
-      let(:sha256_source) { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz' }
+      let(:sha256_dsc) { "#{dsc_file.file_sha256} 671 sample_1.2.3~alpha2.dsc" }
+      let(:sha256_source) { "#{source_file.file_sha256} 864 sample_1.2.3~alpha2.tar.xz" }
       let(:sha256s) { "#{sha256_dsc}\n#{sha256_source}" }
       let(:fields) { { 'Files' => md5s, 'Checksums-Sha1' => sha1s, 'Checksums-Sha256' => sha256s } }
       let(:metadata) { { file_type: :changes, architecture: 'amd64', fields: fields } }
@@ -82,7 +84,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
       end
 
       context 'with different size in Checksums-Sha1' do
-        let(:sha1_dsc) { '32ecbd674f0bfd310df68484d87752490685a8d6 42 sample_1.2.3~alpha2.dsc' }
+        let(:sha1_dsc) { "#{dsc_file.file_sha1} 42 sample_1.2.3~alpha2.dsc" }
 
         it 'raise ArgumentError', :aggregate_failures do
           expect { subject }.to raise_error(described_class::ExtractionError, "Size for sample_1.2.3~alpha2.dsc in Files and Checksums-Sha1 differ")
@@ -99,7 +101,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
       end
 
       context 'with different size in Checksums-Sha256' do
-        let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 42 sample_1.2.3~alpha2.dsc' }
+        let(:sha256_dsc) { "#{dsc_file.file_sha256} 42 sample_1.2.3~alpha2.dsc" }
 
         it 'raise ArgumentError', :aggregate_failures do
           expect { subject }.to raise_error(described_class::ExtractionError, "Size for sample_1.2.3~alpha2.dsc in Files and Checksums-Sha256 differ")
@@ -126,7 +128,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
         let(:md5_dsc) { '1234567890123456789012345678012 671 libs optional sample_1.2.3~alpha2.dsc' }
 
         it 'raise ArgumentError', :aggregate_failures do
-          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Md5sum mismatch for sample_1.2.3~alpha2.dsc: 3b0817804f669e16cdefac583ad88f0e != 1234567890123456789012345678012")
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Md5sum mismatch for sample_1.2.3~alpha2.dsc: #{dsc_file.file_md5} != 1234567890123456789012345678012")
         end
       end
 
@@ -134,7 +136,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
         let(:sha1_dsc) { '1234567890123456789012345678901234567890 671 sample_1.2.3~alpha2.dsc' }
 
         it 'raise ArgumentError', :aggregate_failures do
-          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha1sum mismatch for sample_1.2.3~alpha2.dsc: 32ecbd674f0bfd310df68484d87752490685a8d6 != 1234567890123456789012345678901234567890")
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha1sum mismatch for sample_1.2.3~alpha2.dsc: #{dsc_file.file_sha1} != 1234567890123456789012345678901234567890")
         end
       end
 
@@ -142,7 +144,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
         let(:sha256_dsc) { '1234567890123456789012345678901234567890123456789012345678901234 671 sample_1.2.3~alpha2.dsc' }
 
         it 'raise ArgumentError', :aggregate_failures do
-          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha256sum mismatch for sample_1.2.3~alpha2.dsc: 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba != 1234567890123456789012345678901234567890123456789012345678901234")
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha256sum mismatch for sample_1.2.3~alpha2.dsc: #{dsc_file.file_sha256} != 1234567890123456789012345678901234567890123456789012345678901234")
         end
       end
     end

spec/services/packages/debian/generate_distribution_service_spec.rb

 # frozen_string_literal: true
+
 require 'spec_helper'
 
 RSpec.describe Packages::Debian::GenerateDistributionService do
-  let_it_be(:group) { create(:group, :public) }
-  let_it_be(:project) { create(:project, :public, group: group) }
-  let_it_be(:project_distribution) { create("debian_project_distribution", container: project, codename: 'unstable', valid_time_duration_seconds: 48.hours.to_i) }
-
-  let_it_be(:incoming) { create(:debian_incoming, project: project) }
-
-  before_all do
-    ::Packages::Debian::ProcessChangesService.new(incoming.package_files.last, nil).execute
-  end
-
-  let(:service) { described_class.new(distribution) }
-
   describe '#execute' do
-    subject { service.execute }
-
-    shared_examples 'Generate Distribution' do |container_type|
-      context "for #{container_type}" do
-        if container_type == :group
-          let_it_be(:container) { group }
-          let_it_be(:distribution, reload: true) { create('debian_group_distribution', container: group, codename: 'unstable', valid_time_duration_seconds: 48.hours.to_i) }
-        else
-          let_it_be(:container) { project }
-          let_it_be(:distribution, reload: true) { project_distribution }
-        end
-
-        context 'with components and architectures' do
-          let_it_be(:component_main   ) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
-          let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
-
-          let_it_be(:architecture_all  ) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
-          let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
-          let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
-
-          let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   created_at: '2020-01-24T09:00:00.000Z') } # destroyed
-          let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, created_at: '2020-01-24T10:29:59.000Z') } # destroyed
-          let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   created_at: '2020-01-24T10:30:00.000Z') } # kept
-          let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, created_at: '2020-01-24T11:30:00.000Z') } # kept
-
-          def check_component_file(component_name, component_file_type, architecture_name, expected_content)
-            component_file = distribution
-              .component_files
-              .with_component_name(component_name)
-              .with_file_type(component_file_type)
-              .with_architecture_name(architecture_name)
-              .last
+    subject { described_class.new(distribution).execute }
 
-            expect(component_file).not_to be_nil
-            expect(component_file.file.exists?).to eq(!expected_content.nil?)
+    include_context 'with published Debian package'
 
-            unless expected_content.nil?
-              component_file.file.use_file do |file_path|
-                expect(File.read(file_path)).to eq(expected_content)
-              end
-            end
-          end
-
-          it 'updates distribution and component files', :aggregate_failures do
-            travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
-              expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
-
-              expect { subject }
-                .to not_change { Packages::Package.count }
-                .and not_change { Packages::PackageFile.count }
-                .and change { distribution.component_files.count }.from(4).to(2 + 6)
-
-              expected_main_amd64_content = <<~EOF
-              Package: libsample0
-              Source: sample
-              Version: 1.2.3~alpha2
-              Installed-Size: 7
-              Maintainer: John Doe <john.doe@example.com>
-              Architecture: amd64
-              Description: Some mostly empty lib
-               Used in GitLab tests.
-               .
-               Testing another paragraph.
-              Multi-Arch: same
-              Homepage: https://gitlab.com/
-              Section: libs
-              Priority: optional
-              Filename: pool/unstable/#{project.id}/s/sample/libsample0_1.2.3~alpha2_amd64.deb
-              Size: 409600
-              MD5sum: fb0842b21adc44207996296fe14439dd
-              SHA256: 1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1
-
-              Package: sample-dev
-              Source: sample (1.2.3~alpha2)
-              Version: 1.2.3~binary
-              Installed-Size: 7
-              Maintainer: John Doe <john.doe@example.com>
-              Architecture: amd64
-              Depends: libsample0 (= 1.2.3~binary)
-              Description: Some mostly empty developpement files
-               Used in GitLab tests.
-               .
-               Testing another paragraph.
-              Multi-Arch: same
-              Homepage: https://gitlab.com/
-              Section: libdevel
-              Priority: optional
-              Filename: pool/unstable/#{project.id}/s/sample/sample-dev_1.2.3~binary_amd64.deb
-              Size: 409600
-              MD5sum: d2afbd28e4d74430d22f9504e18bfdf5
-              SHA256: 9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d
-              EOF
-
-              check_component_file('main', :packages, 'all', nil)
-              check_component_file('main', :packages, 'amd64', expected_main_amd64_content)
-              check_component_file('main', :packages, 'arm64', nil)
-
-              check_component_file('contrib', :packages, 'all', nil)
-              check_component_file('contrib', :packages, 'amd64', nil)
-              check_component_file('contrib', :packages, 'arm64', nil)
-
-              size = expected_main_amd64_content.length
-              md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
-              sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
-
-              expected_release_content = <<~EOF
-              Codename: unstable
-              Date: Sat, 25 Jan 2020 15:17:18 +0000
-              Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
-              Architectures: all amd64 arm64
-              Components: contrib main
-              MD5Sum:
-               d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-all/Packages
-               d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
-               d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
-               d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
-               #{md5sum}     #{size} main/binary-amd64/Packages
-               d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
-              SHA256:
-               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-all/Packages
-               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
-               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
-               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
-               #{sha256}     #{size} main/binary-amd64/Packages
-               e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
-              EOF
+    [:project, :group].each do |container_type|
+      context "for #{container_type}" do
+        include_context 'with Debian distribution', container_type
 
-              distribution.file.use_file do |file_path|
-                expect(File.read(file_path)).to eq(expected_release_content)
-              end
-            end
-          end
+        context 'with Debian components and architectures' do
+          it_behaves_like 'Generate Debian Distribution and component files'
         end
 
         context 'without components and architectures' do
-          it 'updates distribution and component files', :aggregate_failures do
-            travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
-              expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
-
-              expect { subject }
-                .to not_change { Packages::Package.count }
-                .and not_change { Packages::PackageFile.count }
-                .and not_change { distribution.component_files.count }
-
-              expected_release_content = <<~EOF
-              Codename: unstable
-              Date: Sat, 25 Jan 2020 15:17:18 +0000
-              Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
-              MD5Sum:
-              SHA256:
-              EOF
-
-              distribution.file.use_file do |file_path|
-                expect(File.read(file_path)).to eq(expected_release_content)
-              end
-            end
-          end
+          it_behaves_like 'Generate minimal Debian Distribution'
         end
       end
     end
-
-    it_behaves_like 'Generate Distribution', :project
-    it_behaves_like 'Generate Distribution', :group
   end
 end

spec/services/packages/debian/parse_debian822_service_spec.rb

@@ -68,7 +68,7 @@ RSpec.describe Packages::Debian::ParseDebian822Service do
           'Architecture' => 'any',
           'Multi-Arch' => 'same',
           'Depends' => 'libsample0 (= ${binary:Version}), ${misc:Depends}',
-          'Description' => "Some mostly empty developpement files\nUsed in GitLab tests.\n\nTesting another paragraph."
+          'Description' => "Some mostly empty development files\nUsed in GitLab tests.\n\nTesting another paragraph."
         },
         'Package: libsample0' => {
           'Package' => 'libsample0',

spec/services/packages/debian/process_changes_service_spec.rb

@@ -13,6 +13,7 @@ RSpec.describe Packages::Debian::ProcessChangesService do
 
     context 'with valid package file' do
       it 'updates package and package file', :aggregate_failures do
+        expect(::Packages::Debian::GenerateDistributionWorker).to receive(:perform_async).with(:project, distribution.id)
         expect { subject.execute }
           .to change { Packages::Package.count }.from(1).to(2)
           .and not_change { Packages::PackageFile.count }
@@ -30,6 +31,7 @@ RSpec.describe Packages::Debian::ProcessChangesService do
       let(:package_file) { incoming.package_files.first }
 
       it 'raise ExtractionError', :aggregate_failures do
+        expect(::Packages::Debian::GenerateDistributionWorker).not_to receive(:perform_async)
         expect { subject.execute }
           .to not_change { Packages::Package.count }
           .and not_change { Packages::PackageFile.count }
@@ -47,6 +49,7 @@ RSpec.describe Packages::Debian::ProcessChangesService do
       end
 
       it 'remove the package file', :aggregate_failures do
+        expect(::Packages::Debian::GenerateDistributionWorker).not_to receive(:perform_async)
         expect { subject.execute }
           .to not_change { Packages::Package.count }
           .and not_change { Packages::PackageFile.count }

spec/support/shared_contexts/services/packages/debian/distribution_shared_context.rb

+# frozen_string_literal: true
+
+RSpec.shared_context 'with published Debian package' do
+  let_it_be(:group) { create(:group, :public) }
+  let_it_be(:project) { create(:project, :public, group: group) }
+  let_it_be(:project_distribution) { create(:debian_project_distribution, container: project, codename: 'unstable', valid_time_duration_seconds: 48.hours.to_i) }
+  let_it_be(:package) { create(:debian_package, project: project, published_in: project_distribution) }
+end
+
+RSpec.shared_context 'with Debian distribution' do |container_type|
+  let_it_be(:container_type) { container_type }
+
+  if container_type == :project
+    let_it_be(:container) { project }
+    let_it_be(:distribution, reload: true) { project_distribution }
+  else
+    let_it_be(:container) { group }
+    let_it_be(:distribution, reload: true) { create(:debian_group_distribution, container: group, codename: 'unstable', valid_time_duration_seconds: 48.hours.to_i) }
+  end
+end

spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb

@@ -142,6 +142,14 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       end
     end
 
+    describe '.with_architecture' do
+      subject { described_class.with_architecture(architecture1_2) }
+
+      it do
+        expect(subject.to_a).to contain_exactly(component_file_other_architecture)
+      end
+    end
+
     describe '.with_architecture_name' do
       subject { described_class.with_architecture_name(architecture1_2.name) }
 
@@ -166,12 +174,12 @@ RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
       end
     end
 
-    describe '.created_before' do
-      let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, created_at: 4.hours.ago) }
-      let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, created_at: 3.hours.ago) }
-      let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, created_at: 1.hour.ago) }
+    describe '.updated_before' do
+      let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, updated_at: 4.hours.ago) }
+      let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, updated_at: 3.hours.ago) }
+      let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, updated_at: 1.hour.ago) }
 
-      subject { described_class.created_before(2.hours.ago) }
+      subject { described_class.updated_before(2.hours.ago) }
 
       it do
         expect(subject.to_a).to contain_exactly(component_file1, component_file2)

spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'Generate Debian Distribution and component files' do
+  let_it_be(:component_main) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
+  let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
+
+  let_it_be(:architecture_all) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
+  let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
+  let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
+
+  let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T08:00:00Z', file_sha256: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', file_md5: 'd41d8cd98f00b204e9800998ecf8427e', file_fixture: nil, size: 0) } # updated
+  let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   updated_at: '2020-01-24T09:00:00Z', file_sha256: 'a') } # destroyed
+  let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, updated_at: '2020-01-24T10:54:59Z', file_sha256: 'b') } # destroyed, 1 second before last generation
+  let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # kept, last generation
+  let_it_be(:component_file5) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'd') } # kept, last generation
+  let_it_be(:component_file6) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'e') } # kept, less than 1 hour ago
+
+  def check_component_file(release_date, component_name, component_file_type, architecture_name, expected_content)
+    component_file = distribution
+      .component_files
+      .with_component_name(component_name)
+      .with_file_type(component_file_type)
+      .with_architecture_name(architecture_name)
+      .order_updated_asc
+      .last
+
+    expect(component_file).not_to be_nil
+    expect(component_file.updated_at).to eq(release_date)
+
+    unless expected_content.nil?
+      component_file.file.use_file do |file_path|
+        expect(File.read(file_path)).to eq(expected_content)
+      end
+    end
+  end
+
+  it 'generates Debian distribution and component files', :aggregate_failures do
+    current_time = Time.utc(2020, 01, 25, 15, 17, 18, 123456)
+
+    travel_to(current_time) do
+      expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+      initial_count = 6
+      destroyed_count = 2
+      # updated_count = 1
+      created_count = 5
+
+      expect { subject }
+        .to not_change { Packages::Package.count }
+        .and not_change { Packages::PackageFile.count }
+        .and change { distribution.reload.updated_at }.to(current_time.round)
+        .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
+        .and change { component_file1.reload.updated_at }.to(current_time.round)
+
+      debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
+      pool_prefix = "pool/unstable/#{project.id}/p/#{package.name}"
+      expected_main_amd64_content = <<~EOF
+      Package: libsample0
+      Source: #{package.name}
+      Version: #{package.version}
+      Installed-Size: 7
+      Maintainer: #{debs[0].debian_fields['Maintainer']}
+      Architecture: amd64
+      Description: Some mostly empty lib
+       Used in GitLab tests.
+       .
+       Testing another paragraph.
+      Multi-Arch: same
+      Homepage: #{debs[0].debian_fields['Homepage']}
+      Section: libs
+      Priority: optional
+      Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
+      Size: 409600
+      MD5sum: #{debs[0].file_md5}
+      SHA256: #{debs[0].file_sha256}
+
+      Package: sample-dev
+      Source: #{package.name} (#{package.version})
+      Version: 1.2.3~binary
+      Installed-Size: 7
+      Maintainer: #{debs[1].debian_fields['Maintainer']}
+      Architecture: amd64
+      Depends: libsample0 (= 1.2.3~binary)
+      Description: Some mostly empty development files
+       Used in GitLab tests.
+       .
+       Testing another paragraph.
+      Multi-Arch: same
+      Homepage: #{debs[1].debian_fields['Homepage']}
+      Section: libdevel
+      Priority: optional
+      Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
+      Size: 409600
+      MD5sum: #{debs[1].file_md5}
+      SHA256: #{debs[1].file_sha256}
+      EOF
+
+      check_component_file(current_time.round, 'main', :packages, 'all', nil)
+      check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
+      check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
+
+      check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
+      check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
+      check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
+
+      main_amd64_size = expected_main_amd64_content.length
+      main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
+      main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
+
+      contrib_all_size = component_file1.size
+      contrib_all_md5sum = component_file1.file_md5
+      contrib_all_sha256 = component_file1.file_sha256
+
+      expected_release_content = <<~EOF
+      Codename: unstable
+      Date: Sat, 25 Jan 2020 15:17:18 +0000
+      Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+      Architectures: all amd64 arm64
+      Components: contrib main
+      MD5Sum:
+       #{contrib_all_md5sum}        #{contrib_all_size} contrib/binary-all/Packages
+       d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
+       d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
+       d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
+       #{main_amd64_md5sum}     #{main_amd64_size} main/binary-amd64/Packages
+       d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
+      SHA256:
+       #{contrib_all_sha256}        #{contrib_all_size} contrib/binary-all/Packages
+       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
+       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
+       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
+       #{main_amd64_sha256}     #{main_amd64_size} main/binary-amd64/Packages
+       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
+      EOF
+
+      distribution.file.use_file do |file_path|
+        expect(File.read(file_path)).to eq(expected_release_content)
+      end
+    end
+  end
+end
+
+RSpec.shared_examples 'Generate minimal Debian Distribution' do
+  it 'generates minimal distribution', :aggregate_failures do
+    travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
+      expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+      expect { subject }
+        .to not_change { Packages::Package.count }
+        .and not_change { Packages::PackageFile.count }
+        .and not_change { distribution.component_files.reset.count }
+
+      expected_release_content = <<~EOF
+      Codename: unstable
+      Date: Sat, 25 Jan 2020 15:17:18 +0000
+      Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+      MD5Sum:
+      SHA256:
+      EOF
+
+      distribution.file.use_file do |file_path|
+        expect(File.read(file_path)).to eq(expected_release_content)
+      end
+    end
+  end
+end

spec/workers/packages/debian/generate_distribution_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::GenerateDistributionWorker, type: :worker do
+  describe '#perform' do
+    let(:container_type) { distribution.container_type }
+    let(:distribution_id) { distribution.id }
+
+    subject { described_class.new.perform(container_type, distribution_id) }
+
+    include_context 'with published Debian package'
+
+    [:project, :group].each do |container_type|
+      context "for #{container_type}" do
+        include_context 'with Debian distribution', container_type
+
+        context 'with mocked service' do
+          it 'calls GenerateDistributionService' do
+            expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+            expect_next_instance_of(::Packages::Debian::GenerateDistributionService) do |service|
+              expect(service).to receive(:execute)
+                .with(no_args)
+            end
+
+            subject
+          end
+        end
+
+        context 'with non existing distribution id' do
+          let(:distribution_id) { non_existing_record_id }
+
+          it 'returns early without error' do
+            expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+            expect(::Packages::Debian::GenerateDistributionService).not_to receive(:new)
+
+            subject
+          end
+        end
+
+        context 'with nil distribution id' do
+          let(:distribution_id) { nil }
+
+          it 'returns early without error' do
+            expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+            expect(::Packages::Debian::GenerateDistributionService).not_to receive(:new)
+
+            subject
+          end
+        end
+
+        context 'with valid parameters' do
+          it_behaves_like 'an idempotent worker' do
+            let(:job_args) { [container_type, distribution_id] }
+
+            it_behaves_like 'Generate Debian Distribution and component files'
+          end
+        end
+      end
+    end
+  end
+end