Merge branch 'debian_extract_changes_metadata_service' into 'master'

Add Packages::Debian:ExtractChangesMetadataService

See merge request gitlab-org/gitlab!55896

app/models/packages/debian/file_entry.rb

+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class FileEntry
+      include ActiveModel::Model
+
+      DIGESTS = %i[md5 sha1 sha256].freeze
+      FILENAME_REGEX = %r{\A[a-zA-Z0-9][a-zA-Z0-9_.~+-]*\z}.freeze
+
+      attr_accessor :filename,
+                    :size,
+                    :md5sum,
+                    :section,
+                    :priority,
+                    :sha1sum,
+                    :sha256sum,
+                    :package_file
+
+      validates :filename, :size, :md5sum, :section, :priority, :sha1sum, :sha256sum, :package_file, presence: true
+      validates :filename, format: { with: FILENAME_REGEX }
+      validate :valid_package_file_digests, if: -> { md5sum.present? && sha1sum.present? && sha256sum.present? && package_file.present? }
+
+      def component
+        return 'main' if section.blank?
+        return 'main' unless section.include?('/')
+
+        section.split('/')[0]
+      end
+
+      private
+
+      def valid_package_file_digests
+        DIGESTS.each do |digest|
+          package_file_digest = package_file["file_#{digest}"]
+          sum = public_send("#{digest}sum") # rubocop:disable GitlabSecurity/PublicSend
+          next if package_file_digest == sum
+
+          errors.add("#{digest}sum".to_sym, "mismatch for #{filename}: #{package_file_digest} != #{sum}")
+        end
+      end
+    end
+  end
+end

app/services/packages/debian/extract_changes_metadata_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    class ExtractChangesMetadataService
+      include Gitlab::Utils::StrongMemoize
+
+      ExtractionError = Class.new(StandardError)
+
+      def initialize(package_file)
+        @package_file = package_file
+        @entries = {}
+      end
+
+      def execute
+        {
+          file_type: file_type,
+          architecture: metadata[:architecture],
+          fields: fields,
+          files: files
+        }
+      rescue ActiveModel::ValidationError => e
+        raise ExtractionError.new(e.message)
+      end
+
+      private
+
+      def metadata
+        strong_memoize(:metadata) do
+          ::Packages::Debian::ExtractMetadataService.new(@package_file).execute
+        end
+      end
+
+      def file_type
+        metadata[:file_type]
+      end
+
+      def fields
+        metadata[:fields]
+      end
+
+      def files
+        strong_memoize(:files) do
+          raise ExtractionError.new("is not a changes file") unless file_type == :changes
+          raise ExtractionError.new("Files field is missing") if fields[:Files].blank?
+          raise ExtractionError.new("Checksums-Sha1 field is missing") if fields[:'Checksums-Sha1'].blank?
+          raise ExtractionError.new("Checksums-Sha256 field is missing") if fields[:'Checksums-Sha256'].blank?
+
+          init_entries_from_files
+          entries_from_checksums_sha1
+          entries_from_checksums_sha256
+          entries_from_package_files
+
+          @entries
+        end
+      end
+
+      def init_entries_from_files
+        each_lines_for(:Files) do |line|
+          md5sum, size, section, priority, filename = line.split
+          entry = FileEntry.new(
+            filename: filename,
+            size: size.to_i,
+            md5sum: md5sum,
+            section: section,
+            priority: priority)
+
+          @entries[filename] = entry
+        end
+      end
+
+      def entries_from_checksums_sha1
+        each_lines_for(:'Checksums-Sha1') do |line|
+          sha1sum, size, filename = line.split
+          entry = @entries[filename]
+          raise ExtractionError.new("#{filename} is listed in Checksums-Sha1 but not in Files") unless entry
+          raise ExtractionError.new("Size for #{filename} in Files and Checksums-Sha1 differ") unless entry.size == size.to_i
+
+          entry.sha1sum = sha1sum
+        end
+      end
+
+      def entries_from_checksums_sha256
+        each_lines_for(:'Checksums-Sha256') do |line|
+          sha256sum, size, filename = line.split
+          entry = @entries[filename]
+          raise ExtractionError.new("#{filename} is listed in Checksums-Sha256 but not in Files") unless entry
+          raise ExtractionError.new("Size for #{filename} in Files and Checksums-Sha256 differ") unless entry.size == size.to_i
+
+          entry.sha256sum = sha256sum
+        end
+      end
+
+      def each_lines_for(field)
+        fields[field].split("\n").each do |line|
+          next if line.blank?
+
+          yield(line)
+        end
+      end
+
+      def entries_from_package_files
+        @entries.each do |filename, entry|
+          entry.package_file = ::Packages::PackageFileFinder.new(@package_file.package, filename).execute!
+          entry.validate!
+        rescue ActiveRecord::RecordNotFound
+          raise ExtractionError.new("#{filename} is listed in Files but was not uploaded")
+        end
+      end
+    end
+  end
+end

spec/factories/packages/package_file.rb

@@ -125,6 +125,9 @@ FactoryBot.define do
 
       trait(:source) do
         file_name { 'sample_1.2.3~alpha2.tar.xz' }
+        file_md5 { 'd79b34f58f61ff4ad696d9bd0b8daa68' }
+        file_sha1 { '5f8bba5574eb01ac3b1f5e2988e8c29307788236' }
+        file_sha256 { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362' }
 
         transient do
           file_metadatum_trait { :source }
@@ -133,6 +136,9 @@ FactoryBot.define do
 
       trait(:dsc) do
         file_name { 'sample_1.2.3~alpha2.dsc' }
+        file_md5 { '3b0817804f669e16cdefac583ad88f0e' }
+        file_sha1 { '32ecbd674f0bfd310df68484d87752490685a8d6' }
+        file_sha256 { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba' }
 
         transient do
           file_metadatum_trait { :dsc }
@@ -141,6 +147,9 @@ FactoryBot.define do
 
       trait(:deb) do
         file_name { 'libsample0_1.2.3~alpha2_amd64.deb' }
+        file_md5 { 'fb0842b21adc44207996296fe14439dd' }
+        file_sha1 { '5248b95600e85bfe7f63c0dfce330a75f5777366' }
+        file_sha256 { '1c383a525bfcba619c7305ccd106d61db501a6bbaf0003bf8d0c429fbdb7fcc1' }
 
         transient do
           file_metadatum_trait { :deb }
@@ -149,6 +158,9 @@ FactoryBot.define do
 
       trait(:deb2) do
         file_name { 'sample-dev_1.2.3~binary_amd64.deb' }
+        file_md5 { 'd2afbd28e4d74430d22f9504e18bfdf5' }
+        file_sha1 { 'f81e4f66c8c6bb899653a3340c157965ee69634f' }
+        file_sha256 { '9fbeee2191ce4dab5288fad5ecac1bd369f58fef9a992a880eadf0caf25f086d' }
 
         transient do
           file_metadatum_trait { :deb }
@@ -157,6 +169,9 @@ FactoryBot.define do
 
       trait(:udeb) do
         file_name { 'sample-udeb_1.2.3~alpha2_amd64.udeb' }
+        file_md5 { '72b1dd7d98229e2fb0355feda1d3a165' }
+        file_sha1 { 'e42e8f2fe04ed1bb73b44a187674480d0e49dcba' }
+        file_sha256 { '2b0c152b3ab4cc07663350424de972c2b7621d69fe6df2e0b94308a191e4632f' }
 
         transient do
           file_metadatum_trait { :udeb }
@@ -165,6 +180,9 @@ FactoryBot.define do
 
       trait(:buildinfo) do
         file_name { 'sample_1.2.3~alpha2_amd64.buildinfo' }
+        file_md5 { '4e085dd67c120ca967ec314f65770a42' }
+        file_sha1 { '0d47e899f3cc67a2253a4629456ff927e0db5c60' }
+        file_sha256 { 'f9900d3c94e94b329232668dcbef3dba2d96c07147b15b6dc0533452e4dd8a43' }
 
         transient do
           file_metadatum_trait { :buildinfo }

spec/models/packages/debian/file_entry_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::FileEntry, type: :model do
+  let_it_be(:package_file) { create(:debian_package_file, :dsc) }
+
+  let(:filename) { 'sample_1.2.3~alpha2.dsc' }
+  let(:size) { 671 }
+  let(:md5sum) { '3b0817804f669e16cdefac583ad88f0e' }
+  let(:section) { 'libs' }
+  let(:priority) { 'optional' }
+  let(:sha1sum) { '32ecbd674f0bfd310df68484d87752490685a8d6' }
+  let(:sha256sum) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba' }
+
+  let(:file_entry) do
+    described_class.new(
+      filename: filename,
+      size: size,
+      md5sum: md5sum,
+      section: section,
+      priority: priority,
+      sha1sum: sha1sum,
+      sha256sum: sha256sum,
+      package_file: package_file
+    )
+  end
+
+  subject { file_entry }
+
+  describe 'validations' do
+    it { is_expected.to be_valid }
+
+    describe '#filename' do
+      it { is_expected.to validate_presence_of(:filename) }
+      it { is_expected.not_to allow_value('Hé').for(:filename) }
+    end
+
+    describe '#size' do
+      it { is_expected.to validate_presence_of(:size) }
+    end
+
+    describe '#md5sum' do
+      it { is_expected.to validate_presence_of(:md5sum) }
+      it { is_expected.not_to allow_value('12345678901234567890123456789012').for(:md5sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 3b0817804f669e16cdefac583ad88f0e != 12345678901234567890123456789012') }
+    end
+
+    describe '#section' do
+      it { is_expected.to validate_presence_of(:section) }
+    end
+
+    describe '#priority' do
+      it { is_expected.to validate_presence_of(:priority) }
+    end
+
+    describe '#sha1sum' do
+      it { is_expected.to validate_presence_of(:sha1sum) }
+      it { is_expected.not_to allow_value('1234567890123456789012345678901234567890').for(:sha1sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 32ecbd674f0bfd310df68484d87752490685a8d6 != 1234567890123456789012345678901234567890') }
+    end
+
+    describe '#sha256sum' do
+      it { is_expected.to validate_presence_of(:sha256sum) }
+      it { is_expected.not_to allow_value('1234567890123456789012345678901234567890123456789012345678901234').for(:sha256sum).with_message('mismatch for sample_1.2.3~alpha2.dsc: 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba != 1234567890123456789012345678901234567890123456789012345678901234') }
+    end
+
+    describe '#package_file' do
+      it { is_expected.to validate_presence_of(:package_file) }
+    end
+  end
+
+  describe '#component' do
+    subject { file_entry.component }
+
+    context 'without section' do
+      let(:section) { nil }
+
+      it { is_expected.to eq 'main' }
+    end
+
+    context 'with empty section' do
+      let(:section) { '' }
+
+      it { is_expected.to eq 'main' }
+    end
+
+    context 'with ruby section' do
+      let(:section) { 'ruby' }
+
+      it { is_expected.to eq 'main' }
+    end
+
+    context 'with contrib/ruby section' do
+      let(:section) { 'contrib/ruby' }
+
+      it { is_expected.to eq 'contrib' }
+    end
+  end
+end

spec/services/packages/debian/extract_changes_metadata_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::ExtractChangesMetadataService do
+  describe '#execute' do
+    let_it_be(:distribution) { create(:debian_project_distribution, codename: 'unstable') }
+    let_it_be(:incoming) { create(:debian_incoming, project: distribution.project) }
+
+    let(:package_file) { incoming.package_files.last }
+    let(:service) { described_class.new(package_file) }
+
+    subject { service.execute }
+
+    context 'with valid package file' do
+      it 'extract metadata', :aggregate_failures do
+        expected_fields = { 'Architecture': 'source amd64', 'Binary': 'libsample0 sample-dev sample-udeb' }
+
+        expect(subject[:file_type]).to eq(:changes)
+        expect(subject[:architecture]).to be_nil
+        expect(subject[:fields]).to include(expected_fields)
+        expect(subject[:files].count).to eq(6)
+      end
+    end
+
+    context 'with invalid package file' do
+      let(:package_file) { incoming.package_files.first }
+
+      it 'raise ArgumentError', :aggregate_failures do
+        expect { subject }.to raise_error(described_class::ExtractionError, "is not a changes file")
+      end
+    end
+
+    context 'with invalid metadata' do
+      let(:md5_dsc) { '3b0817804f669e16cdefac583ad88f0e 671 libs optional sample_1.2.3~alpha2.dsc' }
+      let(:md5_source) { 'd79b34f58f61ff4ad696d9bd0b8daa68 864 libs optional sample_1.2.3~alpha2.tar.xz' }
+      let(:md5s) { "#{md5_dsc}\n#{md5_source}" }
+      let(:sha1_dsc) { '32ecbd674f0bfd310df68484d87752490685a8d6 671 sample_1.2.3~alpha2.dsc' }
+      let(:sha1_source) { '5f8bba5574eb01ac3b1f5e2988e8c29307788236 864 sample_1.2.3~alpha2.tar.xz' }
+      let(:sha1s) { "#{sha1_dsc}\n#{sha1_source}" }
+      let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc' }
+      let(:sha256_source) { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz' }
+      let(:sha256s) { "#{sha256_dsc}\n#{sha256_source}" }
+      let(:fields) { { Files: md5s, 'Checksums-Sha1': sha1s, 'Checksums-Sha256': sha256s } }
+      let(:metadata) { { file_type: :changes, architecture: 'amd64', fields: fields } }
+
+      before do
+        allow_next_instance_of(::Packages::Debian::ExtractMetadataService) do |extract_metadata_service|
+          allow(extract_metadata_service).to receive(:execute).and_return(metadata)
+        end
+      end
+
+      context 'without Files field' do
+        let(:md5s) { nil }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Files field is missing")
+        end
+      end
+
+      context 'without Checksums-Sha1 field' do
+        let(:sha1s) { nil }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Checksums-Sha1 field is missing")
+        end
+      end
+
+      context 'without Checksums-Sha256 field' do
+        let(:sha256s) { nil }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Checksums-Sha256 field is missing")
+        end
+      end
+
+      context 'with file in Checksums-Sha1 but not in Files' do
+        let(:md5_dsc) { '' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "sample_1.2.3~alpha2.dsc is listed in Checksums-Sha1 but not in Files")
+        end
+      end
+
+      context 'with different size in Checksums-Sha1' do
+        let(:sha1_dsc) { '32ecbd674f0bfd310df68484d87752490685a8d6 42 sample_1.2.3~alpha2.dsc' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Size for sample_1.2.3~alpha2.dsc in Files and Checksums-Sha1 differ")
+        end
+      end
+
+      context 'with file in Checksums-Sha256 but not in Files' do
+        let(:md5_dsc) { '' }
+        let(:sha1_dsc) { '' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "sample_1.2.3~alpha2.dsc is listed in Checksums-Sha256 but not in Files")
+        end
+      end
+
+      context 'with different size in Checksums-Sha256' do
+        let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 42 sample_1.2.3~alpha2.dsc' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Size for sample_1.2.3~alpha2.dsc in Files and Checksums-Sha256 differ")
+        end
+      end
+
+      context 'with file in Files but not in Checksums-Sha1' do
+        let(:sha1_dsc) { '' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha1sum can't be blank")
+        end
+      end
+
+      context 'with file in Files but not in Checksums-Sha256' do
+        let(:sha256_dsc) { '' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha256sum can't be blank")
+        end
+      end
+
+      context 'with invalid MD5' do
+        let(:md5_dsc) { '1234567890123456789012345678012 671 libs optional sample_1.2.3~alpha2.dsc' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Md5sum mismatch for sample_1.2.3~alpha2.dsc: 3b0817804f669e16cdefac583ad88f0e != 1234567890123456789012345678012")
+        end
+      end
+
+      context 'with invalid SHA1' do
+        let(:sha1_dsc) { '1234567890123456789012345678901234567890 671 sample_1.2.3~alpha2.dsc' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha1sum mismatch for sample_1.2.3~alpha2.dsc: 32ecbd674f0bfd310df68484d87752490685a8d6 != 1234567890123456789012345678901234567890")
+        end
+      end
+
+      context 'with invalid SHA256' do
+        let(:sha256_dsc) { '1234567890123456789012345678901234567890123456789012345678901234 671 sample_1.2.3~alpha2.dsc' }
+
+        it 'raise ArgumentError', :aggregate_failures do
+          expect { subject }.to raise_error(described_class::ExtractionError, "Validation failed: Sha256sum mismatch for sample_1.2.3~alpha2.dsc: 844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba != 1234567890123456789012345678901234567890123456789012345678901234")
+        end
+      end
+    end
+
+    context 'with missing package file' do
+      before do
+        incoming.package_files.first.destroy!
+      end
+
+      it 'raise ArgumentError' do
+        expect { subject }.to raise_error(described_class::ExtractionError, "sample_1.2.3~alpha2.tar.xz is listed in Files but was not uploaded")
+      end
+    end
+  end
+end