E2E: large github import spec logging improvements

qa/Gemfile

@@ -22,8 +22,8 @@ gem 'rotp', '~> 3.1.0'
 gem 'timecop', '~> 0.9.1'
 gem 'parallel', '~> 1.19'
 gem 'rspec-parameterized', '~> 0.4.2'
-gem "octokit", "~> 4.21"
-gem "webdrivers", "~> 4.6"
+gem 'octokit', '~> 4.21'
+gem 'webdrivers', '~> 4.6'
 
 gem 'chemlab', '~> 0.7'
 gem 'chemlab-library-www-gitlab-com', '~> 0.1'

qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb

@@ -8,6 +8,9 @@ module QA
   # Only executes in custom job/pipeline
   RSpec.describe 'Manage', :github, :requires_admin, only: { job: 'large-github-import' } do
     describe 'Project import' do
+      let(:logger) { Runtime::Logger.logger }
+      let(:differ) { RSpec::Support::Differ.new(color: true) }
+
       let(:api_client) { Runtime::API::Client.as_admin }
       let(:group) do
         Resource::Group.fabricate_via_api! do |resource|
@@ -22,12 +25,10 @@ module QA
         end
       end
 
-      let(:differ) { RSpec::Support::Differ.new(color: true) }
-      let(:github_repo) { 'allure-framework/allure-ruby' }
-
+      let(:github_repo) { 'rspec/rspec-core' }
       let(:github_client) do
         Octokit.middleware = Faraday::RackBuilder.new do |builder|
-          builder.response(:logger, Runtime::Logger.logger, headers: false, bodies: false)
+          builder.response(:logger, logger, headers: false, bodies: false)
         end
 
         Octokit::Client.new(access_token: Runtime::Env.github_access_token, auto_paginate: true)
@@ -36,8 +37,16 @@ module QA
       let(:gh_branches) { github_client.branches(github_repo).map(&:name) }
       let(:gh_commits) { github_client.commits(github_repo).map(&:sha) }
       let(:gh_repo) { github_client.repository(github_repo) }
-      let(:gh_labels) { github_client.labels(github_repo) }
-      let(:gh_milestones) { github_client.list_milestones(github_repo, state: 'all') }
+
+      let(:gh_labels) do
+        github_client.labels(github_repo).map { |label| { name: label.name, color: "##{label.color}" } }
+      end
+
+      let(:gh_milestones) do
+        github_client
+          .list_milestones(github_repo, state: 'all')
+          .map { |ms| { title: ms.title, description: ms.description } }
+      end
 
       let(:gh_all_issues) do
         github_client.list_issues(github_repo, state: 'all')
@@ -88,6 +97,31 @@ module QA
         group.add_member(user, Resource::Members::AccessLevel::MAINTAINER)
       end
 
+      after do
+        # save data for comparison after run finished
+        save_json(
+          "data",
+          {
+            github: {
+              branches: gh_branches,
+              commits: gh_commits,
+              labels: gh_labels,
+              milestones: gh_milestones,
+              prs: gh_prs,
+              issues: gh_issues
+            },
+            gitlab: {
+              branches: gl_branches,
+              commits: gl_commits,
+              labels: gl_labels,
+              milestones: gl_milestones,
+              mrs: mrs,
+              issues: gl_issues
+            }
+          }.to_json
+        )
+      end
+
       it 'imports large Github repo via api' do
         imported_project # import the project
         fetch_github_objects # fetch all objects right after import has started
@@ -99,10 +133,10 @@ module QA
 
         aggregate_failures do
           verify_repository_import
-          verify_merge_requests_import
-          verify_issues_import
           verify_labels_import
           verify_milestones_import
+          verify_merge_requests_import
+          verify_issues_import
         end
       end
 
@@ -110,7 +144,7 @@ module QA
       #
       # @return [void]
       def fetch_github_objects
-        Runtime::Logger.debug("Fetching objects for github repo: '#{github_repo}'")
+        logger.debug("Fetching objects for github repo: '#{github_repo}'")
 
         gh_repo
         gh_branches
@@ -125,50 +159,44 @@ module QA
       #
       # @return [void]
       def verify_repository_import
-        branches = imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] }
-        commits = imported_project.commits(auto_paginate: true).map { |c| c[:id] }
-
+        logger.debug("Verifying repository import")
         expect(imported_project.description).to eq(gh_repo.description)
         # check via include, importer creates more branches
         # https://gitlab.com/gitlab-org/gitlab/-/issues/332711
-        expect(branches).to include(*gh_branches)
-        expect(commits).to match_array(gh_commits)
+        expect(gl_branches).to include(*gh_branches)
+        expect(gl_commits).to match_array(gh_commits)
       end
 
       # Verify imported merge requests and mr issues
       #
       # @return [void]
       def verify_merge_requests_import
-        verify_mrs_or_issues('mrs')
+        logger.debug("Verifying merge request import")
+        verify_mrs_or_issues('mr')
       end
 
       # Verify imported issues and issue comments
       #
       # @return [void]
       def verify_issues_import
-        verify_mrs_or_issues('issues')
+        logger.debug("Verifying issue import")
+        verify_mrs_or_issues('issue')
       end
 
       # Verify imported labels
       #
       # @return [void]
       def verify_labels_import
-        labels = imported_project.labels(auto_paginate: true).map { |label| label.slice(:name, :color) }
-        actual_labels = gh_labels.map { |label| { name: label.name, color: "##{label.color}" } }
-
-        expect(labels.length).to eq(actual_labels.length)
-        expect(labels).to match_array(actual_labels)
+        logger.debug("Verifying label import")
+        expect(gl_labels).to match_array(gh_labels)
       end
 
       # Verify milestones import
       #
       # @return [void]
       def verify_milestones_import
-        milestones = imported_project.milestones(auto_paginate: true).map { |ms| ms.slice(:title, :description) }
-        actual_milestones = gh_milestones.map { |ms| { title: ms.title, description: ms.description } }
-
-        expect(milestones.length).to eq(actual_milestones.length)
-        expect(milestones).to match_array(actual_milestones)
+        logger.debug("Verifying milestones import")
+        expect(gl_milestones).to match_array(gh_milestones)
       end
 
       private
@@ -179,11 +207,17 @@ module QA
       # @return [void]
       def verify_mrs_or_issues(type)
         msg = ->(title) { "expected #{type} with title '#{title}' to have" }
-        expected = type == 'mrs' ? mrs : gl_issues
-        actual = type == 'mrs' ? gh_prs : gh_issues
+        expected = type == 'mr' ? mrs : gl_issues
+        actual = type == 'mr' ? gh_prs : gh_issues
 
-        expect(expected.keys).to match_array(actual.keys)
+        # Compare length to have easy to read overview how many objects are missing
+        expect(expected.length).to(
+          eq(actual.length),
+          "Expected to contain same amount of #{type}s. Expected: #{expected.length}, actual: #{actual.length}"
+        )
         actual.each do |title, actual_item|
+          logger.debug("Comparing #{type} with title '#{title}'")
+
           expected_item = expected[title]
 
           expect(expected_item).to be_truthy, "#{msg.call(title)} been imported"
@@ -191,7 +225,7 @@ module QA
 
           expect(expected_item[:body]).to(
             include(actual_item[:body]),
-            "#{msg.call(title)} same description. #{diff(expected_item[:body], actual_item[:body])}"
+            "#{msg.call(title)} same description. diff:\n#{differ.diff(expected_item[:body], actual_item[:body])}"
           )
           expect(expected_item[:comments].length).to(
             eq(actual_item[:comments].length),
@@ -201,14 +235,44 @@ module QA
         end
       end
 
+      # Imported project branches
+      #
+      # @return [Array]
+      def gl_branches
+        @gl_branches ||= imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] }
+      end
+
+      # Imported project commits
+      #
+      # @return [Array]
+      def gl_commits
+        @gl_commits ||= imported_project.commits(auto_paginate: true).map { |c| c[:id] }
+      end
+
+      # Imported project labels
+      #
+      # @return [Array]
+      def gl_labels
+        @gl_labels ||= imported_project.labels(auto_paginate: true).map { |label| label.slice(:name, :color) }
+      end
+
+      # Imported project milestones
+      #
+      # @return [<Type>] <description>
+      def gl_milestones
+        @gl_milestones ||= imported_project.milestones(auto_paginate: true).map { |ms| ms.slice(:title, :description) }
+      end
+
       # Imported project merge requests
       #
       # @return [Hash]
       def mrs
         @mrs ||= begin
+          logger.debug("Fetching merge requests")
           imported_mrs = imported_project.merge_requests(auto_paginate: true)
           # fetch comments in parallel since we need to do it for each mr separately
-          mrs_hashes = Parallel.map(imported_mrs, in_processes: 5) do |mr|
+          logger.debug("Transforming merge request objects for comparison")
+          mrs_hashes = Parallel.map(imported_mrs) do |mr|
             resource = Resource::MergeRequest.init do |resource|
               resource.project = imported_project
               resource.iid = mr[:iid]
@@ -239,9 +303,11 @@ module QA
       # @return [Hash]
       def gl_issues
         @gl_issues ||= begin
+          logger.debug("Fetching issues")
           imported_issues = imported_project.issues(auto_paginate: true)
           # fetch comments in parallel since we need to do it for each mr separately
-          issue_hashes = Parallel.map(imported_issues, in_processes: 5) do |issue|
+          logger.debug("Transforming issue objects for comparison")
+          issue_hashes = Parallel.map(imported_issues) do |issue|
             resource = Resource::Issue.init do |issue_resource|
               issue_resource.project = imported_project
               issue_resource.iid = issue[:iid]
@@ -272,13 +338,13 @@ module QA
         body.gsub(/\*Created by: \S+\*\n\n/, "")
       end
 
-      # Diff of 2 objects
+      # Save json as file
       #
-      # @param [Object] actual
-      # @param [Object] expected
-      # @return [String]
-      def diff(actual, expected)
-        "diff:\n#{differ.diff(actual, expected)}"
+      # @param [String] name
+      # @param [String] json
+      # @return [void]
+      def save_json(name, json)
+        File.open("tmp/#{name}.json", "w") { |file| file.write(json) }
       end
     end
   end

qa/spec/support/matchers/eventually_matcher.rb

@@ -53,11 +53,15 @@ module Matchers
       # @param [Symbol] expectation_name
       # @return [Boolean]
       def wait_and_check(actual, expectation_name)
+        attempt = 0
+
         QA::Support::Retrier.retry_until(
           max_attempts: @attempts,
           max_duration: @duration,
           sleep_interval: @interval || 0.5
         ) do
+          QA::Runtime::Logger.debug("Evaluating expectation '#{operator_msg}', attempt: #{attempt += 1}")
+
           public_send(expectation_name, actual)
         rescue RSpec::Expectations::ExpectationNotMetError, QA::Resource::ApiFabricator::ResourceNotFoundError
           false