Merge branch 'caalberts-dynamic-map-crystalball' into 'master'

Generate spec mapping with crystalball using test coverage

See merge request gitlab-org/gitlab!44968

.gitignore

@@ -73,6 +73,7 @@ eslint-report.html
 /.gitlab_pages_secret
 /.gitlab_kas_secret
 /webpack-report/
+/crystalball/
 /knapsack/
 /rspec_flaky/
 /locale/**/LC_MESSAGES

.gitlab-ci.yml

@@ -59,6 +59,8 @@ variables:
   GET_SOURCES_ATTEMPTS: "3"
   KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json
   FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
+  RSPEC_TESTS_MAPPING_PATH: crystalball/mapping.json
+  RSPEC_PACKED_TESTS_MAPPING_PATH: crystalball/packed-mapping.json
   BUILD_ASSETS_IMAGE: "false"
   ES_JAVA_OPTS: "-Xms256m -Xmx256m"
   ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200"

.gitlab/ci/rails.gitlab-ci.yml

@@ -20,6 +20,7 @@
   variables:
     RUBY_GC_MALLOC_LIMIT: 67108864
     RUBY_GC_MALLOC_LIMIT_MAX: 134217728
+    CRYSTALBALL: "true"
   needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
   script:
     - *base-script
@@ -29,6 +30,7 @@
     when: always
     paths:
       - coverage/
+      - crystalball/
       - knapsack/
       - rspec_flaky/
       - rspec_profiling/

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -9,6 +9,7 @@
       - knapsack/
       - rspec_flaky/
       - rspec_profiling/
+      - crystalball/
 
 retrieve-tests-metadata:
   extends:
@@ -41,3 +42,4 @@ update-tests-metadata:
     - run_timed_command "retry gem install bundler:1.17.3 fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document"
     - source ./scripts/rspec_helpers.sh
     - update_tests_metadata
+    - update_tests_mapping

Gemfile

@@ -386,6 +386,7 @@ group :development, :test do
   gem 'benchmark-ips', '~> 2.3.0', require: false
 
   gem 'knapsack', '~> 1.17'
+  gem 'crystalball', '~> 0.7.0', require: false
 
   gem 'simple_po_parser', '~> 1.1.2', require: false
 

Gemfile.lock

@@ -199,6 +199,8 @@ GEM
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
     creole (0.5.0)
+    crystalball (0.7.0)
+      git
     css_parser (1.7.0)
       addressable
     daemons (1.2.6)
@@ -1291,6 +1293,7 @@ DEPENDENCIES
   connection_pool (~> 2.0)
   countries (~> 3.0)
   creole (~> 0.5.0)
+  crystalball (~> 0.7.0)
   danger (~> 8.0.6)
   database_cleaner (~> 1.7.0)
   deckar01-task_list (= 2.3.1)

scripts/generate-test-mapping

+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_generator'
+
+test_mapping_json = ARGV.shift
+crystalball_yamls = ARGV
+
+unless test_mapping_json && !crystalball_yamls.empty?
+  puts "usage: #{__FILE__} <test_mapping_json> [crystalball_yamls...]"
+  exit 1
+end
+
+map_generator = Tooling::TestMapGenerator.new
+map_generator.parse(crystalball_yamls)
+mapping = map_generator.mapping
+
+File.write(test_mapping_json, JSON.pretty_generate(mapping))
+puts "Saved #{test_mapping_json}."

scripts/pack-test-mapping

+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_packer'
+
+unpacked_json_mapping, packed_json_mapping = ARGV.shift(2)
+unless packed_json_mapping && unpacked_json_mapping
+  puts "usage: #{__FILE__} <unpacked_json_mapping> <packed_json_mapping>"
+  exit 1
+end
+
+puts "Compressing #{unpacked_json_mapping}"
+
+mapping = JSON.parse(File.read(unpacked_json_mapping))
+packed_mapping = Tooling::TestMapPacker.new.pack(mapping)
+
+puts "Writing packed #{packed_json_mapping}"
+File.write(packed_json_mapping, JSON.generate(packed_mapping))
+puts "Saved #{packed_json_mapping}."

scripts/rspec_helpers.sh

@@ -48,6 +48,43 @@ function update_tests_metadata() {
   fi
 }
 
+function retrieve_tests_mapping() {
+  mkdir -p crystalball/
+
+  if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
+    (wget -O "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+  fi
+
+  scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
+}
+
+function update_tests_mapping() {
+  if ! crystalball_rspec_data_exists; then
+    echo "No crystalball rspec data found."
+    return 0
+  fi
+
+  scripts/generate-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" crystalball/rspec*.yml
+
+  scripts/pack-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+  gzip "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
+
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz"
+    else
+      echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+    fi
+  fi
+
+  rm -f crystalball/rspec*.yml
+}
+
+function crystalball_rspec_data_exists() {
+  compgen -G "crystalball/rspec*.yml" > /dev/null;
+}
+
 function rspec_simple_job() {
   local rspec_opts="${1}"
 

scripts/unpack-test-mapping

+#!/usr/bin/env ruby
+
+require 'json'
+require_relative '../tooling/lib/tooling/test_map_packer'
+
+packed_json_mapping, unpacked_json_mapping = ARGV.shift(2)
+unless packed_json_mapping && unpacked_json_mapping
+  puts "usage: #{__FILE__} <packed_json_mapping> <unpacked_json_mapping>"
+  exit 1
+end
+
+packed_mapping = JSON.parse(File.read(packed_json_mapping))
+mapping = Tooling::TestMapPacker.new.unpack(packed_mapping)
+
+puts "Writing unpacked #{unpacked_json_mapping}"
+File.write(unpacked_json_mapping, JSON.generate(mapping))
+puts "Saved #{unpacked_json_mapping}."

spec/crystalball_env.rb

+# frozen_string_literal: true
+
+module CrystalballEnv
+  EXCLUDED_PREFIXES = %w[vendor/ruby].freeze
+
+  extend self
+
+  def start!
+    return unless ENV['CRYSTALBALL'] && ENV['CI_PIPELINE_SOURCE'] == 'schedule'
+
+    require 'crystalball'
+    require_relative '../tooling/lib/tooling/crystalball/coverage_lines_execution_detector'
+    require_relative '../tooling/lib/tooling/crystalball/coverage_lines_strategy'
+
+    map_storage_path_base = ENV['CI_JOB_NAME'] || 'crystalball_data'
+    map_storage_path = "crystalball/#{map_storage_path_base.gsub(%r{[/ ]}, '_')}.yml"
+
+    execution_detector = Tooling::Crystalball::CoverageLinesExecutionDetector.new(exclude_prefixes: EXCLUDED_PREFIXES)
+
+    Crystalball::MapGenerator.start! do |config|
+      config.map_storage_path = map_storage_path
+      config.register Tooling::Crystalball::CoverageLinesStrategy.new(execution_detector)
+    end
+  end
+end

spec/spec_helper.rb

@@ -3,6 +3,9 @@
 require './spec/simplecov_env'
 SimpleCovEnv.start!
 
+require './spec/crystalball_env'
+CrystalballEnv.start!
+
 ENV["RAILS_ENV"] = 'test'
 ENV["IN_MEMORY_APPLICATION_SETTINGS"] = 'true'
 ENV["RSPEC_ALLOW_INVALID_URLS"] = 'true'

spec/tooling/lib/tooling/crystalball/coverage_lines_execution_detector_spec.rb

+# frozen_string_literal: true
+
+require_relative '../../../../../tooling/lib/tooling/crystalball/coverage_lines_execution_detector'
+
+RSpec.describe Tooling::Crystalball::CoverageLinesExecutionDetector do
+  subject(:detector) { described_class.new(root, exclude_prefixes: %w[vendor/ruby]) }
+
+  let(:root) { '/tmp' }
+  let(:before_map) { { path => { lines: [0, 2, nil] } } }
+  let(:after_map) { { path => { lines: [0, 3, nil] } } }
+  let(:path) { '/tmp/file.rb' }
+
+  describe '#detect' do
+    subject { detector.detect(before_map, after_map) }
+
+    it { is_expected.to eq(%w[file.rb]) }
+
+    context 'with no changes' do
+      let(:after_map) { { path => { lines: [0, 2, nil] } } }
+
+      it { is_expected.to eq([]) }
+    end
+
+    context 'with previously uncovered file' do
+      let(:before_map) { {} }
+
+      it { is_expected.to eq(%w[file.rb]) }
+    end
+
+    context 'with path outside of root' do
+      let(:path) { '/abc/file.rb' }
+
+      it { is_expected.to eq([]) }
+    end
+
+    context 'with path in excluded prefix' do
+      let(:path) { '/tmp/vendor/ruby/dependency.rb' }
+
+      it { is_expected.to eq([]) }
+    end
+  end
+end

spec/tooling/lib/tooling/crystalball/coverage_lines_strategy_spec.rb

+# frozen_string_literal: true
+
+require_relative '../../../../../tooling/lib/tooling/crystalball/coverage_lines_strategy'
+
+RSpec.describe Tooling::Crystalball::CoverageLinesStrategy do
+  subject { described_class.new(execution_detector) }
+
+  let(:execution_detector) { instance_double('Tooling::Crystalball::CoverageLinesExecutionDetector') }
+
+  describe '#after_register' do
+    it 'starts coverage' do
+      expect(Coverage).to receive(:start).with(lines: true)
+      subject.after_register
+    end
+  end
+end

spec/tooling/lib/tooling/test_map_generator_spec.rb

+# frozen_string_literal: true
+
+require_relative '../../../../tooling/lib/tooling/test_map_generator'
+
+RSpec.describe Tooling::TestMapGenerator do
+  subject { described_class.new }
+
+  describe '#parse' do
+    let(:yaml1) do
+      <<~YAML
+        ---
+        :type: Crystalball::ExecutionMap
+        :commit: a7d57d333042f3b0334b2f8a282354eef7365976
+        :timestamp: 1602668405
+        :version:
+        ---
+        "./spec/factories_spec.rb[1]":
+        - lib/gitlab/current_settings.rb
+        - lib/feature.rb
+        - lib/gitlab/marginalia.rb
+      YAML
+    end
+
+    let(:yaml2) do
+      <<~YAML
+        ---
+        :type: Crystalball::ExecutionMap
+        :commit: 74056e8d9cf3773f43faa1cf5416f8779c8284c8
+        :timestamp: 1602671965
+        :version:
+        ---
+        "./spec/models/project_spec.rb[1]":
+        - lib/gitlab/current_settings.rb
+        - lib/feature.rb
+        - lib/gitlab/marginalia.rb
+      YAML
+    end
+
+    let(:pathname) { instance_double(Pathname) }
+
+    before do
+      allow(File).to receive(:read).with('yaml1.yml').and_return(yaml1)
+      allow(File).to receive(:read).with('yaml2.yml').and_return(yaml2)
+    end
+
+    context 'with single yaml' do
+      let(:expected_mapping) do
+        {
+          'lib/gitlab/current_settings.rb' => [
+            './spec/factories_spec.rb'
+          ],
+          'lib/feature.rb' => [
+            './spec/factories_spec.rb'
+          ],
+          'lib/gitlab/marginalia.rb' => [
+            './spec/factories_spec.rb'
+          ]
+        }
+      end
+
+      it 'parses crystalball data into test mapping' do
+        subject.parse('yaml1.yml')
+
+        expect(subject.mapping.keys).to match_array(expected_mapping.keys)
+      end
+
+      it 'stores test files without example uid' do
+        subject.parse('yaml1.yml')
+
+        expected_mapping.each do |file, tests|
+          expect(subject.mapping[file]).to match_array(tests)
+        end
+      end
+    end
+
+    context 'with multiple yamls' do
+      let(:expected_mapping) do
+        {
+          'lib/gitlab/current_settings.rb' => [
+            './spec/factories_spec.rb',
+            './spec/models/project_spec.rb'
+          ],
+          'lib/feature.rb' => [
+            './spec/factories_spec.rb',
+            './spec/models/project_spec.rb'
+          ],
+          'lib/gitlab/marginalia.rb' => [
+            './spec/factories_spec.rb',
+            './spec/models/project_spec.rb'
+          ]
+        }
+      end
+
+      it 'parses crystalball data into test mapping' do
+        subject.parse(%w[yaml1.yml yaml2.yml])
+
+        expect(subject.mapping.keys).to match_array(expected_mapping.keys)
+      end
+
+      it 'stores test files without example uid' do
+        subject.parse(%w[yaml1.yml yaml2.yml])
+
+        expected_mapping.each do |file, tests|
+          expect(subject.mapping[file]).to match_array(tests)
+        end
+      end
+    end
+  end
+end

spec/tooling/lib/tooling/test_map_packer_spec.rb

+# frozen_string_literal: true
+
+require_relative '../../../../tooling/lib/tooling/test_map_packer'
+
+RSpec.describe Tooling::TestMapPacker do
+  subject { described_class.new }
+
+  let(:map) do
+    {
+      'file1.rb' => [
+        './a/b/c/test_1.rb',
+        './a/b/test_2.rb',
+        './a/b/test_3.rb',
+        './a/test_4.rb',
+        './test_5.rb'
+      ],
+      'file2.rb' => [
+        './a/b/c/test_1.rb',
+        './a/test_4.rb',
+        './test_5.rb'
+      ]
+    }
+  end
+
+  let(:compact_map) do
+    {
+      'file1.rb' => {
+        '.' => {
+          'a' => {
+            'b' => {
+              'c' => {
+                'test_1.rb' => 1
+              },
+              'test_2.rb' => 1,
+              'test_3.rb' => 1
+            },
+            'test_4.rb' => 1
+          },
+          'test_5.rb' => 1
+        }
+      },
+      'file2.rb' => {
+        '.' => {
+          'a' => {
+            'b' => {
+              'c' => {
+                'test_1.rb' => 1
+              }
+            },
+            'test_4.rb' => 1
+          },
+          'test_5.rb' => 1
+        }
+      }
+    }
+  end
+
+  describe '#pack' do
+    it 'compacts list of test files into a prefix tree' do
+      expect(subject.pack(map)).to eq(compact_map)
+    end
+
+    it 'does nothing to empty hash' do
+      expect(subject.pack({})).to eq({})
+    end
+  end
+
+  describe '#unpack' do
+    it 'unpack prefix tree into list of test files' do
+      expect(subject.unpack(compact_map)).to eq(map)
+    end
+
+    it 'does nothing to empty hash' do
+      expect(subject.unpack({})).to eq({})
+    end
+  end
+end

tooling/lib/tooling/crystalball/coverage_lines_execution_detector.rb

+# frozen_string_literal: true
+
+require 'crystalball/map_generator/helpers/path_filter'
+
+module Tooling
+  module Crystalball
+    # Class for detecting code execution path based on coverage information diff
+    class CoverageLinesExecutionDetector
+      include ::Crystalball::MapGenerator::Helpers::PathFilter
+
+      attr_reader :exclude_prefixes
+
+      def initialize(*args, exclude_prefixes: [])
+        super(*args)
+        @exclude_prefixes = exclude_prefixes
+      end
+
+      # Detects files affected during example execution based on line coverage.
+      # Transforms absolute paths to relative.
+      # Exclude paths outside of repository and in excluded prefixes
+      #
+      # @param[Hash] hash of files affected before example execution
+      # @param[Hash] hash of files affected after example execution
+      # @return [Array<String>]
+      def detect(before, after)
+        file_names = after.keys
+        covered_files = file_names.reject { |file_name| same_coverage?(before, after, file_name) }
+        filter(covered_files)
+      end
+
+      private
+
+      def same_coverage?(before, after, file_name)
+        before[file_name] && before[file_name][:lines] == after[file_name][:lines]
+      end
+
+      def filter(paths)
+        super.reject do |file_name|
+          exclude_prefixes.any? { |prefix| file_name.start_with?(prefix) }
+        end
+      end
+    end
+  end
+end

tooling/lib/tooling/crystalball/coverage_lines_strategy.rb

+# frozen_string_literal: true
+
+require 'coverage'
+require 'crystalball/map_generator/coverage_strategy'
+require_relative './coverage_lines_execution_detector'
+
+module Tooling
+  module Crystalball
+    # Crystalball map generator strategy based on Crystalball::MapGenerator::CoverageStrategy,
+    # modified to use Coverage.start(lines: true)
+    # This maintains compatibility with SimpleCov on Ruby >= 2.5 with start arguments
+    # and SimpleCov.start uses Coverage.start(lines: true) by default
+    class CoverageLinesStrategy < ::Crystalball::MapGenerator::CoverageStrategy
+      def initialize(execution_detector = CoverageLinesExecutionDetector)
+        super(execution_detector)
+      end
+
+      def after_register
+        Coverage.start(lines: true)
+      end
+    end
+  end
+end

tooling/lib/tooling/test_map_generator.rb

+# frozen_string_literal: true
+
+require 'set'
+require 'yaml'
+
+module Tooling
+  class TestMapGenerator
+    def initialize
+      @mapping = Hash.new { |h, k| h[k] = Set.new }
+    end
+
+    def parse(yaml_files)
+      Array(yaml_files).each do |yaml_file|
+        data = File.read(yaml_file)
+        _metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+
+        example_groups.each do |example_id, files|
+          files.each do |file|
+            spec_file = strip_example_uid(example_id)
+            @mapping[file] << spec_file
+          end
+        end
+      end
+    end
+
+    def mapping
+      @mapping.transform_values { |set| set.to_a }
+    end
+
+    private
+
+    def strip_example_uid(example_id)
+      example_id.gsub(/\[.+\]/, '')
+    end
+  end
+end

tooling/lib/tooling/test_map_packer.rb

+# frozen_string_literal: true
+
+module Tooling
+  class TestMapPacker
+    SEPARATOR = '/'.freeze
+    MARKER = 1
+
+    def pack(map)
+      map.transform_values(&method(:create_tree_from_tests))
+    end
+
+    def unpack(compact_map)
+      compact_map.transform_values(&method(:retrieve_tests_from_tree))
+    end
+
+    private
+
+    def create_tree_from_tests(tests)
+      tests.inject({}) do |tree, test|
+        segments = test.split(SEPARATOR)
+        branch = create_branch_from_segments(segments)
+        deep_merge(tree, branch)
+      end
+    end
+
+    def create_branch_from_segments(segments)
+      segments.reverse.inject(MARKER) { |node, parent| { parent => node } }
+    end
+
+    def deep_merge(hash, other)
+      hash.merge(other) do |_, this_val, other_val|
+        if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+          deep_merge(this_val, other_val)
+        else
+          other_val
+        end
+      end
+    end
+
+    def retrieve_tests_from_tree(tree)
+      traverse(tree).inject([]) do |tests, test|
+        tests << test
+      end
+    end
+
+    def traverse(tree, segments = [], &block)
+      return to_enum(__method__, tree, segments) unless block_given?
+
+      if tree == MARKER
+        return yield segments.join(SEPARATOR)
+      end
+
+      tree.each do |key, value|
+        traverse(value, segments + [key], &block)
+      end
+    end
+  end
+end