Merge branch 'collect-new-ios-projects' into 'master'

Detect if a project targets Apple iOS platform

See merge request gitlab-org/gitlab!80361

app/models/programming_language.rb

@@ -4,9 +4,10 @@ class ProgrammingLanguage < ApplicationRecord
   validates :name, presence: true
   validates :color, allow_blank: false, color: true
 
-  # Returns all programming languages which match the given name (case
+  # Returns all programming languages which match any of the given names (case
   # insensitively).
-  scope :with_name_case_insensitive, ->(name) do
-    where(arel_table[:name].matches(sanitize_sql_like(name)))
+  scope :with_name_case_insensitive, ->(*names) do
+    sanitized_names = names.map(&method(:sanitize_sql_like))
+    where(arel_table[:name].matches_any(sanitized_names))
   end
 end

app/models/project.rb

@@ -1989,6 +1989,8 @@ class Project < ApplicationRecord
     ProjectCacheWorker.perform_async(self.id, [], [:repository_size])
     AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(id)
 
+    enqueue_record_project_target_platforms
+
     # The import assigns iid values on its own, e.g. by re-using GitHub ids.
     # Flush existing InternalId records for this project for consistency reasons.
     # Those records are going to be recreated with the next normal creation
@@ -2848,6 +2850,13 @@ class Project < ApplicationRecord
     group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self, default_enabled: :yaml)
   end
 
+  def enqueue_record_project_target_platforms
+    return unless Gitlab.com?
+    return unless Feature.enabled?(:record_projects_target_platforms, self, default_enabled: :yaml)
+
+    Projects::RecordTargetPlatformsWorker.perform_async(id)
+  end
+
   private
 
   # overridden in EE

app/models/project_setting.rb

 # frozen_string_literal: true
 
 class ProjectSetting < ApplicationRecord
+  ALLOWED_TARGET_PLATFORMS = %w(ios osx tvos watchos).freeze
+
   belongs_to :project, inverse_of: :project_setting
 
   enum squash_option: {
@@ -14,6 +16,9 @@ class ProjectSetting < ApplicationRecord
 
   validates :merge_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH }
   validates :squash_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH }
+  validates :target_platforms, inclusion: { in: ALLOWED_TARGET_PLATFORMS }
+
+  validate :validates_mr_default_target_self
 
   default_value_for(:legacy_open_source_license_available) do
     Feature.enabled?(:legacy_open_source_license_available, default_enabled: :yaml, type: :ops)
@@ -27,7 +32,9 @@ class ProjectSetting < ApplicationRecord
     %w[always never].include?(squash_option)
   end
 
-  validate :validates_mr_default_target_self
+  def target_platforms=(val)
+    super(val&.map(&:to_s)&.sort)
+  end
 
   private
 

app/models/repository_language.rb

@@ -8,8 +8,8 @@ class RepositoryLanguage < ApplicationRecord
 
   default_scope { includes(:programming_language) } # rubocop:disable Cop/DefaultScope
 
-  scope :with_programming_language, ->(name) do
-    joins(:programming_language).merge(ProgrammingLanguage.with_name_case_insensitive(name))
+  scope :with_programming_language, ->(*names) do
+    joins(:programming_language).merge(ProgrammingLanguage.with_name_case_insensitive(*names))
   end
 
   validates :project, presence: true

app/services/git/branch_push_service.rb

@@ -24,6 +24,7 @@ module Git
 
       enqueue_update_mrs
       enqueue_detect_repository_languages
+      enqueue_record_project_target_platforms
 
       execute_related_hooks
 
@@ -53,6 +54,12 @@ module Git
       DetectRepositoryLanguagesWorker.perform_async(project.id)
     end
 
+    def enqueue_record_project_target_platforms
+      return unless default_branch?
+
+      project.enqueue_record_project_target_platforms
+    end
+
     # Only stop environments if the ref is a branch that is being deleted
     def stop_environments
       return unless removing_branch?

app/services/projects/apple_target_platform_detector_service.rb

+# frozen_string_literal: true
+
+module Projects
+  # Service class to detect target platforms of a project made for the Apple
+  # Ecosystem.
+  #
+  # This service searches project.pbxproj and *.xcconfig files (contains build
+  # settings) for the string "SDKROOT = <SDK_name>" where SDK_name can be
+  # 'iphoneos', 'macosx', 'appletvos' or 'watchos'. Currently, the service is
+  # intentionally limited (for performance reasons) to detect if a project
+  # targets iOS.
+  #
+  # Ref: https://developer.apple.com/documentation/xcode/build-settings-reference/
+  #
+  # Example usage:
+  # > AppleTargetPlatformDetectorService.new(a_project).execute
+  # => []
+  # > AppleTargetPlatformDetectorService.new(an_ios_project).execute
+  # => [:ios]
+  # > AppleTargetPlatformDetectorService.new(multiplatform_project).execute
+  # => [:ios, :osx, :tvos, :watchos]
+  class AppleTargetPlatformDetectorService < BaseService
+    BUILD_CONFIG_FILENAMES = %w(project.pbxproj *.xcconfig).freeze
+
+    # For the current iteration, we only want to detect when the project targets
+    # iOS. In the future, we can use the same logic to detect projects that
+    # target OSX, TvOS, and WatchOS platforms with SDK names 'macosx', 'appletvos',
+    # and 'watchos', respectively.
+    PLATFORM_SDK_NAMES = { ios: 'iphoneos' }.freeze
+
+    def execute
+      detect_platforms
+    end
+
+    private
+
+    def file_finder
+      @file_finder ||= ::Gitlab::FileFinder.new(project, project.default_branch)
+    end
+
+    def detect_platforms
+      # Return array of SDK names for which "SDKROOT = <sdk_name>" setting
+      # definition can be found in either project.pbxproj or *.xcconfig files.
+      PLATFORM_SDK_NAMES.select do |_, sdk|
+        config_files_containing_sdk_setting(sdk).present?
+      end.keys
+    end
+
+    # Return array of project.pbxproj and/or *.xcconfig files
+    # (Gitlab::Search::FoundBlob) that contain the setting definition string
+    # "SDKROOT = <sdk_name>"
+    def config_files_containing_sdk_setting(sdk)
+      BUILD_CONFIG_FILENAMES.map do |filename|
+        file_finder.find("SDKROOT = #{sdk} filename:#{filename}")
+      end.flatten
+    end
+  end
+end

app/services/projects/record_target_platforms_service.rb

+# frozen_string_literal: true
+
+module Projects
+  class RecordTargetPlatformsService < BaseService
+    include Gitlab::Utils::StrongMemoize
+
+    def execute
+      record_target_platforms
+    end
+
+    private
+
+    def target_platforms
+      strong_memoize(:target_platforms) do
+        AppleTargetPlatformDetectorService.new(project).execute
+      end
+    end
+
+    def record_target_platforms
+      return unless target_platforms.present?
+
+      setting = ::ProjectSetting.find_or_initialize_by(project: project) # rubocop:disable CodeReuse/ActiveRecord
+      setting.target_platforms = target_platforms
+      setting.save
+
+      setting.target_platforms
+    end
+  end
+end

app/workers/all_queues.yml

@@ -2812,6 +2812,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: projects_record_target_platforms
+  :worker_name: Projects::RecordTargetPlatformsWorker
+  :feature_category: :experimentation_activation
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: projects_refresh_build_artifacts_size_statistics
   :worker_name: Projects::RefreshBuildArtifactsSizeStatisticsWorker
   :feature_category: :build_artifacts

app/workers/projects/record_target_platforms_worker.rb

+# frozen_string_literal: true
+
+module Projects
+  class RecordTargetPlatformsWorker
+    include ApplicationWorker
+    include ExclusiveLeaseGuard
+
+    LEASE_TIMEOUT = 1.hour.to_i
+    APPLE_PLATFORM_LANGUAGES = %w(swift objective-c).freeze
+
+    feature_category :experimentation_activation
+    data_consistency :always
+    deduplicate :until_executed
+    urgency :low
+    idempotent!
+
+    def perform(project_id)
+      @project = Project.find_by_id(project_id)
+
+      return unless project
+      return unless uses_apple_platform_languages?
+
+      try_obtain_lease do
+        @target_platforms = Projects::RecordTargetPlatformsService.new(project).execute
+        log_target_platforms_metadata
+      end
+    end
+
+    private
+
+    attr_reader :target_platforms, :project
+
+    def uses_apple_platform_languages?
+      project.repository_languages.with_programming_language(*APPLE_PLATFORM_LANGUAGES).present?
+    end
+
+    def log_target_platforms_metadata
+      return unless target_platforms.present?
+
+      log_extra_metadata_on_done(:target_platforms, target_platforms)
+    end
+
+    def lease_key
+      @lease_key ||= "#{self.class.name.underscore}:#{project.id}"
+    end
+
+    def lease_timeout
+      LEASE_TIMEOUT
+    end
+
+    def lease_release?
+      false
+    end
+  end
+end

config/feature_flags/development/record_projects_target_platforms.yml

+---
+name: record_projects_target_platforms
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80361
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/354286
+milestone: '14.10'
+type: development
+group: group::activation
+default_enabled: false

config/sidekiq_queues.yml

@@ -361,6 +361,8 @@
   - 1
 - - projects_process_sync_events
   - 1
+- - projects_record_target_platforms
+  - 1
 - - projects_refresh_build_artifacts_size_statistics
   - 1
 - - projects_schedule_bulk_repository_shard_moves

db/migrate/20220318120802_add_target_platforms_to_project_setting.rb

+# frozen_string_literal: true
+
+class AddTargetPlatformsToProjectSetting < Gitlab::Database::Migration[1.0]
+  def change
+    add_column :project_settings, :target_platforms, :string, array: true, default: [], null: false, if_not_exists: true
+  end
+end

db/schema_migrations/20220318120802

+19f25b2f373e7c2799812661baca1902c9c74df67f7a5e88116862fb078a5957
\ No newline at end of file

db/structure.sql

@@ -19371,6 +19371,7 @@ CREATE TABLE project_settings (
     has_shimo boolean DEFAULT false NOT NULL,
     squash_commit_template text,
     legacy_open_source_license_available boolean DEFAULT true NOT NULL,
+    target_platforms character varying[] DEFAULT '{}'::character varying[] NOT NULL,
     CONSTRAINT check_3a03e7557a CHECK ((char_length(previous_default_branch) <= 4096)),
     CONSTRAINT check_b09644994b CHECK ((char_length(squash_commit_template) <= 500)),
     CONSTRAINT check_bde223416c CHECK ((show_default_award_emojis IS NOT NULL)),

spec/models/programming_language_spec.rb

@@ -10,4 +10,22 @@ RSpec.describe ProgrammingLanguage do
   it { is_expected.to allow_value("#000000").for(:color) }
   it { is_expected.not_to allow_value("000000").for(:color) }
   it { is_expected.not_to allow_value("#0z0000").for(:color) }
+
+  describe '.with_name_case_insensitive scope' do
+    let_it_be(:ruby) { create(:programming_language, name: 'Ruby') }
+    let_it_be(:python) { create(:programming_language, name: 'Python') }
+    let_it_be(:swift) { create(:programming_language, name: 'Swift') }
+
+    it 'accepts a single name parameter' do
+      expect(described_class.with_name_case_insensitive('swift')).to(
+        contain_exactly(swift)
+      )
+    end
+
+    it 'accepts multiple names' do
+      expect(described_class.with_name_case_insensitive('ruby', 'python')).to(
+        contain_exactly(ruby, python)
+      )
+    end
+  end
 end

spec/models/project_setting_spec.rb

@@ -4,4 +4,34 @@ require 'spec_helper'
 
 RSpec.describe ProjectSetting, type: :model do
   it { is_expected.to belong_to(:project) }
+
+  describe 'validations' do
+    it { is_expected.not_to allow_value(nil).for(:target_platforms) }
+    it { is_expected.to allow_value([]).for(:target_platforms) }
+
+    it 'allows any combination of the allowed target platforms' do
+      valid_target_platform_combinations.each do |target_platforms|
+        expect(subject).to allow_value(target_platforms).for(:target_platforms)
+      end
+    end
+
+    [nil, 'not_allowed', :invalid].each do |invalid_value|
+      it { is_expected.not_to allow_value([invalid_value]).for(:target_platforms) }
+    end
+  end
+
+  describe 'target_platforms=' do
+    it 'stringifies and sorts' do
+      project_setting = build(:project_setting, target_platforms: [:watchos, :ios])
+      expect(project_setting.target_platforms).to eq %w(ios watchos)
+    end
+  end
+
+  def valid_target_platform_combinations
+    target_platforms = described_class::ALLOWED_TARGET_PLATFORMS
+
+    0.upto(target_platforms.size).flat_map do |n|
+      target_platforms.permutation(n).to_a
+    end
+  end
 end

spec/models/project_spec.rb

@@ -5622,6 +5622,18 @@ RSpec.describe Project, factory_default: :keep do
         expect(project.protected_branches.first.merge_access_levels.map(&:access_level)).to eq([Gitlab::Access::MAINTAINER])
       end
     end
+
+    describe 'project target platforms detection' do
+      before do
+        create(:import_state, :started, project: project)
+      end
+
+      it 'calls enqueue_record_project_target_platforms' do
+        expect(project).to receive(:enqueue_record_project_target_platforms)
+
+        project.after_import
+      end
+    end
   end
 
   describe '#update_project_counter_caches' do
@@ -8091,6 +8103,44 @@ RSpec.describe Project, factory_default: :keep do
     it_behaves_like 'blocks unsafe serialization'
   end
 
+  describe '#enqueue_record_project_target_platforms' do
+    let_it_be(:project) { create(:project) }
+
+    let(:com) { true }
+
+    before do
+      allow(Gitlab).to receive(:com?).and_return(com)
+    end
+
+    it 'enqueues a Projects::RecordTargetPlatformsWorker' do
+      expect(Projects::RecordTargetPlatformsWorker).to receive(:perform_async).with(project.id)
+
+      project.enqueue_record_project_target_platforms
+    end
+
+    shared_examples 'does not enqueue a Projects::RecordTargetPlatformsWorker' do
+      it 'does not enqueue a Projects::RecordTargetPlatformsWorker' do
+        expect(Projects::RecordTargetPlatformsWorker).not_to receive(:perform_async)
+
+        project.enqueue_record_project_target_platforms
+      end
+    end
+
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(record_projects_target_platforms: false)
+      end
+
+      it_behaves_like 'does not enqueue a Projects::RecordTargetPlatformsWorker'
+    end
+
+    context 'when not in gitlab.com' do
+      let(:com) { false }
+
+      it_behaves_like 'does not enqueue a Projects::RecordTargetPlatformsWorker'
+    end
+  end
+
   private
 
   def finish_job(export_job)

spec/requests/api/project_attributes.yml

@@ -148,6 +148,7 @@ project_setting:
     - updated_at
     - cve_id_request_enabled
     - mr_default_target_self
+    - target_platforms
 
 build_service_desk_setting: # service_desk_setting
   unexposed_attributes:

spec/services/git/branch_push_service_spec.rb

@@ -721,4 +721,14 @@ RSpec.describe Git::BranchPushService, services: true do
       it_behaves_like 'does not enqueue Jira sync worker'
     end
   end
+
+  describe 'project target platforms detection' do
+    subject(:execute) { execute_service(project, user, oldrev: blankrev, newrev: newrev, ref: ref) }
+
+    it 'calls enqueue_record_project_target_platforms on the project' do
+      expect(project).to receive(:enqueue_record_project_target_platforms)
+
+      execute
+    end
+  end
 end

spec/services/projects/apple_target_platform_detector_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::AppleTargetPlatformDetectorService do
+  let_it_be(:project) { build(:project) }
+
+  subject { described_class.new(project).execute }
+
+  context 'when project is not an xcode project' do
+    before do
+      allow(Gitlab::FileFinder).to receive(:new) { instance_double(Gitlab::FileFinder, find: []) }
+    end
+
+    it 'returns an empty array' do
+      is_expected.to match_array []
+    end
+  end
+
+  context 'when project is an xcode project' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:finder) { instance_double(Gitlab::FileFinder) }
+
+    before do
+      allow(Gitlab::FileFinder).to receive(:new) { finder }
+    end
+
+    def search_query(sdk, filename)
+      "SDKROOT = #{sdk} filename:#{filename}"
+    end
+
+    context 'when setting string is found' do
+      where(:sdk, :filename, :result) do
+        'iphoneos'  | 'project.pbxproj' | [:ios]
+        'iphoneos'  | '*.xcconfig'      | [:ios]
+      end
+
+      with_them do
+        before do
+          allow(finder).to receive(:find).with(anything) { [] }
+          allow(finder).to receive(:find).with(search_query(sdk, filename)) { [instance_double(Gitlab::Search::FoundBlob)] }
+        end
+
+        it 'returns an array of unique detected targets' do
+          is_expected.to match_array result
+        end
+      end
+    end
+
+    context 'when setting string is not found' do
+      before do
+        allow(finder).to receive(:find).with(anything) { [] }
+      end
+
+      it 'returns an empty array' do
+        is_expected.to match_array []
+      end
+    end
+  end
+end

spec/services/projects/record_target_platforms_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::RecordTargetPlatformsService, '#execute' do
+  let_it_be(:project) { create(:project) }
+
+  subject(:execute) { described_class.new(project).execute }
+
+  context 'when project is an XCode project' do
+    before do
+      double = instance_double(Projects::AppleTargetPlatformDetectorService, execute: [:ios, :osx])
+      allow(Projects::AppleTargetPlatformDetectorService).to receive(:new) { double }
+    end
+
+    it 'creates a new setting record for the project', :aggregate_failures do
+      expect { execute }.to change { ProjectSetting.count }.from(0).to(1)
+      expect(ProjectSetting.last.target_platforms).to match_array(%w(ios osx))
+    end
+
+    it 'returns array of detected target platforms' do
+      expect(execute).to match_array %w(ios osx)
+    end
+
+    context 'when a project has an existing setting record' do
+      before do
+        create(:project_setting, project: project, target_platforms: saved_target_platforms)
+      end
+
+      def project_setting
+        ProjectSetting.find_by_project_id(project.id)
+      end
+
+      context 'when target platforms changed' do
+        let(:saved_target_platforms) { %w(tvos) }
+
+        it 'updates' do
+          expect { execute }.to change { project_setting.target_platforms }.from(%w(tvos)).to(%w(ios osx))
+        end
+
+        it { is_expected.to match_array %w(ios osx) }
+      end
+
+      context 'when target platforms are the same' do
+        let(:saved_target_platforms) { %w(osx ios) }
+
+        it 'does not update' do
+          expect { execute }.not_to change { project_setting.updated_at }
+        end
+      end
+    end
+  end
+
+  context 'when project is not an XCode project' do
+    before do
+      double = instance_double(Projects::AppleTargetPlatformDetectorService, execute: [])
+      allow(Projects::AppleTargetPlatformDetectorService).to receive(:new).with(project) { double }
+    end
+
+    it 'does nothing' do
+      expect { execute }.not_to change { ProjectSetting.count }
+    end
+
+    it { is_expected.to be_nil }
+  end
+end

spec/workers/projects/record_target_platforms_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::RecordTargetPlatformsWorker do
+  include ExclusiveLeaseHelpers
+
+  let_it_be(:swift) { create(:programming_language, name: 'Swift') }
+  let_it_be(:objective_c) { create(:programming_language, name: 'Objective-C') }
+  let_it_be(:project) { create(:project, :repository, detected_repository_languages: true) }
+
+  let(:worker) { described_class.new }
+  let(:service_result) { %w(ios osx watchos) }
+  let(:service_double) { instance_double(Projects::RecordTargetPlatformsService, execute: service_result) }
+  let(:lease_key) { "#{described_class.name.underscore}:#{project.id}" }
+  let(:lease_timeout) { described_class::LEASE_TIMEOUT }
+
+  subject(:perform) { worker.perform(project.id) }
+
+  before do
+    stub_exclusive_lease(lease_key, timeout: lease_timeout)
+  end
+
+  shared_examples 'performs detection' do
+    it 'creates and executes a Projects::RecordTargetPlatformService instance for the project', :aggregate_failures do
+      expect(Projects::RecordTargetPlatformsService).to receive(:new).with(project) { service_double }
+      expect(service_double).to receive(:execute)
+
+      perform
+    end
+
+    it 'logs extra metadata on done', :aggregate_failures do
+      expect(Projects::RecordTargetPlatformsService).to receive(:new).with(project) { service_double }
+      expect(worker).to receive(:log_extra_metadata_on_done).with(:target_platforms, service_result)
+
+      perform
+    end
+  end
+
+  shared_examples 'does nothing' do
+    it 'does nothing' do
+      expect(Projects::RecordTargetPlatformsService).not_to receive(:new)
+
+      perform
+    end
+  end
+
+  context 'when project uses Swift programming language' do
+    let!(:repository_language) { create(:repository_language, project: project, programming_language: swift) }
+
+    include_examples 'performs detection'
+  end
+
+  context 'when project uses Objective-C programming language' do
+    let!(:repository_language) { create(:repository_language, project: project, programming_language: objective_c) }
+
+    include_examples 'performs detection'
+  end
+
+  context 'when the project does not contain programming languages for Apple platforms' do
+    it_behaves_like 'does nothing'
+  end
+
+  context 'when project is not found' do
+    it 'does nothing' do
+      expect(Projects::RecordTargetPlatformsService).not_to receive(:new)
+
+      worker.perform(non_existing_record_id)
+    end
+  end
+
+  context 'when exclusive lease cannot be obtained' do
+    before do
+      stub_exclusive_lease_taken(lease_key)
+    end
+
+    it_behaves_like 'does nothing'
+  end
+
+  it 'has the `until_executed` deduplicate strategy' do
+    expect(described_class.get_deduplicate_strategy).to eq(:until_executed)
+  end
+
+  it 'overrides #lease_release? to return false' do
+    expect(worker.send(:lease_release?)).to eq false
+  end
+end