Commit b26d73a4 authored by Eugie Limpin's avatar Eugie Limpin Committed by Mayra Cabrera

Detect projects built for Apple iOS platform

Like repository language detection feature, we'll try to detect if a
project is built for Apple iOS platform. This information will be used
in experiments to customize the experience (e.g. suggest setting up a
MacOS runners) of developers involved in such projects.

This detection will happen every time changes are pushed to the
project’s main branch and after a project is imported (using a project
template when creating a project is also an import). Detection is
executed in the background as a Sidekiq job at most once every hour
(using ExclusiveLease) and only if the project uses Swift or Objective-C
programming language.

If the project is detected as built for iOS we mark it by setting the
project’s corresponding ProjectSetting `target_platforms` attribute to
[‘ios’].

Changelog: added
parent 0fce8e36
......@@ -4,9 +4,10 @@ class ProgrammingLanguage < ApplicationRecord
validates :name, presence: true
validates :color, allow_blank: false, color: true
# Returns all programming languages which match the given name (case
# Returns all programming languages which match any of the given names (case
# insensitively).
scope :with_name_case_insensitive, ->(name) do
where(arel_table[:name].matches(sanitize_sql_like(name)))
scope :with_name_case_insensitive, ->(*names) do
sanitized_names = names.map(&method(:sanitize_sql_like))
where(arel_table[:name].matches_any(sanitized_names))
end
end
......@@ -1989,6 +1989,8 @@ class Project < ApplicationRecord
ProjectCacheWorker.perform_async(self.id, [], [:repository_size])
AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(id)
enqueue_record_project_target_platforms
# The import assigns iid values on its own, e.g. by re-using GitHub ids.
# Flush existing InternalId records for this project for consistency reasons.
# Those records are going to be recreated with the next normal creation
......@@ -2848,6 +2850,13 @@ class Project < ApplicationRecord
group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self, default_enabled: :yaml)
end
def enqueue_record_project_target_platforms
return unless Gitlab.com?
return unless Feature.enabled?(:record_projects_target_platforms, self, default_enabled: :yaml)
Projects::RecordTargetPlatformsWorker.perform_async(id)
end
private
# overridden in EE
......
# frozen_string_literal: true
class ProjectSetting < ApplicationRecord
ALLOWED_TARGET_PLATFORMS = %w(ios osx tvos watchos).freeze
belongs_to :project, inverse_of: :project_setting
enum squash_option: {
......@@ -14,6 +16,9 @@ class ProjectSetting < ApplicationRecord
validates :merge_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH }
validates :squash_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH }
validates :target_platforms, inclusion: { in: ALLOWED_TARGET_PLATFORMS }
validate :validates_mr_default_target_self
default_value_for(:legacy_open_source_license_available) do
Feature.enabled?(:legacy_open_source_license_available, default_enabled: :yaml, type: :ops)
......@@ -27,7 +32,9 @@ class ProjectSetting < ApplicationRecord
%w[always never].include?(squash_option)
end
validate :validates_mr_default_target_self
def target_platforms=(val)
super(val&.map(&:to_s)&.sort)
end
private
......
......@@ -8,8 +8,8 @@ class RepositoryLanguage < ApplicationRecord
default_scope { includes(:programming_language) } # rubocop:disable Cop/DefaultScope
scope :with_programming_language, ->(name) do
joins(:programming_language).merge(ProgrammingLanguage.with_name_case_insensitive(name))
scope :with_programming_language, ->(*names) do
joins(:programming_language).merge(ProgrammingLanguage.with_name_case_insensitive(*names))
end
validates :project, presence: true
......
......@@ -24,6 +24,7 @@ module Git
enqueue_update_mrs
enqueue_detect_repository_languages
enqueue_record_project_target_platforms
execute_related_hooks
......@@ -53,6 +54,12 @@ module Git
DetectRepositoryLanguagesWorker.perform_async(project.id)
end
def enqueue_record_project_target_platforms
return unless default_branch?
project.enqueue_record_project_target_platforms
end
# Only stop environments if the ref is a branch that is being deleted
def stop_environments
return unless removing_branch?
......
# frozen_string_literal: true
module Projects
# Service class to detect target platforms of a project made for the Apple
# Ecosystem.
#
# This service searches project.pbxproj and *.xcconfig files (contains build
# settings) for the string "SDKROOT = <SDK_name>" where SDK_name can be
# 'iphoneos', 'macosx', 'appletvos' or 'watchos'. Currently, the service is
# intentionally limited (for performance reasons) to detect if a project
# targets iOS.
#
# Ref: https://developer.apple.com/documentation/xcode/build-settings-reference/
#
# Example usage:
# > AppleTargetPlatformDetectorService.new(a_project).execute
# => []
# > AppleTargetPlatformDetectorService.new(an_ios_project).execute
# => [:ios]
# > AppleTargetPlatformDetectorService.new(multiplatform_project).execute
# => [:ios, :osx, :tvos, :watchos]
class AppleTargetPlatformDetectorService < BaseService
BUILD_CONFIG_FILENAMES = %w(project.pbxproj *.xcconfig).freeze
# For the current iteration, we only want to detect when the project targets
# iOS. In the future, we can use the same logic to detect projects that
# target OSX, TvOS, and WatchOS platforms with SDK names 'macosx', 'appletvos',
# and 'watchos', respectively.
PLATFORM_SDK_NAMES = { ios: 'iphoneos' }.freeze
def execute
detect_platforms
end
private
def file_finder
@file_finder ||= ::Gitlab::FileFinder.new(project, project.default_branch)
end
def detect_platforms
# Return array of SDK names for which "SDKROOT = <sdk_name>" setting
# definition can be found in either project.pbxproj or *.xcconfig files.
PLATFORM_SDK_NAMES.select do |_, sdk|
config_files_containing_sdk_setting(sdk).present?
end.keys
end
# Return array of project.pbxproj and/or *.xcconfig files
# (Gitlab::Search::FoundBlob) that contain the setting definition string
# "SDKROOT = <sdk_name>"
def config_files_containing_sdk_setting(sdk)
BUILD_CONFIG_FILENAMES.map do |filename|
file_finder.find("SDKROOT = #{sdk} filename:#{filename}")
end.flatten
end
end
end
# frozen_string_literal: true
module Projects
class RecordTargetPlatformsService < BaseService
include Gitlab::Utils::StrongMemoize
def execute
record_target_platforms
end
private
def target_platforms
strong_memoize(:target_platforms) do
AppleTargetPlatformDetectorService.new(project).execute
end
end
def record_target_platforms
return unless target_platforms.present?
setting = ::ProjectSetting.find_or_initialize_by(project: project) # rubocop:disable CodeReuse/ActiveRecord
setting.target_platforms = target_platforms
setting.save
setting.target_platforms
end
end
end
......@@ -2812,6 +2812,15 @@
:weight: 1
:idempotent: true
:tags: []
- :name: projects_record_target_platforms
:worker_name: Projects::RecordTargetPlatformsWorker
:feature_category: :experimentation_activation
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: projects_refresh_build_artifacts_size_statistics
:worker_name: Projects::RefreshBuildArtifactsSizeStatisticsWorker
:feature_category: :build_artifacts
......
# frozen_string_literal: true
module Projects
class RecordTargetPlatformsWorker
include ApplicationWorker
include ExclusiveLeaseGuard
LEASE_TIMEOUT = 1.hour.to_i
APPLE_PLATFORM_LANGUAGES = %w(swift objective-c).freeze
feature_category :experimentation_activation
data_consistency :always
deduplicate :until_executed
urgency :low
idempotent!
def perform(project_id)
@project = Project.find_by_id(project_id)
return unless project
return unless uses_apple_platform_languages?
try_obtain_lease do
@target_platforms = Projects::RecordTargetPlatformsService.new(project).execute
log_target_platforms_metadata
end
end
private
attr_reader :target_platforms, :project
def uses_apple_platform_languages?
project.repository_languages.with_programming_language(*APPLE_PLATFORM_LANGUAGES).present?
end
def log_target_platforms_metadata
return unless target_platforms.present?
log_extra_metadata_on_done(:target_platforms, target_platforms)
end
def lease_key
@lease_key ||= "#{self.class.name.underscore}:#{project.id}"
end
def lease_timeout
LEASE_TIMEOUT
end
def lease_release?
false
end
end
end
---
name: record_projects_target_platforms
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80361
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/354286
milestone: '14.10'
type: development
group: group::activation
default_enabled: false
......@@ -361,6 +361,8 @@
- 1
- - projects_process_sync_events
- 1
- - projects_record_target_platforms
- 1
- - projects_refresh_build_artifacts_size_statistics
- 1
- - projects_schedule_bulk_repository_shard_moves
......
# frozen_string_literal: true
class AddTargetPlatformsToProjectSetting < Gitlab::Database::Migration[1.0]
def change
add_column :project_settings, :target_platforms, :string, array: true, default: [], null: false, if_not_exists: true
end
end
19f25b2f373e7c2799812661baca1902c9c74df67f7a5e88116862fb078a5957
\ No newline at end of file
......@@ -19371,6 +19371,7 @@ CREATE TABLE project_settings (
has_shimo boolean DEFAULT false NOT NULL,
squash_commit_template text,
legacy_open_source_license_available boolean DEFAULT true NOT NULL,
target_platforms character varying[] DEFAULT '{}'::character varying[] NOT NULL,
CONSTRAINT check_3a03e7557a CHECK ((char_length(previous_default_branch) <= 4096)),
CONSTRAINT check_b09644994b CHECK ((char_length(squash_commit_template) <= 500)),
CONSTRAINT check_bde223416c CHECK ((show_default_award_emojis IS NOT NULL)),
......@@ -10,4 +10,22 @@ RSpec.describe ProgrammingLanguage do
it { is_expected.to allow_value("#000000").for(:color) }
it { is_expected.not_to allow_value("000000").for(:color) }
it { is_expected.not_to allow_value("#0z0000").for(:color) }
describe '.with_name_case_insensitive scope' do
let_it_be(:ruby) { create(:programming_language, name: 'Ruby') }
let_it_be(:python) { create(:programming_language, name: 'Python') }
let_it_be(:swift) { create(:programming_language, name: 'Swift') }
it 'accepts a single name parameter' do
expect(described_class.with_name_case_insensitive('swift')).to(
contain_exactly(swift)
)
end
it 'accepts multiple names' do
expect(described_class.with_name_case_insensitive('ruby', 'python')).to(
contain_exactly(ruby, python)
)
end
end
end
......@@ -4,4 +4,34 @@ require 'spec_helper'
RSpec.describe ProjectSetting, type: :model do
it { is_expected.to belong_to(:project) }
describe 'validations' do
it { is_expected.not_to allow_value(nil).for(:target_platforms) }
it { is_expected.to allow_value([]).for(:target_platforms) }
it 'allows any combination of the allowed target platforms' do
valid_target_platform_combinations.each do |target_platforms|
expect(subject).to allow_value(target_platforms).for(:target_platforms)
end
end
[nil, 'not_allowed', :invalid].each do |invalid_value|
it { is_expected.not_to allow_value([invalid_value]).for(:target_platforms) }
end
end
describe 'target_platforms=' do
it 'stringifies and sorts' do
project_setting = build(:project_setting, target_platforms: [:watchos, :ios])
expect(project_setting.target_platforms).to eq %w(ios watchos)
end
end
def valid_target_platform_combinations
target_platforms = described_class::ALLOWED_TARGET_PLATFORMS
0.upto(target_platforms.size).flat_map do |n|
target_platforms.permutation(n).to_a
end
end
end
......@@ -5622,6 +5622,18 @@ RSpec.describe Project, factory_default: :keep do
expect(project.protected_branches.first.merge_access_levels.map(&:access_level)).to eq([Gitlab::Access::MAINTAINER])
end
end
describe 'project target platforms detection' do
before do
create(:import_state, :started, project: project)
end
it 'calls enqueue_record_project_target_platforms' do
expect(project).to receive(:enqueue_record_project_target_platforms)
project.after_import
end
end
end
describe '#update_project_counter_caches' do
......@@ -8091,6 +8103,44 @@ RSpec.describe Project, factory_default: :keep do
it_behaves_like 'blocks unsafe serialization'
end
describe '#enqueue_record_project_target_platforms' do
let_it_be(:project) { create(:project) }
let(:com) { true }
before do
allow(Gitlab).to receive(:com?).and_return(com)
end
it 'enqueues a Projects::RecordTargetPlatformsWorker' do
expect(Projects::RecordTargetPlatformsWorker).to receive(:perform_async).with(project.id)
project.enqueue_record_project_target_platforms
end
shared_examples 'does not enqueue a Projects::RecordTargetPlatformsWorker' do
it 'does not enqueue a Projects::RecordTargetPlatformsWorker' do
expect(Projects::RecordTargetPlatformsWorker).not_to receive(:perform_async)
project.enqueue_record_project_target_platforms
end
end
context 'when feature flag is disabled' do
before do
stub_feature_flags(record_projects_target_platforms: false)
end
it_behaves_like 'does not enqueue a Projects::RecordTargetPlatformsWorker'
end
context 'when not in gitlab.com' do
let(:com) { false }
it_behaves_like 'does not enqueue a Projects::RecordTargetPlatformsWorker'
end
end
private
def finish_job(export_job)
......
......@@ -148,6 +148,7 @@ project_setting:
- updated_at
- cve_id_request_enabled
- mr_default_target_self
- target_platforms
build_service_desk_setting: # service_desk_setting
unexposed_attributes:
......
......@@ -721,4 +721,14 @@ RSpec.describe Git::BranchPushService, services: true do
it_behaves_like 'does not enqueue Jira sync worker'
end
end
describe 'project target platforms detection' do
subject(:execute) { execute_service(project, user, oldrev: blankrev, newrev: newrev, ref: ref) }
it 'calls enqueue_record_project_target_platforms on the project' do
expect(project).to receive(:enqueue_record_project_target_platforms)
execute
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::AppleTargetPlatformDetectorService do
let_it_be(:project) { build(:project) }
subject { described_class.new(project).execute }
context 'when project is not an xcode project' do
before do
allow(Gitlab::FileFinder).to receive(:new) { instance_double(Gitlab::FileFinder, find: []) }
end
it 'returns an empty array' do
is_expected.to match_array []
end
end
context 'when project is an xcode project' do
using RSpec::Parameterized::TableSyntax
let(:finder) { instance_double(Gitlab::FileFinder) }
before do
allow(Gitlab::FileFinder).to receive(:new) { finder }
end
def search_query(sdk, filename)
"SDKROOT = #{sdk} filename:#{filename}"
end
context 'when setting string is found' do
where(:sdk, :filename, :result) do
'iphoneos' | 'project.pbxproj' | [:ios]
'iphoneos' | '*.xcconfig' | [:ios]
end
with_them do
before do
allow(finder).to receive(:find).with(anything) { [] }
allow(finder).to receive(:find).with(search_query(sdk, filename)) { [instance_double(Gitlab::Search::FoundBlob)] }
end
it 'returns an array of unique detected targets' do
is_expected.to match_array result
end
end
end
context 'when setting string is not found' do
before do
allow(finder).to receive(:find).with(anything) { [] }
end
it 'returns an empty array' do
is_expected.to match_array []
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::RecordTargetPlatformsService, '#execute' do
let_it_be(:project) { create(:project) }
subject(:execute) { described_class.new(project).execute }
context 'when project is an XCode project' do
before do
double = instance_double(Projects::AppleTargetPlatformDetectorService, execute: [:ios, :osx])
allow(Projects::AppleTargetPlatformDetectorService).to receive(:new) { double }
end
it 'creates a new setting record for the project', :aggregate_failures do
expect { execute }.to change { ProjectSetting.count }.from(0).to(1)
expect(ProjectSetting.last.target_platforms).to match_array(%w(ios osx))
end
it 'returns array of detected target platforms' do
expect(execute).to match_array %w(ios osx)
end
context 'when a project has an existing setting record' do
before do
create(:project_setting, project: project, target_platforms: saved_target_platforms)
end
def project_setting
ProjectSetting.find_by_project_id(project.id)
end
context 'when target platforms changed' do
let(:saved_target_platforms) { %w(tvos) }
it 'updates' do
expect { execute }.to change { project_setting.target_platforms }.from(%w(tvos)).to(%w(ios osx))
end
it { is_expected.to match_array %w(ios osx) }
end
context 'when target platforms are the same' do
let(:saved_target_platforms) { %w(osx ios) }
it 'does not update' do
expect { execute }.not_to change { project_setting.updated_at }
end
end
end
end
context 'when project is not an XCode project' do
before do
double = instance_double(Projects::AppleTargetPlatformDetectorService, execute: [])
allow(Projects::AppleTargetPlatformDetectorService).to receive(:new).with(project) { double }
end
it 'does nothing' do
expect { execute }.not_to change { ProjectSetting.count }
end
it { is_expected.to be_nil }
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Projects::RecordTargetPlatformsWorker do
include ExclusiveLeaseHelpers
let_it_be(:swift) { create(:programming_language, name: 'Swift') }
let_it_be(:objective_c) { create(:programming_language, name: 'Objective-C') }
let_it_be(:project) { create(:project, :repository, detected_repository_languages: true) }
let(:worker) { described_class.new }
let(:service_result) { %w(ios osx watchos) }
let(:service_double) { instance_double(Projects::RecordTargetPlatformsService, execute: service_result) }
let(:lease_key) { "#{described_class.name.underscore}:#{project.id}" }
let(:lease_timeout) { described_class::LEASE_TIMEOUT }
subject(:perform) { worker.perform(project.id) }
before do
stub_exclusive_lease(lease_key, timeout: lease_timeout)
end
shared_examples 'performs detection' do
it 'creates and executes a Projects::RecordTargetPlatformService instance for the project', :aggregate_failures do
expect(Projects::RecordTargetPlatformsService).to receive(:new).with(project) { service_double }
expect(service_double).to receive(:execute)
perform
end
it 'logs extra metadata on done', :aggregate_failures do
expect(Projects::RecordTargetPlatformsService).to receive(:new).with(project) { service_double }
expect(worker).to receive(:log_extra_metadata_on_done).with(:target_platforms, service_result)
perform
end
end
shared_examples 'does nothing' do
it 'does nothing' do
expect(Projects::RecordTargetPlatformsService).not_to receive(:new)
perform
end
end
context 'when project uses Swift programming language' do
let!(:repository_language) { create(:repository_language, project: project, programming_language: swift) }
include_examples 'performs detection'
end
context 'when project uses Objective-C programming language' do
let!(:repository_language) { create(:repository_language, project: project, programming_language: objective_c) }
include_examples 'performs detection'
end
context 'when the project does not contain programming languages for Apple platforms' do
it_behaves_like 'does nothing'
end
context 'when project is not found' do
it 'does nothing' do
expect(Projects::RecordTargetPlatformsService).not_to receive(:new)
worker.perform(non_existing_record_id)
end
end
context 'when exclusive lease cannot be obtained' do
before do
stub_exclusive_lease_taken(lease_key)
end
it_behaves_like 'does nothing'
end
it 'has the `until_executed` deduplicate strategy' do
expect(described_class.get_deduplicate_strategy).to eq(:until_executed)
end
it 'overrides #lease_release? to return false' do
expect(worker.send(:lease_release?)).to eq false
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment