Commit 04d07cc5 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'feature/runner-per-group' into 'master'

Shared CI runners for groups

See merge request gitlab-org/gitlab-ce!9646
parents bee5e26e 11f38dd1
......@@ -52,6 +52,12 @@ class Projects::RunnersController < Projects::ApplicationController
redirect_to project_settings_ci_cd_path(@project)
end
def toggle_group_runners
project.toggle_ci_cd_settings!(:group_runners_enabled)
redirect_to project_settings_ci_cd_path(@project)
end
protected
def set_runner
......
......@@ -67,10 +67,18 @@ module Projects
def define_runners_variables
@project_runners = @project.runners.ordered
@assignable_runners = current_user.ci_authorized_runners
.assignable_for(project).ordered.page(params[:page]).per(20)
@assignable_runners = current_user
.ci_authorized_runners
.assignable_for(project)
.ordered
.page(params[:page]).per(20)
@shared_runners = ::Ci::Runner.shared.active
@shared_runners_count = @shared_runners.count(:all)
@group_runners = ::Ci::Runner.belonging_to_parent_group_of_project(@project.id)
end
def define_secret_variables
......
......@@ -14,31 +14,49 @@ module Ci
has_many :builds
has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :projects, through: :runner_projects
has_many :runner_namespaces
has_many :groups, through: :runner_namespaces
has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build'
before_validation :set_default_values
scope :specific, ->() { where(is_shared: false) }
scope :shared, ->() { where(is_shared: true) }
scope :active, ->() { where(active: true) }
scope :paused, ->() { where(active: false) }
scope :online, ->() { where('contacted_at > ?', contact_time_deadline) }
scope :ordered, ->() { order(id: :desc) }
scope :specific, -> { where(is_shared: false) }
scope :shared, -> { where(is_shared: true) }
scope :active, -> { where(active: true) }
scope :paused, -> { where(active: false) }
scope :online, -> { where('contacted_at > ?', contact_time_deadline) }
scope :ordered, -> { order(id: :desc) }
scope :owned_or_shared, ->(project_id) do
joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id')
.where("ci_runner_projects.project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id)
scope :belonging_to_project, -> (project_id) {
joins(:runner_projects).where(ci_runner_projects: { project_id: project_id })
}
scope :belonging_to_parent_group_of_project, -> (project_id) {
project_groups = ::Group.joins(:projects).where(projects: { id: project_id })
hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors
joins(:groups).where(namespaces: { id: hierarchy_groups })
}
scope :owned_or_shared, -> (project_id) do
union = Gitlab::SQL::Union.new(
[belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared],
remove_duplicates: false
)
from("(#{union.to_sql}) ci_runners")
end
scope :assignable_for, ->(project) do
# FIXME: That `to_sql` is needed to workaround a weird Rails bug.
# Without that, placeholders would miss one and couldn't match.
where(locked: false)
.where.not("id IN (#{project.runners.select(:id).to_sql})").specific
.where.not("ci_runners.id IN (#{project.runners.select(:id).to_sql})")
.specific
end
validate :tag_constraints
validate :either_projects_or_group
validates :access_level, presence: true
acts_as_taggable
......@@ -50,6 +68,12 @@ module Ci
ref_protected: 1
}
enum runner_type: {
instance_type: 1,
group_type: 2,
project_type: 3
}
cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address
chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout
......@@ -120,6 +144,14 @@ module Ci
!shared?
end
def assigned_to_group?
runner_namespaces.any?
end
def assigned_to_project?
runner_projects.any?
end
def can_pick?(build)
return false if self.ref_protected? && !build.protected?
......@@ -174,6 +206,12 @@ module Ci
end
end
def pick_build!(build)
if can_pick?(build)
tick_runner_queue
end
end
private
def cleanup_runner_queue
......@@ -205,7 +243,17 @@ module Ci
end
def assignable_for?(project_id)
is_shared? || projects.exists?(id: project_id)
self.class.owned_or_shared(project_id).where(id: self.id).any?
end
def either_projects_or_group
if groups.many?
errors.add(:runner, 'can only be assigned to one group')
end
if assigned_to_group? && assigned_to_project?
errors.add(:runner, 'can only be assigned either to projects or to a group')
end
end
def accepting_tags?(build)
......
module Ci
class RunnerNamespace < ActiveRecord::Base
extend Gitlab::Ci::Model
belongs_to :runner
belongs_to :namespace, class_name: '::Namespace'
belongs_to :group, class_name: '::Group', foreign_key: :namespace_id
end
end
......@@ -9,6 +9,7 @@ class Group < Namespace
include SelectForProjectAuthorization
include LoadedInGroupList
include GroupDescendant
include TokenAuthenticatable
has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
alias_method :members, :group_members
......@@ -43,6 +44,8 @@ class Group < Namespace
validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 }
add_authentication_token_field :runners_token
after_create :post_create_hook
after_destroy :post_destroy_hook
after_save :update_two_factor_requirement
......@@ -294,6 +297,13 @@ class Group < Namespace
refresh_members_authorized_projects(blocking: false)
end
# each existing group needs to have a `runners_token`.
# we do this on read since migrating all existing groups is not a feasible
# solution.
def runners_token
ensure_runners_token!
end
private
def update_two_factor_requirement
......
......@@ -21,6 +21,9 @@ class Namespace < ActiveRecord::Base
has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :project_statistics
has_many :runner_namespaces, class_name: 'Ci::RunnerNamespace'
has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner'
# This should _not_ be `inverse_of: :namespace`, because that would also set
# `user.namespace` when this user creates a group with themselves as `owner`.
belongs_to :owner, class_name: "User"
......
......@@ -230,13 +230,11 @@ class Project < ActiveRecord::Base
has_many :project_deploy_tokens
has_many :deploy_tokens, through: :project_deploy_tokens
has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
has_one :auto_devops, class_name: 'ProjectAutoDevops'
has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
has_many :project_badges, class_name: 'ProjectBadge'
has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting'
has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true
accepts_nested_attributes_for :variables, allow_destroy: true
accepts_nested_attributes_for :project_feature, update_only: true
......@@ -247,6 +245,7 @@ class Project < ActiveRecord::Base
delegate :members, to: :team, prefix: true
delegate :add_user, :add_users, to: :team
delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team
delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings
# Validations
validates :creator, presence: true, on: :create
......@@ -332,6 +331,11 @@ class Project < ActiveRecord::Base
scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
scope :with_group_runners_enabled, -> do
joins(:ci_cd_settings)
.where(project_ci_cd_settings: { group_runners_enabled: true })
end
enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 }
chronic_duration_attr :build_timeout_human_readable, :build_timeout, default: 3600
......@@ -1301,12 +1305,17 @@ class Project < ActiveRecord::Base
@shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none
end
def active_shared_runners
@active_shared_runners ||= shared_runners.active
def group_runners
@group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none
end
def all_runners
union = Gitlab::SQL::Union.new([runners, group_runners, shared_runners])
Ci::Runner.from("(#{union.to_sql}) ci_runners")
end
def any_runners?(&block)
active_runners.any?(&block) || active_shared_runners.any?(&block)
all_runners.active.any?(&block)
end
def valid_runners_token?(token)
......@@ -1874,6 +1883,10 @@ class Project < ActiveRecord::Base
[]
end
def toggle_ci_cd_settings!(settings_attribute)
ci_cd_settings.toggle!(settings_attribute)
end
def gitlab_deploy_token
@gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token
end
......
class ProjectCiCdSetting < ActiveRecord::Base
belongs_to :project
belongs_to :project, inverse_of: :ci_cd_settings
# The version of the schema that first introduced this model/table.
MINIMUM_SCHEMA_VERSION = 20180403035759
......
......@@ -17,8 +17,10 @@ module Ci
builds =
if runner.shared?
builds_for_shared_runner
elsif runner.group_type?
builds_for_group_runner
else
builds_for_specific_runner
builds_for_project_runner
end
valid = true
......@@ -75,15 +77,24 @@ module Ci
.joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id')
.where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0').
# Implement fair scheduling
# this returns builds that are ordered by number of running builds
# we prefer projects that don't use shared runners at all
joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id")
# Implement fair scheduling
# this returns builds that are ordered by number of running builds
# we prefer projects that don't use shared runners at all
joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id")
.order('COALESCE(project_builds.running_builds, 0) ASC', 'ci_builds.id ASC')
end
def builds_for_specific_runner
new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('created_at ASC')
def builds_for_project_runner
new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('id ASC')
end
def builds_for_group_runner
hierarchy_groups = Gitlab::GroupHierarchy.new(runner.groups).base_and_descendants
projects = Project.where(namespace_id: hierarchy_groups)
.with_group_runners_enabled
.with_builds_enabled
.without_deleted
new_builds.where(project: projects).order('id ASC')
end
def running_builds_for_shared_runners
......@@ -97,10 +108,6 @@ module Ci
builds
end
def shared_runner_build_limits_feature_enabled?
ENV['DISABLE_SHARED_RUNNER_BUILD_MINUTES_LIMIT'].to_s != 'true'
end
def register_failure
failed_attempt_counter.increment
attempt_counter.increment
......
module Ci
class UpdateBuildQueueService
def execute(build)
build.project.runners.each do |runner|
if runner.can_pick?(build)
runner.tick_runner_queue
end
end
tick_for(build, build.project.all_runners)
end
return unless build.project.shared_runners_enabled?
private
Ci::Runner.shared.each do |runner|
if runner.can_pick?(build)
runner.tick_runner_queue
end
def tick_for(build, runners)
runners.each do |runner|
runner.pick_build!(build)
end
end
end
......
......@@ -2,6 +2,8 @@
%td
- if runner.shared?
%span.label.label-success shared
- elsif runner.group_type?
%span.label.label-success group
- else
%span.label.label-info specific
- if runner.locked?
......@@ -19,7 +21,7 @@
%td
= runner.ip_address
%td
- if runner.shared?
- if runner.shared? || runner.group_type?
n/a
- else
= runner.projects.count(:all)
......
......@@ -16,6 +16,9 @@
%li
%span.label.label-success shared
\- Runner runs jobs from all unassigned projects
%li
%span.label.label-success group
\- Runner runs jobs from all unassigned projects in its group
%li
%span.label.label-info specific
\- Runner runs jobs from assigned projects
......
......@@ -19,6 +19,9 @@
%p
If you want Runners to build only specific projects, enable them in the table below.
Keep in mind that this is a one way transition.
- elsif @runner.group_type?
.bs-callout.bs-callout-success
%h4 This runner will process jobs from all projects in its group and subgroups
- else
.bs-callout.bs-callout-info
%h4 This Runner will process jobs only from ASSIGNED projects
......
%h3 Group Runners
.bs-callout.bs-callout-warning
GitLab Group Runners can execute code for all the projects in this group.
They can be managed using the #{link_to 'Runners API', help_page_path('api/runners.md')}.
- if @project.group
%hr
- if @project.group_runners_enabled?
= link_to toggle_group_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do
Disable group Runners
- else
= link_to toggle_group_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do
Enable group Runners
&nbsp; for this project
- if !@project.group
This project does not belong to a group and can therefore not make use of group Runners.
- elsif @group_runners.empty?
This group does not provide any group Runners yet.
- if can?(current_user, :admin_pipeline, @project.group)
= render partial: 'ci/runner/how_to_setup_runner',
locals: { registration_token: @project.group.runners_token, type: 'group' }
- else
Ask your group master to setup a group Runner.
- else
%h4.underlined-title Available group Runners : #{@group_runners.count}
%ul.bordered-list
= render partial: 'projects/runners/runner', collection: @group_runners, as: :runner
......@@ -23,3 +23,7 @@
= render 'projects/runners/specific_runners'
.col-sm-6
= render 'projects/runners/shared_runners'
.row
.col-sm-6
.col-sm-6
= render 'projects/runners/group_runners'
......@@ -26,7 +26,7 @@
- else
- runner_project = @project.runner_projects.find_by(runner_id: runner)
= link_to 'Disable for this project', project_runner_project_path(@project, runner_project), data: { confirm: "Are you sure?" }, method: :delete, class: 'btn btn-danger btn-sm'
- elsif runner.specific?
- elsif !(runner.is_shared? || runner.group_type?) # We can simplify this to `runner.project_type?` when migrating #runner_type is complete
= form_for [@project.namespace.becomes(Namespace), @project, @project.runner_projects.new] do |f|
= f.hidden_field :runner_id, value: runner.id
= f.submit 'Enable for this project', class: 'btn btn-sm'
......
---
title: Allow group masters to configure runners for groups
merge_request: 9646
author: Alexis Reigel
type: added
......@@ -409,6 +409,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
collection do
post :toggle_shared_runners
post :toggle_group_runners
end
end
......
class AddCiRunnerNamespaces < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :ci_runner_namespaces do |t|
t.integer :runner_id
t.integer :namespace_id
t.index [:runner_id, :namespace_id], unique: true
t.index :namespace_id
t.foreign_key :ci_runners, column: :runner_id, on_delete: :cascade
t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade
end
end
end
class AddRunnersTokenToGroups < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :namespaces, :runners_token, :string
end
end
class AddRunnerTypeToCiRunners < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :ci_runners, :runner_type, :smallint
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddIndexToNamespacesRunnersToken < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :namespaces, :runners_token, unique: true
end
def down
if index_exists?(:namespaces, :runners_token, unique: true)
remove_index :namespaces, :runners_token
end
end
end
class BackfillRunnerTypeForCiRunnersPostMigrate < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INSTANCE_RUNNER_TYPE = 1
PROJECT_RUNNER_TYPE = 3
disable_ddl_transaction!
def up
update_column_in_batches(:ci_runners, :runner_type, INSTANCE_RUNNER_TYPE) do |table, query|
query.where(table[:is_shared].eq(true)).where(table[:runner_type].eq(nil))
end
update_column_in_batches(:ci_runners, :runner_type, PROJECT_RUNNER_TYPE) do |table, query|
query.where(table[:is_shared].eq(false)).where(table[:runner_type].eq(nil))
end
end
def down
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20180425131009) do
ActiveRecord::Schema.define(version: 20180503150427) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -444,6 +444,14 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree
create_table "ci_runner_namespaces", force: :cascade do |t|
t.integer "runner_id"
t.integer "namespace_id"
end
add_index "ci_runner_namespaces", ["namespace_id"], name: "index_ci_runner_namespaces_on_namespace_id", using: :btree
add_index "ci_runner_namespaces", ["runner_id", "namespace_id"], name: "index_ci_runner_namespaces_on_runner_id_and_namespace_id", unique: true, using: :btree
create_table "ci_runner_projects", force: :cascade do |t|
t.integer "runner_id", null: false
t.datetime "created_at"
......@@ -472,6 +480,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
t.integer "access_level", default: 0, null: false
t.string "ip_address"
t.integer "maximum_timeout"
t.integer "runner_type", limit: 2
end
add_index "ci_runners", ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree
......@@ -1261,6 +1270,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
t.boolean "require_two_factor_authentication", default: false, null: false
t.integer "two_factor_grace_period", default: 48, null: false
t.integer "cached_markdown_version"
t.string "runners_token"
end
add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
......@@ -1271,6 +1281,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_index "namespaces", ["path"], name: "index_namespaces_on_path", using: :btree
add_index "namespaces", ["path"], name: "index_namespaces_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"}
add_index "namespaces", ["require_two_factor_authentication"], name: "index_namespaces_on_require_two_factor_authentication", using: :btree
add_index "namespaces", ["runners_token"], name: "index_namespaces_on_runners_token", unique: true, using: :btree
add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree
create_table "notes", force: :cascade do |t|
......@@ -2087,6 +2098,8 @@ ActiveRecord::Schema.define(version: 20180425131009) do
add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade
add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
......
......@@ -242,13 +242,18 @@ module API
expose :requested_at
end
class Group < Grape::Entity
expose :id, :name, :path, :description, :visibility
class BasicGroupDetails < Grape::Entity
expose :id
expose :web_url
expose :name
end
class Group < BasicGroupDetails
expose :path, :description, :visibility
expose :lfs_enabled?, as: :lfs_enabled
expose :avatar_url do |group, options|
group.avatar_url(only_path: false)
end
expose :web_url
expose :request_access_enabled
expose :full_name, :full_path
......@@ -984,6 +989,13 @@ module API
options[:current_user].authorized_projects.where(id: runner.projects)
end
end
expose :groups, with: Entities::BasicGroupDetails do |runner, options|
if options[:current_user].admin?
runner.groups
else
options[:current_user].authorized_groups.where(id: runner.groups)
end
end
end
class RunnerRegistrationDetails < Grape::Entity
......
......@@ -23,10 +23,13 @@ module API
runner =
if runner_registration_token_valid?
# Create shared runner. Requires admin access
Ci::Runner.create(attributes.merge(is_shared: true))
Ci::Runner.create(attributes.merge(is_shared: true, runner_type: :instance_type))
elsif project = Project.find_by(runners_token: params[:token])
# Create a specific runner for project.
project.runners.create(attributes)
# Create a specific runner for the project
project.runners.create(attributes.merge(runner_type: :project_type))
elsif group = Group.find_by(runners_token: params[:token])
# Create a specific runner for the group
group.runners.create(attributes.merge(runner_type: :group_type))
end
break forbidden! unless runner
......
......@@ -17,6 +17,23 @@ describe Projects::Settings::CiCdController do
expect(response).to have_gitlab_http_status(200)
expect(response).to render_template(:show)
end
context 'with group runners' do
let(:group_runner) { create(:ci_runner) }
let(:parent_group) { create(:group) }
let(:group) { create(:group, runners: [group_runner], parent: parent_group) }
let(:other_project) { create(:project, group: group) }
let!(:project_runner) { create(:ci_runner, projects: [other_project]) }
let!(:shared_runner) { create(:ci_runner, :shared) }
it 'sets assignable project runners only' do
group.add_master(user)
get :show, namespace_id: project.namespace, project_id: project
expect(assigns(:assignable_runners)).to eq [project_runner]
end
end
end
describe '#reset_cache' do
......
......@@ -15,14 +15,18 @@ FactoryBot.define do
namespace
creator { group ? create(:user) : namespace&.owner }
# Nest Project Feature attributes
transient do
# Nest Project Feature attributes
wiki_access_level ProjectFeature::ENABLED
builds_access_level ProjectFeature::ENABLED
snippets_access_level ProjectFeature::ENABLED
issues_access_level ProjectFeature::ENABLED
merge_requests_access_level ProjectFeature::ENABLED
repository_access_level ProjectFeature::ENABLED
# we can't assign the delegated `#ci_cd_settings` attributes directly, as the
# `#ci_cd_settings` relation needs to be created first
group_runners_enabled nil
end
after(:create) do |project, evaluator|
......@@ -47,6 +51,9 @@ FactoryBot.define do
end
project.group&.refresh_members_authorized_projects
# assign the delegated `#ci_cd_settings` attributes after create
project.reload.group_runners_enabled = evaluator.group_runners_enabled unless evaluator.group_runners_enabled.nil?
end
trait :public do
......
......@@ -59,6 +59,47 @@ describe "Admin Runners" do
expect(page).to have_text 'No runners found'
end
end
context 'group runner' do
let(:group) { create(:group) }
let!(:runner) { create(:ci_runner, groups: [group], runner_type: :group_type) }
it 'shows the label and does not show the project count' do
visit admin_runners_path
within "#runner_#{runner.id}" do
expect(page).to have_selector '.label', text: 'group'
expect(page).to have_text 'n/a'
end
end
end
context 'shared runner' do
it 'shows the label and does not show the project count' do
runner = create :ci_runner, :shared
visit admin_runners_path
within "#runner_#{runner.id}" do
expect(page).to have_selector '.label', text: 'shared'
expect(page).to have_text 'n/a'
end
end
end
context 'specific runner' do
it 'shows the label and the project count' do
project = create :project
runner = create :ci_runner, projects: [project]
visit admin_runners_path
within "#runner_#{runner.id}" do
expect(page).to have_selector '.label', text: 'specific'
expect(page).to have_text '1'
end
end
end
end
describe "Runner show page" do
......
......@@ -181,4 +181,84 @@ feature 'Runners' do
expect(page.find('.shared-runners-description')).to have_content('Disable shared Runners')
end
end
context 'group runners' do
background do
project.add_master(user)
end
given(:group) { create :group }
context 'as project and group master' do
background do
group.add_master(user)
end
context 'project with a group but no group runner' do
given(:project) { create :project, group: group }
scenario 'group runners are not available' do
visit runners_path(project)
expect(page).to have_content 'This group does not provide any group Runners yet.'
expect(page).to have_content 'Setup a group Runner manually'
expect(page).not_to have_content 'Ask your group master to setup a group Runner.'
end
end
end
context 'as project master' do
context 'project without a group' do
given(:project) { create :project }
scenario 'group runners are not available' do
visit runners_path(project)
expect(page).to have_content 'This project does not belong to a group and can therefore not make use of group Runners.'
end
end
context 'project with a group but no group runner' do
given(:group) { create :group }
given(:project) { create :project, group: group }
scenario 'group runners are not available' do
visit runners_path(project)
expect(page).to have_content 'This group does not provide any group Runners yet.'
expect(page).not_to have_content 'Setup a group Runner manually'
expect(page).to have_content 'Ask your group master to setup a group Runner.'
end
end
context 'project with a group and a group runner' do
given(:group) { create :group }
given(:project) { create :project, group: group }
given!(:ci_runner) { create :ci_runner, groups: [group], description: 'group-runner' }
scenario 'group runners are available' do
visit runners_path(project)
expect(page).to have_content 'Available group Runners : 1'
expect(page).to have_content 'group-runner'
end
scenario 'group runners may be disabled for a project' do
visit runners_path(project)
click_on 'Disable group Runners'
expect(page).to have_content 'Enable group Runners'
expect(project.reload.group_runners_enabled).to be false
click_on 'Enable group Runners'
expect(page).to have_content 'Disable group Runners'
expect(project.reload.group_runners_enabled).to be true
end
end
end
end
end
......@@ -258,7 +258,6 @@ project:
- builds
- runner_projects
- runners
- active_runners
- variables
- triggers
- pipeline_schedules
......@@ -286,6 +285,7 @@ project:
- internal_ids
- project_deploy_tokens
- deploy_tokens
- settings
- ci_cd_settings
award_emoji:
- awardable
......
......@@ -19,6 +19,63 @@ describe Ci::Runner do
end
end
end
context 'either_projects_or_group' do
let(:group) { create(:group) }
it 'disallows assigning to a group if already assigned to a group' do
runner = create(:ci_runner, groups: [group])
runner.groups << build(:group)
expect(runner).not_to be_valid
expect(runner.errors.full_messages).to eq ['Runner can only be assigned to one group']
end
it 'disallows assigning to a group if already assigned to a project' do
project = create(:project)
runner = create(:ci_runner, projects: [project])
runner.groups << build(:group)
expect(runner).not_to be_valid
expect(runner.errors.full_messages).to eq ['Runner can only be assigned either to projects or to a group']
end
it 'disallows assigning to a project if already assigned to a group' do
runner = create(:ci_runner, groups: [group])
runner.projects << build(:project)
expect(runner).not_to be_valid
expect(runner.errors.full_messages).to eq ['Runner can only be assigned either to projects or to a group']
end
it 'allows assigning to a group if not assigned to a group nor a project' do
runner = create(:ci_runner)
runner.groups << build(:group)
expect(runner).to be_valid
end
it 'allows assigning to a project if not assigned to a group nor a project' do
runner = create(:ci_runner)
runner.projects << build(:project)
expect(runner).to be_valid
end
it 'allows assigning to a project if already assigned to a project' do
project = create(:project)
runner = create(:ci_runner, projects: [project])
runner.projects << build(:project)
expect(runner).to be_valid
end
end
end
describe '#access_level' do
......@@ -49,6 +106,80 @@ describe Ci::Runner do
end
end
describe '.shared' do
let(:group) { create(:group) }
let(:project) { create(:project) }
it 'returns the shared group runner' do
runner = create(:ci_runner, :shared, groups: [group])
expect(described_class.shared).to eq [runner]
end
it 'returns the shared project runner' do
runner = create(:ci_runner, :shared, projects: [project])
expect(described_class.shared).to eq [runner]
end
end
describe '.belonging_to_project' do
it 'returns the specific project runner' do
# own
specific_project = create(:project)
specific_runner = create(:ci_runner, :specific, projects: [specific_project])
# other
other_project = create(:project)
create(:ci_runner, :specific, projects: [other_project])
expect(described_class.belonging_to_project(specific_project.id)).to eq [specific_runner]
end
end
describe '.belonging_to_parent_group_of_project' do
let(:project) { create(:project, group: group) }
let(:group) { create(:group) }
let(:runner) { create(:ci_runner, :specific, groups: [group]) }
let!(:unrelated_group) { create(:group) }
let!(:unrelated_project) { create(:project, group: unrelated_group) }
let!(:unrelated_runner) { create(:ci_runner, :specific, groups: [unrelated_group]) }
it 'returns the specific group runner' do
expect(described_class.belonging_to_parent_group_of_project(project.id)).to contain_exactly(runner)
end
context 'with a parent group with a runner', :nested_groups do
let(:runner) { create(:ci_runner, :specific, groups: [parent_group]) }
let(:project) { create(:project, group: group) }
let(:group) { create(:group, parent: parent_group) }
let(:parent_group) { create(:group) }
it 'returns the group runner from the parent group' do
expect(described_class.belonging_to_parent_group_of_project(project.id)).to contain_exactly(runner)
end
end
end
describe '.owned_or_shared' do
it 'returns a globally shared, a project specific and a group specific runner' do
# group specific
group = create(:group)
project = create(:project, group: group)
group_runner = create(:ci_runner, :specific, groups: [group])
# project specific
project_runner = create(:ci_runner, :specific, projects: [project])
# globally shared
shared_runner = create(:ci_runner, :shared)
expect(described_class.owned_or_shared(project.id)).to contain_exactly(
group_runner, project_runner, shared_runner
)
end
end
describe '#display_name' do
it 'returns the description if it has a value' do
runner = FactoryBot.build(:ci_runner, description: 'Linux/Ruby-1.9.3-p448')
......@@ -163,7 +294,9 @@ describe Ci::Runner do
describe '#can_pick?' do
let(:pipeline) { create(:ci_pipeline) }
let(:build) { create(:ci_build, pipeline: pipeline) }
let(:runner) { create(:ci_runner) }
let(:runner) { create(:ci_runner, tag_list: tag_list, run_untagged: run_untagged) }
let(:tag_list) { [] }
let(:run_untagged) { true }
subject { runner.can_pick?(build) }
......@@ -171,6 +304,13 @@ describe Ci::Runner do
build.project.runners << runner
end
context 'a different runner' do
it 'cannot handle builds' do
other_runner = create(:ci_runner)
expect(other_runner.can_pick?(build)).to be_falsey
end
end
context 'when runner does not have tags' do
it 'can handle builds without tags' do
expect(runner.can_pick?(build)).to be_truthy
......@@ -184,9 +324,7 @@ describe Ci::Runner do
end
context 'when runner has tags' do
before do
runner.tag_list = %w(bb cc)
end
let(:tag_list) { %w(bb cc) }
shared_examples 'tagged build picker' do
it 'can handle build with matching tags' do
......@@ -211,9 +349,7 @@ describe Ci::Runner do
end
context 'when runner cannot pick untagged jobs' do
before do
runner.run_untagged = false
end
let(:run_untagged) { false }
it 'cannot handle builds without tags' do
expect(runner.can_pick?(build)).to be_falsey
......@@ -224,8 +360,9 @@ describe Ci::Runner do
end
context 'when runner is shared' do
let(:runner) { create(:ci_runner, :shared) }
before do
runner.is_shared = true
build.project.runners = []
end
......@@ -234,9 +371,7 @@ describe Ci::Runner do
end
context 'when runner is locked' do
before do
runner.locked = true
end
let(:runner) { create(:ci_runner, :shared, locked: true) }
it 'can handle builds' do
expect(runner.can_pick?(build)).to be_truthy
......@@ -260,6 +395,17 @@ describe Ci::Runner do
expect(runner.can_pick?(build)).to be_falsey
end
end
context 'when runner is assigned to a group' do
before do
build.project.runners = []
runner.groups << create(:group, projects: [build.project])
end
it 'can handle builds' do
expect(runner.can_pick?(build)).to be_truthy
end
end
end
context 'when access_level of runner is not_protected' do
......@@ -583,4 +729,76 @@ describe Ci::Runner do
expect(described_class.search(runner.description.upcase)).to eq([runner])
end
end
describe '#assigned_to_group?' do
subject { runner.assigned_to_group? }
context 'when project runner' do
let(:runner) { create(:ci_runner, description: 'Project runner', projects: [project]) }
let(:project) { create(:project) }
it { is_expected.to be_falsey }
end
context 'when shared runner' do
let(:runner) { create(:ci_runner, :shared, description: 'Shared runner') }
it { is_expected.to be_falsey }
end
context 'when group runner' do
let(:group) { create(:group) }
let(:runner) { create(:ci_runner, description: 'Group runner', groups: [group]) }
it { is_expected.to be_truthy }
end
end
describe '#assigned_to_project?' do
subject { runner.assigned_to_project? }
context 'when group runner' do
let(:runner) { create(:ci_runner, description: 'Group runner', groups: [group]) }
let(:group) { create(:group) }
it { is_expected.to be_falsey }
end
context 'when shared runner' do
let(:runner) { create(:ci_runner, :shared, description: 'Shared runner') }
it { is_expected.to be_falsey }
end
context 'when project runner' do
let(:runner) { create(:ci_runner, description: 'Group runner', projects: [project]) }
let(:project) { create(:project) }
it { is_expected.to be_truthy }
end
end
describe '#pick_build!' do
context 'runner can pick the build' do
it 'calls #tick_runner_queue' do
ci_build = build(:ci_build)
runner = build(:ci_runner)
allow(runner).to receive(:can_pick?).with(ci_build).and_return(true)
expect(runner).to receive(:tick_runner_queue)
runner.pick_build!(ci_build)
end
end
context 'runner cannot pick the build' do
it 'does not call #tick_runner_queue' do
ci_build = build(:ci_build)
runner = build(:ci_runner)
allow(runner).to receive(:can_pick?).with(ci_build).and_return(false)
expect(runner).not_to receive(:tick_runner_queue)
runner.pick_build!(ci_build)
end
end
end
end
......@@ -63,7 +63,6 @@ describe Project do
it { is_expected.to have_many(:build_trace_section_names)}
it { is_expected.to have_many(:runner_projects) }
it { is_expected.to have_many(:runners) }
it { is_expected.to have_many(:active_runners) }
it { is_expected.to have_many(:variables) }
it { is_expected.to have_many(:triggers) }
it { is_expected.to have_many(:pages_domains) }
......@@ -1139,45 +1138,106 @@ describe Project do
end
end
describe '#any_runners' do
let(:project) { create(:project, shared_runners_enabled: shared_runners_enabled) }
let(:specific_runner) { create(:ci_runner) }
let(:shared_runner) { create(:ci_runner, :shared) }
describe '#any_runners?' do
context 'shared runners' do
let(:project) { create :project, shared_runners_enabled: shared_runners_enabled }
let(:specific_runner) { create :ci_runner }
let(:shared_runner) { create :ci_runner, :shared }
context 'for shared runners disabled' do
let(:shared_runners_enabled) { false }
context 'for shared runners disabled' do
let(:shared_runners_enabled) { false }
it 'has no runners available' do
expect(project.any_runners?).to be_falsey
end
it 'has no runners available' do
expect(project.any_runners?).to be_falsey
end
it 'has a specific runner' do
project.runners << specific_runner
expect(project.any_runners?).to be_truthy
end
it 'has a specific runner' do
project.runners << specific_runner
expect(project.any_runners?).to be_truthy
end
it 'has a shared runner, but they are prohibited to use' do
shared_runner
expect(project.any_runners?).to be_falsey
end
it 'checks the presence of specific runner' do
project.runners << specific_runner
expect(project.any_runners? { |runner| runner == specific_runner }).to be_truthy
end
it 'has a shared runner, but they are prohibited to use' do
shared_runner
expect(project.any_runners?).to be_falsey
it 'returns false if match cannot be found' do
project.runners << specific_runner
expect(project.any_runners? { false }).to be_falsey
end
end
it 'checks the presence of specific runner' do
project.runners << specific_runner
expect(project.any_runners? { |runner| runner == specific_runner }).to be_truthy
context 'for shared runners enabled' do
let(:shared_runners_enabled) { true }
it 'has a shared runner' do
shared_runner
expect(project.any_runners?).to be_truthy
end
it 'checks the presence of shared runner' do
shared_runner
expect(project.any_runners? { |runner| runner == shared_runner }).to be_truthy
end
it 'returns false if match cannot be found' do
shared_runner
expect(project.any_runners? { false }).to be_falsey
end
end
end
context 'for shared runners enabled' do
let(:shared_runners_enabled) { true }
context 'group runners' do
let(:project) { create :project, group_runners_enabled: group_runners_enabled }
let(:group) { create :group, projects: [project] }
let(:group_runner) { create :ci_runner, groups: [group] }
context 'for group runners disabled' do
let(:group_runners_enabled) { false }
it 'has no runners available' do
expect(project.any_runners?).to be_falsey
end
it 'has a group runner, but they are prohibited to use' do
group_runner
it 'has a shared runner' do
shared_runner
expect(project.any_runners?).to be_truthy
expect(project.any_runners?).to be_falsey
end
end
it 'checks the presence of shared runner' do
shared_runner
expect(project.any_runners? { |runner| runner == shared_runner }).to be_truthy
context 'for group runners enabled' do
let(:group_runners_enabled) { true }
it 'has a group runner' do
group_runner
expect(project.any_runners?).to be_truthy
end
it 'checks the presence of group runner' do
group_runner
expect(project.any_runners? { |runner| runner == group_runner }).to be_truthy
end
it 'returns false if match cannot be found' do
group_runner
expect(project.any_runners? { false }).to be_falsey
end
end
end
end
......@@ -3541,6 +3601,18 @@ describe Project do
end
end
describe '#toggle_ci_cd_settings!' do
it 'toggles the value on #settings' do
project = create(:project, group_runners_enabled: false)
expect(project.group_runners_enabled).to be false
project.toggle_ci_cd_settings!(:group_runners_enabled)
expect(project.group_runners_enabled).to be true
end
end
describe '#gitlab_deploy_token' do
let(:project) { create(:project) }
......
......@@ -40,18 +40,36 @@ describe API::Runner do
expect(json_response['token']).to eq(runner.token)
expect(runner.run_untagged).to be true
expect(runner.token).not_to eq(registration_token)
expect(runner).to be_instance_type
end
context 'when project token is used' do
let(:project) { create(:project) }
it 'creates runner' do
it 'creates project runner' do
post api('/runners'), token: project.runners_token
expect(response).to have_gitlab_http_status 201
expect(project.runners.size).to eq(1)
expect(Ci::Runner.first.token).not_to eq(registration_token)
expect(Ci::Runner.first.token).not_to eq(project.runners_token)
runner = Ci::Runner.first
expect(runner.token).not_to eq(registration_token)
expect(runner.token).not_to eq(project.runners_token)
expect(runner).to be_project_type
end
end
context 'when group token is used' do
let(:group) { create(:group) }
it 'creates a group runner' do
post api('/runners'), token: group.runners_token
expect(response).to have_http_status 201
expect(group.runners.size).to eq(1)
runner = Ci::Runner.first
expect(runner.token).not_to eq(registration_token)
expect(runner.token).not_to eq(group.runners_token)
expect(runner).to be_group_type
end
end
end
......
This diff is collapsed.
......@@ -118,7 +118,7 @@ describe PipelineSerializer do
it 'verifies number of queries', :request_store do
recorded = ActiveRecord::QueryRecorder.new { subject }
expect(recorded.count).to be_within(1).of(36)
expect(recorded.count).to be_within(1).of(44)
expect(recorded.cached_count).to eq(0)
end
end
......
......@@ -2,11 +2,13 @@ require 'spec_helper'
module Ci
describe RegisterJobService do
let!(:project) { FactoryBot.create :project, shared_runners_enabled: false }
let!(:pipeline) { FactoryBot.create :ci_pipeline, project: project }
let!(:pending_job) { FactoryBot.create :ci_build, pipeline: pipeline }
let!(:shared_runner) { FactoryBot.create(:ci_runner, is_shared: true) }
let!(:specific_runner) { FactoryBot.create(:ci_runner, is_shared: false) }
set(:group) { create(:group) }
set(:project) { create(:project, group: group, shared_runners_enabled: false, group_runners_enabled: false) }
set(:pipeline) { create(:ci_pipeline, project: project) }
let!(:shared_runner) { create(:ci_runner, is_shared: true) }
let!(:specific_runner) { create(:ci_runner, is_shared: false) }
let!(:group_runner) { create(:ci_runner, groups: [group], runner_type: :group_type) }
let!(:pending_job) { create(:ci_build, pipeline: pipeline) }
before do
specific_runner.assign_to(project)
......@@ -150,7 +152,7 @@ module Ci
context 'disallow when builds are disabled' do
before do
project.update(shared_runners_enabled: true)
project.update(shared_runners_enabled: true, group_runners_enabled: true)
project.project_feature.update_attribute(:builds_access_level, ProjectFeature::DISABLED)
end
......@@ -160,13 +162,90 @@ module Ci
it { expect(build).to be_nil }
end
context 'and uses specific runner' do
context 'and uses group runner' do
let(:build) { execute(group_runner) }
it { expect(build).to be_nil }
end
context 'and uses project runner' do
let(:build) { execute(specific_runner) }
it { expect(build).to be_nil }
end
end
context 'allow group runners' do
before do
project.update!(group_runners_enabled: true)
end
context 'for multiple builds' do
let!(:project2) { create :project, group_runners_enabled: true, group: group }
let!(:pipeline2) { create :ci_pipeline, project: project2 }
let!(:project3) { create :project, group_runners_enabled: true, group: group }
let!(:pipeline3) { create :ci_pipeline, project: project3 }
let!(:build1_project1) { pending_job }
let!(:build2_project1) { create :ci_build, pipeline: pipeline }
let!(:build3_project1) { create :ci_build, pipeline: pipeline }
let!(:build1_project2) { create :ci_build, pipeline: pipeline2 }
let!(:build2_project2) { create :ci_build, pipeline: pipeline2 }
let!(:build1_project3) { create :ci_build, pipeline: pipeline3 }
# these shouldn't influence the scheduling
let!(:unrelated_group) { create :group }
let!(:unrelated_project) { create :project, group_runners_enabled: true, group: unrelated_group }
let!(:unrelated_pipeline) { create :ci_pipeline, project: unrelated_project }
let!(:build1_unrelated_project) { create :ci_build, pipeline: unrelated_pipeline }
let!(:unrelated_group_runner) { create :ci_runner, groups: [unrelated_group] }
it 'does not consider builds from other group runners' do
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 6
execute(group_runner)
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 5
execute(group_runner)
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 4
execute(group_runner)
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 3
execute(group_runner)
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 2
execute(group_runner)
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 1
execute(group_runner)
expect(described_class.new(group_runner).send(:builds_for_group_runner).count).to eq 0
expect(execute(group_runner)).to be_nil
end
end
context 'group runner' do
let(:build) { execute(group_runner) }
it { expect(build).to be_kind_of(Build) }
it { expect(build).to be_valid }
it { expect(build).to be_running }
it { expect(build.runner).to eq(group_runner) }
end
end
context 'disallow group runners' do
before do
project.update!(group_runners_enabled: false)
end
context 'group runner' do
let(:build) { execute(group_runner) }
it { expect(build).to be_nil }
end
end
context 'when first build is stalled' do
before do
pending_job.update(lock_version: 0)
......@@ -178,7 +257,7 @@ module Ci
let!(:other_build) { create :ci_build, pipeline: pipeline }
before do
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_project_runner)
.and_return(Ci::Build.where(id: [pending_job, other_build]))
end
......@@ -190,7 +269,7 @@ module Ci
context 'when single build is in queue' do
before do
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_project_runner)
.and_return(Ci::Build.where(id: pending_job))
end
......@@ -201,7 +280,7 @@ module Ci
context 'when there is no build in queue' do
before do
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_project_runner)
.and_return(Ci::Build.none)
end
......
......@@ -8,21 +8,19 @@ describe Ci::UpdateBuildQueueService do
context 'when updating specific runners' do
let(:runner) { create(:ci_runner) }
context 'when there are runner that can pick build' do
context 'when there is a runner that can pick build' do
before do
build.project.runners << runner
end
it 'ticks runner queue value' do
expect { subject.execute(build) }
.to change { runner.ensure_runner_queue_value }
expect { subject.execute(build) }.to change { runner.ensure_runner_queue_value }
end
end
context 'when there are no runners that can pick build' do
context 'when there is no runner that can pick build' do
it 'does not tick runner queue value' do
expect { subject.execute(build) }
.not_to change { runner.ensure_runner_queue_value }
expect { subject.execute(build) }.not_to change { runner.ensure_runner_queue_value }
end
end
end
......@@ -30,21 +28,61 @@ describe Ci::UpdateBuildQueueService do
context 'when updating shared runners' do
let(:runner) { create(:ci_runner, :shared) }
context 'when there are runner that can pick build' do
context 'when there is no runner that can pick build' do
it 'ticks runner queue value' do
expect { subject.execute(build) }
.to change { runner.ensure_runner_queue_value }
expect { subject.execute(build) }.to change { runner.ensure_runner_queue_value }
end
end
context 'when there are no runners that can pick build' do
context 'when there is no runner that can pick build due to tag mismatch' do
before do
build.tag_list = [:docker]
end
it 'does not tick runner queue value' do
expect { subject.execute(build) }
.not_to change { runner.ensure_runner_queue_value }
expect { subject.execute(build) }.not_to change { runner.ensure_runner_queue_value }
end
end
context 'when there is no runner that can pick build due to being disabled on project' do
before do
build.project.shared_runners_enabled = false
end
it 'does not tick runner queue value' do
expect { subject.execute(build) }.not_to change { runner.ensure_runner_queue_value }
end
end
end
context 'when updating group runners' do
let(:group) { create :group }
let(:project) { create :project, group: group }
let(:runner) { create :ci_runner, groups: [group] }
context 'when there is a runner that can pick build' do
it 'ticks runner queue value' do
expect { subject.execute(build) }.to change { runner.ensure_runner_queue_value }
end
end
context 'when there is no runner that can pick build due to tag mismatch' do
before do
build.tag_list = [:docker]
end
it 'does not tick runner queue value' do
expect { subject.execute(build) }.not_to change { runner.ensure_runner_queue_value }
end
end
context 'when there is no runner that can pick build due to being disabled on project' do
before do
build.project.group_runners_enabled = false
end
it 'does not tick runner queue value' do
expect { subject.execute(build) }.not_to change { runner.ensure_runner_queue_value }
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment