Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all

.gitignore

@@ -68,6 +68,8 @@ eslint-report.html
 /shared/*
 /.gitlab_workhorse_secret
 /webpack-report/
+/knapsack/
+/rspec_flaky/
 /locale/**/LC_MESSAGES
 /locale/**/*.time_stamp
 /.rspec

app/assets/javascripts/issuable_form.js

@@ -30,7 +30,7 @@ export default class IssuableForm {
     }
 
     this.initAutosave();
-    this.form.on('submit', this.handleSubmit);
+    this.form.on('submit:success', this.handleSubmit);
     this.form.on('click', '.btn-cancel', this.resetAutosave);
     this.initWip();
 

app/assets/javascripts/pages/projects/pipelines/new/index.js

 import $ from 'jquery';
 import NewBranchForm from '~/new_branch_form';
+import setupNativeFormVariableList from '~/ci_variable_list/native_form_variable_list';
 
 document.addEventListener('DOMContentLoaded', () => {
   new NewBranchForm($('.js-new-pipeline-form')); // eslint-disable-line no-new
+
+  setupNativeFormVariableList({
+    container: $('.js-ci-variable-list-section'),
+    formField: 'variables_attributes',
+  });
 });

app/assets/stylesheets/framework.scss

@@ -61,3 +61,4 @@
 @import 'framework/stacked_progress_bar';
 @import 'framework/ci_variable_list';
 @import 'framework/feature_highlight';
+@import 'framework/terms';

app/assets/stylesheets/framework/terms.scss

+.terms {
+  .alert-wrapper {
+    min-height: $header-height + $gl-padding;
+  }
+
+  .content {
+    padding-top: $gl-padding;
+  }
+
+  .panel {
+    .panel-heading {
+      display: -webkit-flex;
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+
+      .title {
+        display: flex;
+        align-items: center;
+
+        .logo-text {
+          width: 55px;
+          height: 24px;
+          display: flex;
+          flex-direction: column;
+          justify-content: center;
+        }
+      }
+
+      .navbar-collapse {
+        padding-right: 0;
+      }
+
+      .nav li a {
+        color: $theme-gray-700;
+      }
+    }
+
+    .panel-content {
+      padding: $gl-padding;
+
+      *:first-child {
+        margin-top: 0;
+      }
+
+      *:last-child {
+        margin-bottom: 0;
+      }
+    }
+
+    .footer-block {
+      margin: 0;
+    }
+  }
+}

app/assets/stylesheets/pages/repo.scss

@@ -440,6 +440,7 @@
   padding-right: 3px;
 
   .projects-sidebar {
+    min-height: 0;
     display: flex;
     flex-direction: column;
     flex: 1;

app/controllers/application_controller.rb

@@ -13,12 +13,14 @@ class ApplicationController < ActionController::Base
 
   before_action :authenticate_sessionless_user!
   before_action :authenticate_user!
+  before_action :enforce_terms!, if: -> { Gitlab::CurrentSettings.current_application_settings.enforce_terms },
+                                 unless: :peek_request?
   before_action :validate_user_service_ticket!
   before_action :check_password_expiration
   before_action :ldap_security_check
   before_action :sentry_context
   before_action :default_headers
-  before_action :add_gon_variables, unless: -> { request.path.start_with?('/-/peek') }
+  before_action :add_gon_variables, unless: :peek_request?
   before_action :configure_permitted_parameters, if: :devise_controller?
   before_action :require_email, unless: :devise_controller?
 
@@ -269,6 +271,27 @@ class ApplicationController < ActionController::Base
     end
   end
 
+  def enforce_terms!
+    return unless current_user
+    return if current_user.terms_accepted?
+
+    if sessionless_user?
+      render_403
+    else
+      # Redirect to the destination if the request is a get.
+      # Redirect to the source if it was a post, so the user can re-submit after
+      # accepting the terms.
+      redirect_path = if request.get?
+                        request.fullpath
+                      else
+                        URI(request.referer).path if request.referer
+                      end
+
+      flash[:notice] = _("Please accept the Terms of Service before continuing.")
+      redirect_to terms_path(redirect: redirect_path), status: :found
+    end
+  end
+
   def import_sources_enabled?
     !Gitlab::CurrentSettings.import_sources.empty?
   end
@@ -342,4 +365,12 @@ class ApplicationController < ActionController::Base
     # Per https://tools.ietf.org/html/rfc5987, headers need to be ISO-8859-1, not UTF-8
     response.headers['Page-Title'] = URI.escape(page_title('GitLab'))
   end
+
+  def sessionless_user?
+    current_user && !session.keys.include?('warden.user.user.key')
+  end
+
+  def peek_request?
+    request.path.start_with?('/-/peek')
+  end
 end

app/controllers/concerns/continue_params.rb

 module ContinueParams
+  include InternalRedirect
   extend ActiveSupport::Concern
 
   def continue_params
@@ -6,8 +7,7 @@ module ContinueParams
     return nil unless continue_params
 
     continue_params = continue_params.permit(:to, :notice, :notice_now)
-    return unless continue_params[:to] && continue_params[:to].start_with?('/')
-    return if continue_params[:to].start_with?('//')
+    continue_params[:to] = safe_redirect_path(continue_params[:to])
 
     continue_params
   end

app/controllers/concerns/internal_redirect.rb

+module InternalRedirect
+  extend ActiveSupport::Concern
+
+  def safe_redirect_path(path)
+    return unless path
+    # Verify that the string starts with a `/` but not a double `/`.
+    return unless path =~ %r{^/\w.*$}
+
+    uri = URI(path)
+    # Ignore anything path of the redirect except for the path, querystring and,
+    # fragment, forcing the redirect within the same host.
+    full_path_for_uri(uri)
+  rescue URI::InvalidURIError
+    nil
+  end
+
+  def safe_redirect_path_for_url(url)
+    return unless url
+
+    uri = URI(url)
+    safe_redirect_path(full_path_for_uri(uri)) if host_allowed?(uri)
+  rescue URI::InvalidURIError
+    nil
+  end
+
+  def host_allowed?(uri)
+    uri.host == request.host &&
+      uri.port == request.port
+  end
+
+  def full_path_for_uri(uri)
+    path_with_query = [uri.path, uri.query].compact.join('?')
+    [path_with_query, uri.fragment].compact.join("#")
+  end
+end

app/controllers/import/base_controller.rb

 class Import::BaseController < ApplicationController
   private
 
+  def find_already_added_projects(import_type)
+    current_user.created_projects.where(import_type: import_type).includes(:import_state)
+  end
+
+  def find_jobs(import_type)
+    current_user.created_projects
+        .includes(:import_state)
+        .where(import_type: import_type)
+        .to_json(only: [:id], methods: [:import_status])
+  end
+
   def find_or_create_namespace(names, owner)
     names = params[:target_namespace].presence || names
 

app/controllers/import/bitbucket_controller.rb

@@ -22,16 +22,14 @@ class Import::BitbucketController < Import::BaseController
 
     @repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
 
-    @already_added_projects = current_user.created_projects.where(import_type: 'bitbucket')
+    @already_added_projects = find_already_added_projects('bitbucket')
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
     @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) }
   end
 
   def jobs
-    render json: current_user.created_projects
-                             .where(import_type: 'bitbucket')
-                             .to_json(only: [:id, :import_status])
+    render json: find_jobs('bitbucket')
   end
 
   def create

app/controllers/import/fogbugz_controller.rb

@@ -46,15 +46,14 @@ class Import::FogbugzController < Import::BaseController
 
     @repos = client.repos
 
-    @already_added_projects = current_user.created_projects.where(import_type: 'fogbugz')
+    @already_added_projects = find_already_added_projects('fogbugz')
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
     @repos.reject! { |repo| already_added_projects_names.include? repo.name }
   end
 
   def jobs
-    jobs = current_user.created_projects.where(import_type: 'fogbugz').to_json(only: [:id, :import_status])
-    render json: jobs
+    render json: find_jobs('fogbugz')
   end
 
   def create

app/controllers/import/github_controller.rb

@@ -24,15 +24,14 @@ class Import::GithubController < Import::BaseController
 
   def status
     @repos = client.repos
-    @already_added_projects = current_user.created_projects.where(import_type: provider)
+    @already_added_projects = find_already_added_projects(provider)
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
     @repos.reject! { |repo| already_added_projects_names.include? repo.full_name }
   end
 
   def jobs
-    jobs = current_user.created_projects.where(import_type: provider).to_json(only: [:id, :import_status])
-    render json: jobs
+    render json: find_jobs(provider)
   end
 
   def create

app/controllers/import/gitlab_controller.rb

@@ -12,15 +12,14 @@ class Import::GitlabController < Import::BaseController
   def status
     @repos = client.projects
 
-    @already_added_projects = current_user.created_projects.where(import_type: "gitlab")
+    @already_added_projects = find_already_added_projects('gitlab')
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
     @repos = @repos.to_a.reject { |repo| already_added_projects_names.include? repo["path_with_namespace"] }
   end
 
   def jobs
-    jobs = current_user.created_projects.where(import_type: "gitlab").to_json(only: [:id, :import_status])
-    render json: jobs
+    render json: find_jobs('gitlab')
   end
 
   def create

app/controllers/import/google_code_controller.rb

@@ -73,15 +73,14 @@ class Import::GoogleCodeController < Import::BaseController
     @repos = client.repos
     @incompatible_repos = client.incompatible_repos
 
-    @already_added_projects = current_user.created_projects.where(import_type: "google_code")
+    @already_added_projects = find_already_added_projects('google_code')
     already_added_projects_names = @already_added_projects.pluck(:import_source)
 
     @repos.reject! { |repo| already_added_projects_names.include? repo.name }
   end
 
   def jobs
-    jobs = current_user.created_projects.where(import_type: "google_code").to_json(only: [:id, :import_status])
-    render json: jobs
+    render json: find_jobs('google_code')
   end
 
   def create

app/controllers/omniauth_callbacks_controller.rb

@@ -82,7 +82,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController
 
       if identity_linker.changed?
         redirect_identity_linked
-      elsif identity_linker.error_message.present?
+      elsif identity_linker.failed?
         redirect_identity_link_failed(identity_linker.error_message)
       else
         redirect_identity_exists

app/controllers/projects/pipelines_controller.rb

@@ -157,7 +157,7 @@ class Projects::PipelinesController < Projects::ApplicationController
   end
 
   def create_params
-    params.require(:pipeline).permit(:ref)
+    params.require(:pipeline).permit(:ref, variables_attributes: %i[key secret_value])
   end
 
   def pipeline

app/controllers/projects/runners_controller.rb

@@ -52,6 +52,12 @@ class Projects::RunnersController < Projects::ApplicationController
     redirect_to project_settings_ci_cd_path(@project)
   end
 
+  def toggle_group_runners
+    project.toggle_ci_cd_settings!(:group_runners_enabled)
+
+    redirect_to project_settings_ci_cd_path(@project)
+  end
+
   protected
 
   def set_runner

app/controllers/projects/settings/ci_cd_controller.rb

@@ -67,10 +67,18 @@ module Projects
 
       def define_runners_variables
         @project_runners = @project.runners.ordered
-        @assignable_runners = current_user.ci_authorized_runners
-          .assignable_for(project).ordered.page(params[:page]).per(20)
+
+        @assignable_runners = current_user
+          .ci_authorized_runners
+          .assignable_for(project)
+          .ordered
+          .page(params[:page]).per(20)
+
         @shared_runners = ::Ci::Runner.shared.active
+
         @shared_runners_count = @shared_runners.count(:all)
+
+        @group_runners = ::Ci::Runner.belonging_to_parent_group_of_project(@project.id)
       end
 
       def define_secret_variables

app/controllers/sessions_controller.rb

 class SessionsController < Devise::SessionsController
+  include InternalRedirect
   include AuthenticatesWithTwoFactor
   include Devise::Controllers::Rememberable
   include Recaptcha::ClientHelper
@@ -102,18 +103,12 @@ class SessionsController < Devise::SessionsController
     # we should never redirect to '/users/sign_in' after signing in successfully.
     return true if redirect_uri.path == new_user_session_path
 
-    redirect_to = redirect_uri.to_s if redirect_allowed_to?(redirect_uri)
+    redirect_to = redirect_uri.to_s if host_allowed?(redirect_uri)
 
     @redirect_to = redirect_to
     store_location_for(:redirect, redirect_to)
   end
 
-  # Overridden in EE
-  def redirect_allowed_to?(uri)
-    uri.host == Gitlab.config.gitlab.host &&
-      uri.port == Gitlab.config.gitlab.port
-  end
-
   def two_factor_enabled?
     find_user&.two_factor_enabled?
   end

app/controllers/users/terms_controller.rb

+module Users
+  class TermsController < ApplicationController
+    include InternalRedirect
+
+    skip_before_action :enforce_terms!
+    before_action :terms
+
+    layout 'terms'
+
+    def index
+      @redirect = redirect_path
+    end
+
+    def accept
+      agreement = Users::RespondToTermsService.new(current_user, viewed_term)
+                    .execute(accepted: true)
+
+      if agreement.persisted?
+        redirect_to redirect_path
+      else
+        flash[:alert] = agreement.errors.full_messages.join(', ')
+        redirect_to terms_path, redirect: redirect_path
+      end
+    end
+
+    def decline
+      agreement = Users::RespondToTermsService.new(current_user, viewed_term)
+                    .execute(accepted: false)
+
+      if agreement.persisted?
+        sign_out(current_user)
+        redirect_to root_path
+      else
+        flash[:alert] = agreement.errors.full_messages.join(', ')
+        redirect_to terms_path, redirect: redirect_path
+      end
+    end
+
+    private
+
+    def viewed_term
+      @viewed_term ||= ApplicationSetting::Term.find(params[:id])
+    end
+
+    def terms
+      unless @term = Gitlab::CurrentSettings.current_application_settings.latest_terms
+        redirect_to redirect_path
+      end
+    end
+
+    def redirect_path
+      redirect_to_path = safe_redirect_path(params[:redirect]) || safe_redirect_path_for_url(request.referer)
+
+      if redirect_to_path &&
+          excluded_redirect_paths.none? { |excluded| redirect_to_path.include?(excluded) }
+        redirect_to_path
+      else
+        root_path
+      end
+    end
+
+    def excluded_redirect_paths
+      [terms_path, new_user_session_path]
+    end
+  end
+end

app/helpers/application_settings_helper.rb

@@ -248,7 +248,9 @@ module ApplicationSettingsHelper
       :user_default_external,
       :user_oauth_applications,
       :version_check_enabled,
-      :allow_local_requests_from_hooks_and_services
+      :allow_local_requests_from_hooks_and_services,
+      :enforce_terms,
+      :terms
     ]
   end
 end

app/helpers/users_helper.rb

@@ -23,9 +23,42 @@ module UsersHelper
     profile_tabs.include?(tab)
   end
 
+  def current_user_menu_items
+    @current_user_menu_items ||= get_current_user_menu_items
+  end
+
+  def current_user_menu?(item)
+    current_user_menu_items.include?(item)
+  end
+
   private
 
   def get_profile_tabs
     [:activity, :groups, :contributed, :projects, :snippets]
   end
+
+  def get_current_user_menu_items
+    items = []
+
+    items << :sign_out if current_user
+
+    # TODO: Remove these conditions when the permissions are prevented in
+    # https://gitlab.com/gitlab-org/gitlab-ce/issues/45849
+    terms_not_enforced = !Gitlab::CurrentSettings
+                                .current_application_settings
+                                .enforce_terms?
+    required_terms_accepted = terms_not_enforced || current_user.terms_accepted?
+
+    items << :help if required_terms_accepted
+
+    if can?(current_user, :read_user, current_user) && required_terms_accepted
+      items << :profile
+    end
+
+    if can?(current_user, :update_user, current_user) && required_terms_accepted
+      items << :settings
+    end
+
+    items
+  end
 end

app/models/application_setting.rb

@@ -220,12 +220,15 @@ class ApplicationSetting < ActiveRecord::Base
     end
   end
 
+  validate :terms_exist, if: :enforce_terms?
+
   before_validation :ensure_uuid!
 
   before_save :ensure_runners_registration_token
   before_save :ensure_health_check_access_token
 
   after_commit do
+    reset_memoized_terms
     Rails.cache.write(CACHE_KEY, self)
   end
 
@@ -507,6 +510,16 @@ class ApplicationSetting < ActiveRecord::Base
     password_authentication_enabled_for_web? || password_authentication_enabled_for_git?
   end
 
+  delegate :terms, to: :latest_terms, allow_nil: true
+  def latest_terms
+    @latest_terms ||= Term.latest
+  end
+
+  def reset_memoized_terms
+    @latest_terms = nil
+    latest_terms
+  end
+
   private
 
   def ensure_uuid!
@@ -520,4 +533,10 @@ class ApplicationSetting < ActiveRecord::Base
     errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless
       invalid.empty?
   end
+
+  def terms_exist
+    return unless enforce_terms?
+
+    errors.add(:terms, "You need to set terms to be enforced") unless terms.present?
+  end
 end

app/models/application_setting/term.rb

+class ApplicationSetting
+  class Term < ActiveRecord::Base
+    include CacheMarkdownField
+
+    validates :terms, presence: true
+
+    cache_markdown_field :terms
+
+    def self.latest
+      order(:id).last
+    end
+  end
+end

app/models/ci/pipeline.rb

@@ -32,6 +32,8 @@ module Ci
     has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
     has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
 
+    accepts_nested_attributes_for :variables, reject_if: :persisted?
+
     delegate :id, to: :project, prefix: true
     delegate :full_path, to: :project, prefix: true
 

app/models/ci/pipeline_variable.rb

@@ -5,6 +5,8 @@ module Ci
 
     belongs_to :pipeline
 
+    alias_attribute :secret_value, :value
+
     validates :key, uniqueness: { scope: :pipeline_id }
   end
 end

app/models/ci/runner.rb

@@ -14,31 +14,49 @@ module Ci
     has_many :builds
     has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_many :projects, through: :runner_projects
+    has_many :runner_namespaces
+    has_many :groups, through: :runner_namespaces
 
     has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build'
 
     before_validation :set_default_values
 
-    scope :specific, ->() { where(is_shared: false) }
-    scope :shared, ->() { where(is_shared: true) }
-    scope :active, ->() { where(active: true) }
-    scope :paused, ->() { where(active: false) }
-    scope :online, ->() { where('contacted_at > ?', contact_time_deadline) }
-    scope :ordered, ->() { order(id: :desc) }
+    scope :specific, -> { where(is_shared: false) }
+    scope :shared, -> { where(is_shared: true) }
+    scope :active, -> { where(active: true) }
+    scope :paused, -> { where(active: false) }
+    scope :online, -> { where('contacted_at > ?', contact_time_deadline) }
+    scope :ordered, -> { order(id: :desc) }
 
-    scope :owned_or_shared, ->(project_id) do
-      joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id')
-        .where("ci_runner_projects.project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id)
+    scope :belonging_to_project, -> (project_id) {
+      joins(:runner_projects).where(ci_runner_projects: { project_id: project_id })
+    }
+
+    scope :belonging_to_parent_group_of_project, -> (project_id) {
+      project_groups = ::Group.joins(:projects).where(projects: { id: project_id })
+      hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors
+
+      joins(:groups).where(namespaces: { id: hierarchy_groups })
+    }
+
+    scope :owned_or_shared, -> (project_id) do
+      union = Gitlab::SQL::Union.new(
+        [belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared],
+        remove_duplicates: false
+      )
+      from("(#{union.to_sql}) ci_runners")
     end
 
     scope :assignable_for, ->(project) do
       # FIXME: That `to_sql` is needed to workaround a weird Rails bug.
       #        Without that, placeholders would miss one and couldn't match.
       where(locked: false)
-        .where.not("id IN (#{project.runners.select(:id).to_sql})").specific
+        .where.not("ci_runners.id IN (#{project.runners.select(:id).to_sql})")
+        .specific
     end
 
     validate :tag_constraints
+    validate :either_projects_or_group
     validates :access_level, presence: true
 
     acts_as_taggable
@@ -50,6 +68,12 @@ module Ci
       ref_protected: 1
     }
 
+    enum runner_type: {
+      instance_type: 1,
+      group_type: 2,
+      project_type: 3
+    }
+
     cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address
 
     chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout
@@ -120,6 +144,14 @@ module Ci
       !shared?
     end
 
+    def assigned_to_group?
+      runner_namespaces.any?
+    end
+
+    def assigned_to_project?
+      runner_projects.any?
+    end
+
     def can_pick?(build)
       return false if self.ref_protected? && !build.protected?
 
@@ -174,6 +206,12 @@ module Ci
       end
     end
 
+    def pick_build!(build)
+      if can_pick?(build)
+        tick_runner_queue
+      end
+    end
+
     private
 
     def cleanup_runner_queue
@@ -205,7 +243,17 @@ module Ci
     end
 
     def assignable_for?(project_id)
-      is_shared? || projects.exists?(id: project_id)
+      self.class.owned_or_shared(project_id).where(id: self.id).any?
+    end
+
+    def either_projects_or_group
+      if groups.many?
+        errors.add(:runner, 'can only be assigned to one group')
+      end
+
+      if assigned_to_group? && assigned_to_project?
+        errors.add(:runner, 'can only be assigned either to projects or to a group')
+      end
     end
 
     def accepting_tags?(build)

app/models/ci/runner_namespace.rb

+module Ci
+  class RunnerNamespace < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :runner
+    belongs_to :namespace, class_name: '::Namespace'
+    belongs_to :group, class_name: '::Group', foreign_key: :namespace_id
+  end
+end

app/models/group.rb

@@ -9,6 +9,7 @@ class Group < Namespace
   include SelectForProjectAuthorization
   include LoadedInGroupList
   include GroupDescendant
+  include TokenAuthenticatable
 
   has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   alias_method :members, :group_members
@@ -43,6 +44,8 @@ class Group < Namespace
 
   validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 }
 
+  add_authentication_token_field :runners_token
+
   after_create :post_create_hook
   after_destroy :post_destroy_hook
   after_save :update_two_factor_requirement
@@ -294,6 +297,13 @@ class Group < Namespace
     refresh_members_authorized_projects(blocking: false)
   end
 
+  # each existing group needs to have a `runners_token`.
+  # we do this on read since migrating all existing groups is not a feasible
+  # solution.
+  def runners_token
+    ensure_runners_token!
+  end
+
   private
 
   def update_two_factor_requirement

app/models/namespace.rb

@@ -21,6 +21,9 @@ class Namespace < ActiveRecord::Base
   has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :project_statistics
 
+  has_many :runner_namespaces, class_name: 'Ci::RunnerNamespace'
+  has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner'
+
   # This should _not_ be `inverse_of: :namespace`, because that would also set
   # `user.namespace` when this user creates a group with themselves as `owner`.
   belongs_to :owner, class_name: "User"

app/models/project.rb

@@ -68,6 +68,9 @@ class Project < ActiveRecord::Base
   before_save :ensure_runners_token
 
   after_save :update_project_statistics, if: :namespace_id_changed?
+
+  after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? }
+
   after_create :create_project_feature, unless: :project_feature
 
   after_create :create_ci_cd_settings,
@@ -161,6 +164,8 @@ class Project < ActiveRecord::Base
   has_one :fork_network_member
   has_one :fork_network, through: :fork_network_member
 
+  has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
+
   # Merge Requests for target project should be removed with it
   has_many :merge_requests, foreign_key: 'target_project_id'
   has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest'
@@ -235,13 +240,11 @@ class Project < ActiveRecord::Base
   has_many :project_deploy_tokens
   has_many :deploy_tokens, through: :project_deploy_tokens
 
-  has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
-
   has_one :auto_devops, class_name: 'ProjectAutoDevops'
   has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
 
   has_many :project_badges, class_name: 'ProjectBadge'
-  has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting'
+  has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
@@ -252,6 +255,7 @@ class Project < ActiveRecord::Base
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
   delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team
+  delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings
 
   # Validations
   validates :creator, presence: true, on: :create
@@ -337,6 +341,11 @@ class Project < ActiveRecord::Base
   scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
   scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
 
+  scope :with_group_runners_enabled, -> do
+    joins(:ci_cd_settings)
+    .where(project_ci_cd_settings: { group_runners_enabled: true })
+  end
+
   enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 }
 
   chronic_duration_attr :build_timeout_human_readable, :build_timeout, default: 3600
@@ -386,55 +395,9 @@ class Project < ActiveRecord::Base
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 
   scope :excluding_project, ->(project) { where.not(id: project) }
-  scope :import_started, -> { where(import_status: 'started') }
 
-  state_machine :import_status, initial: :none do
-    event :import_schedule do
-      transition [:none, :finished, :failed] => :scheduled
-    end
-
-    event :force_import_start do
-      transition [:none, :finished, :failed] => :started
-    end
-
-    event :import_start do
-      transition scheduled: :started
-    end
-
-    event :import_finish do
-      transition started: :finished
-    end
-
-    event :import_fail do
-      transition [:scheduled, :started] => :failed
-    end
-
-    event :import_retry do
-      transition failed: :started
-    end
-
-    state :scheduled
-    state :started
-    state :finished
-    state :failed
-
-    after_transition [:none, :finished, :failed] => :scheduled do |project, _|
-      project.run_after_commit do
-        job_id = add_import_job
-        update(import_jid: job_id) if job_id
-      end
-    end
-
-    after_transition started: :finished do |project, _|
-      project.reset_cache_and_import_attrs
-
-      if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists?
-        project.run_after_commit do
-          Projects::AfterImportService.new(project).execute
-        end
-      end
-    end
-  end
+  scope :joins_import_state, -> { joins("LEFT JOIN project_mirror_data import_state ON import_state.project_id = projects.id") }
+  scope :import_started, -> { joins_import_state.where("import_state.status = 'started' OR projects.import_status = 'started'") }
 
   class << self
     # Searches for a list of projects based on the query given in `query`.
@@ -664,10 +627,6 @@ class Project < ActiveRecord::Base
     external_import? || forked? || gitlab_project_import? || bare_repository_import?
   end
 
-  def no_import?
-    import_status == 'none'
-  end
-
   def external_import?
     import_url.present?
   end
@@ -680,6 +639,93 @@ class Project < ActiveRecord::Base
     import_started? || import_scheduled?
   end
 
+  def import_state_args
+    {
+      status: self[:import_status],
+      jid: self[:import_jid],
+      last_error: self[:import_error]
+    }
+  end
+
+  def ensure_import_state
+    return if self[:import_status] == 'none' || self[:import_status].nil?
+    return unless import_state.nil?
+
+    create_import_state(import_state_args)
+
+    update_column(:import_status, 'none')
+  end
+
+  def import_schedule
+    ensure_import_state
+
+    import_state&.schedule
+  end
+
+  def force_import_start
+    ensure_import_state
+
+    import_state&.force_start
+  end
+
+  def import_start
+    ensure_import_state
+
+    import_state&.start
+  end
+
+  def import_fail
+    ensure_import_state
+
+    import_state&.fail_op
+  end
+
+  def import_finish
+    ensure_import_state
+
+    import_state&.finish
+  end
+
+  def import_jid=(new_jid)
+    ensure_import_state
+
+    import_state&.jid = new_jid
+  end
+
+  def import_jid
+    ensure_import_state
+
+    import_state&.jid
+  end
+
+  def import_error=(new_error)
+    ensure_import_state
+
+    import_state&.last_error = new_error
+  end
+
+  def import_error
+    ensure_import_state
+
+    import_state&.last_error
+  end
+
+  def import_status=(new_status)
+    ensure_import_state
+
+    import_state&.status = new_status
+  end
+
+  def import_status
+    ensure_import_state
+
+    import_state&.status || 'none'
+  end
+
+  def no_import?
+    import_status == 'none'
+  end
+
   def import_started?
     # import? does SQL work so only run it if it looks like there's an import running
     import_status == 'started' && import?
@@ -1306,12 +1352,17 @@ class Project < ActiveRecord::Base
     @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none
   end
 
-  def active_shared_runners
-    @active_shared_runners ||= shared_runners.active
+  def group_runners
+    @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none
+  end
+
+  def all_runners
+    union = Gitlab::SQL::Union.new([runners, group_runners, shared_runners])
+    Ci::Runner.from("(#{union.to_sql}) ci_runners")
   end
 
   def any_runners?(&block)
-    active_runners.any?(&block) || active_shared_runners.any?(&block)
+    all_runners.active.any?(&block)
   end
 
   def valid_runners_token?(token)
@@ -1476,7 +1527,7 @@ class Project < ActiveRecord::Base
   def rename_repo_notify!
     # When we import a project overwriting the original project, there
     # is a move operation. In that case we don't want to send the instructions.
-    send_move_instructions(full_path_was) unless started?
+    send_move_instructions(full_path_was) unless import_started?
     expires_full_path_cache
 
     self.old_path_with_namespace = full_path_was
@@ -1530,7 +1581,8 @@ class Project < ActiveRecord::Base
     return unless import_jid
 
     Gitlab::SidekiqStatus.unset(import_jid)
-    update_column(:import_jid, nil)
+
+    import_state.update_column(:jid, nil)
   end
 
   def running_or_pending_build_count(force: false)
@@ -1549,7 +1601,8 @@ class Project < ActiveRecord::Base
     sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message)
 
     import_fail
-    update_column(:import_error, sanitized_message)
+
+    import_state.update_column(:last_error, sanitized_message)
   rescue ActiveRecord::ActiveRecordError => e
     Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}")
   ensure
@@ -1879,6 +1932,10 @@ class Project < ActiveRecord::Base
     []
   end
 
+  def toggle_ci_cd_settings!(settings_attribute)
+    ci_cd_settings.toggle!(settings_attribute)
+  end
+
   def gitlab_deploy_token
     @gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token
   end

app/models/project_ci_cd_setting.rb

 class ProjectCiCdSetting < ActiveRecord::Base
-  belongs_to :project
+  belongs_to :project, inverse_of: :ci_cd_settings
 
   # The version of the schema that first introduced this model/table.
   MINIMUM_SCHEMA_VERSION = 20180403035759

app/models/project_import_state.rb

+class ProjectImportState < ActiveRecord::Base
+  include AfterCommitQueue
+
+  self.table_name = "project_mirror_data"
+
+  belongs_to :project, inverse_of: :import_state
+
+  validates :project, presence: true
+
+  state_machine :status, initial: :none do
+    event :schedule do
+      transition [:none, :finished, :failed] => :scheduled
+    end
+
+    event :force_start do
+      transition [:none, :finished, :failed] => :started
+    end
+
+    event :start do
+      transition scheduled: :started
+    end
+
+    event :finish do
+      transition started: :finished
+    end
+
+    event :fail_op do
+      transition [:scheduled, :started] => :failed
+    end
+
+    state :scheduled
+    state :started
+    state :finished
+    state :failed
+
+    after_transition [:none, :finished, :failed] => :scheduled do |state, _|
+      state.run_after_commit do
+        job_id = project.add_import_job
+        update(jid: job_id) if job_id
+      end
+    end
+
+    after_transition started: :finished do |state, _|
+      project = state.project
+
+      project.reset_cache_and_import_attrs
+
+      if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists?
+        state.run_after_commit do
+          Projects::AfterImportService.new(project).execute
+        end
+      end
+    end
+  end
+end

app/models/term_agreement.rb

+class TermAgreement < ActiveRecord::Base
+  belongs_to :term, class_name: 'ApplicationSetting::Term'
+  belongs_to :user
+
+  validates :user, :term, presence: true
+end

app/models/user.rb

@@ -138,6 +138,8 @@ class User < ActiveRecord::Base
   has_many :custom_attributes, class_name: 'UserCustomAttribute'
   has_many :callouts, class_name: 'UserCallout'
   has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_many :term_agreements
+  belongs_to :accepted_term, class_name: 'ApplicationSetting::Term'
 
   #
   # Validations
@@ -1187,6 +1189,10 @@ class User < ActiveRecord::Base
     max_member_access_for_group_ids([group_id])[group_id]
   end
 
+  def terms_accepted?
+    accepted_term_id.present?
+  end
+
   protected
 
   # override, from Devise::Validatable

app/policies/application_setting/term_policy.rb

+class ApplicationSetting
+  class TermPolicy < BasePolicy
+    include Gitlab::Utils::StrongMemoize
+
+    condition(:current_terms, scope: :subject) do
+      Gitlab::CurrentSettings.current_application_settings.latest_terms == @subject
+    end
+
+    condition(:terms_accepted, score: 1) do
+      agreement&.accepted
+    end
+
+    rule { ~anonymous & current_terms }.policy do
+      enable :accept_terms
+      enable :decline_terms
+    end
+
+    rule { terms_accepted }.prevent :accept_terms
+
+    def agreement
+      strong_memoize(:agreement) do
+        next nil if @user.nil? || @subject.nil?
+
+        @user.term_agreements.find_by(term: @subject)
+      end
+    end
+  end
+end

app/policies/user_policy.rb

@@ -8,6 +8,8 @@ class UserPolicy < BasePolicy
   rule { ~restricted_public_level }.enable :read_user
   rule { ~anonymous }.enable :read_user
 
-  rule { user_is_self | admin }.enable :destroy_user
-  rule { subject_ghost }.prevent :destroy_user
+  rule { ~subject_ghost & (user_is_self | admin) }.policy do
+    enable :destroy_user
+    enable :update_user
+  end
 end

app/services/application_settings/update_service.rb

 module ApplicationSettings
   class UpdateService < ApplicationSettings::BaseService
     def execute
+      update_terms(@params.delete(:terms))
+
       @application_setting.update(@params)
     end
+
+    private
+
+    def update_terms(terms)
+      return unless terms.present?
+
+      # Avoid creating a new terms record if the text is exactly the same.
+      terms = terms.strip
+      return if terms == @application_setting.terms
+
+      ApplicationSetting::Term.create(terms: terms)
+      @application_setting.reset_memoized_terms
+    end
   end
 end

app/services/ci/create_pipeline_service.rb

@@ -24,6 +24,7 @@ module Ci
         ignore_skip_ci: ignore_skip_ci,
         save_incompleted: save_on_errors,
         seeds_block: block,
+        variables_attributes: params[:variables_attributes],
         project: project,
         current_user: current_user)
 

app/services/ci/register_job_service.rb

@@ -17,8 +17,10 @@ module Ci
       builds =
         if runner.shared?
           builds_for_shared_runner
+        elsif runner.group_type?
+          builds_for_group_runner
         else
-          builds_for_specific_runner
+          builds_for_project_runner
         end
 
       valid = true
@@ -82,8 +84,17 @@ module Ci
         .order('COALESCE(project_builds.running_builds, 0) ASC', 'ci_builds.id ASC')
     end
 
-    def builds_for_specific_runner
-      new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('created_at ASC')
+    def builds_for_project_runner
+      new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('id ASC')
+    end
+
+    def builds_for_group_runner
+      hierarchy_groups = Gitlab::GroupHierarchy.new(runner.groups).base_and_descendants
+      projects = Project.where(namespace_id: hierarchy_groups)
+        .with_group_runners_enabled
+        .with_builds_enabled
+        .without_deleted
+      new_builds.where(project: projects).order('id ASC')
     end
 
     def running_builds_for_shared_runners
@@ -97,10 +108,6 @@ module Ci
       builds
     end
 
-    def shared_runner_build_limits_feature_enabled?
-      ENV['DISABLE_SHARED_RUNNER_BUILD_MINUTES_LIMIT'].to_s != 'true'
-    end
-
     def register_failure
       failed_attempt_counter.increment
       attempt_counter.increment

app/services/ci/update_build_queue_service.rb

 module Ci
   class UpdateBuildQueueService
     def execute(build)
-      build.project.runners.each do |runner|
-        if runner.can_pick?(build)
-          runner.tick_runner_queue
-        end
+      tick_for(build, build.project.all_runners)
     end
 
-      return unless build.project.shared_runners_enabled?
+    private
 
-      Ci::Runner.shared.each do |runner|
-        if runner.can_pick?(build)
-          runner.tick_runner_queue
-        end
+    def tick_for(build, runners)
+      runners.each do |runner|
+        runner.pick_build!(build)
       end
     end
   end

app/services/projects/create_service.rb

@@ -142,7 +142,7 @@ module Projects
 
       if @project
         @project.errors.add(:base, message)
-        @project.mark_import_as_failed(message) if @project.import?
+        @project.mark_import_as_failed(message) if @project.persisted? && @project.import?
       end
 
       @project

app/services/users/respond_to_terms_service.rb

+module Users
+  class RespondToTermsService
+    def initialize(user, term)
+      @user, @term = user, term
+    end
+
+    def execute(accepted:)
+      agreement = @user.term_agreements.find_or_initialize_by(term: @term)
+      agreement.accepted = accepted
+
+      if agreement.save
+        store_accepted_term(accepted)
+      end
+
+      agreement
+    end
+
+    private
+
+    def store_accepted_term(accepted)
+      @user.update_column(:accepted_term_id, accepted ? @term.id : nil)
+    end
+  end
+end

app/services/web_hook_service.rb

@@ -41,7 +41,7 @@ class WebHookService
       http_status: response.code,
       message: response.to_s
     }
-  rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout => e
+  rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError => e
     log_execution(
       trigger: hook_name,
       url: hook.url,

app/views/admin/application_settings/_terms.html.haml

+= form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
+  = form_errors(@application_setting)
+
+  %fieldset
+    .form-group
+      .col-sm-12
+        .checkbox
+          = f.label :enforce_terms do
+            = f.check_box :enforce_terms
+            = _("Require all users to accept Terms of Service when they access GitLab.")
+          .help-block
+            = _("When enabled, users cannot use GitLab until the terms have been accepted.")
+    .form-group
+      .col-sm-12
+        = f.label :terms do
+          = _("Terms of Service Agreement")
+      .col-sm-12
+        = f.text_area :terms, class: 'form-control', rows: 8
+        .help-block
+          = _("Markdown enabled")
+
+  = f.submit _("Save changes"), class: "btn btn-success"

app/views/admin/runners/_runner.html.haml

@@ -2,6 +2,8 @@
   %td
     - if runner.shared?
       %span.label.label-success shared
+    - elsif runner.group_type?
+      %span.label.label-success group
     - else
       %span.label.label-info specific
     - if runner.locked?
@@ -19,7 +21,7 @@
   %td
     = runner.ip_address
   %td
-    - if runner.shared?
+    - if runner.shared? || runner.group_type?
       n/a
     - else
       = runner.projects.count(:all)
@@ -31,7 +33,7 @@
         = tag
   %td
     - if runner.contacted_at
-      = time_ago_with_tooltip runner.contacted_at
+      #{time_ago_in_words(runner.contacted_at)} ago
     - else
       Never
   %td.admin-runner-btn-group-cell

app/views/admin/runners/index.html.haml

@@ -16,6 +16,9 @@
         %li
           %span.label.label-success shared
           \- Runner runs jobs from all unassigned projects
+        %li
+          %span.label.label-success group
+          \- Runner runs jobs from all unassigned projects in its group
         %li
           %span.label.label-info specific
           \- Runner runs jobs from assigned projects

app/views/admin/runners/show.html.haml

@@ -19,6 +19,9 @@
     %p
       If you want Runners to build only specific projects, enable them in the table below.
       Keep in mind that this is a one way transition.
+- elsif @runner.group_type?
+  .bs-callout.bs-callout-success
+    %h4 This runner will process jobs from all projects in its group and subgroups
 - else
   .bs-callout.bs-callout-info
     %h4 This Runner will process jobs only from ASSIGNED projects

app/views/admin/users/index.html.haml

@@ -42,31 +42,31 @@
         = nav_link(html_options: { class: active_when(params[:filter].nil?) }) do
           = link_to admin_users_path do
             Active
-            %small.badge= number_with_delimiter(User.active.count)
+            %small.badge= limited_counter_with_delimiter(User.active)
         = nav_link(html_options: { class: active_when(params[:filter] == 'admins') }) do
           = link_to admin_users_path(filter: "admins") do
             Admins
-            %small.badge= number_with_delimiter(User.admins.count)
+            %small.badge= limited_counter_with_delimiter(User.admins)
         = nav_link(html_options: { class: "#{active_when(params[:filter] == 'two_factor_enabled')} filter-two-factor-enabled" }) do
           = link_to admin_users_path(filter: 'two_factor_enabled') do
             2FA Enabled
-            %small.badge= number_with_delimiter(User.with_two_factor.count)
+            %small.badge= limited_counter_with_delimiter(User.with_two_factor)
         = nav_link(html_options: { class: "#{active_when(params[:filter] == 'two_factor_disabled')} filter-two-factor-disabled" }) do
           = link_to admin_users_path(filter: 'two_factor_disabled') do
             2FA Disabled
-            %small.badge= number_with_delimiter(User.without_two_factor.count)
+            %small.badge= limited_counter_with_delimiter(User.without_two_factor)
         = nav_link(html_options: { class: active_when(params[:filter] == 'external') }) do
           = link_to admin_users_path(filter: 'external') do
             External
-            %small.badge= number_with_delimiter(User.external.count)
+            %small.badge= limited_counter_with_delimiter(User.external)
         = nav_link(html_options: { class: active_when(params[:filter] == 'blocked') }) do
           = link_to admin_users_path(filter: "blocked") do
             Blocked
-            %small.badge= number_with_delimiter(User.blocked.count)
+            %small.badge= limited_counter_with_delimiter(User.blocked)
         = nav_link(html_options: { class: active_when(params[:filter] == 'wop') }) do
           = link_to admin_users_path(filter: "wop") do
             Without projects
-            %small.badge= number_with_delimiter(User.without_projects.count)
+            %small.badge= limited_counter_with_delimiter(User.without_projects)
 
   %ul.flex-list.content-list
     - if @users.empty?

app/views/layouts/_flash.html.haml

+- extra_flash_class = local_assigns.fetch(:extra_flash_class, nil)
+
 .flash-container.flash-container-page
   -# We currently only support `alert`, `notice`, `success`
   - flash.each do |key, value|
     -# Don't show a flash message if the message is nil
     - if value
       %div{ class: "flash-#{key}" }
-        %div{ class: (container_class) }
+        %div{ class: "#{container_class} #{extra_flash_class}" }
           %span= value

app/views/layouts/header/_current_user_dropdown.html.haml

+- return unless current_user
+
+%ul
+  %li.current-user
+    .user-name.bold
+      = current_user.name
+    = current_user.to_reference
+  %li.divider
+  - if current_user_menu?(:profile)
+    %li
+      = link_to s_("CurrentUser|Profile"), current_user, class: 'profile-link', data: { user: current_user.username }
+  - if current_user_menu?(:settings)
+    %li
+      = link_to s_("CurrentUser|Settings"), profile_path
+  - if current_user_menu?(:help)
+    %li
+      = link_to _("Help"), help_path
+  - if current_user_menu?(:help) || current_user_menu?(:settings) || current_user_menu?(:profile)
+    %li.divider
+  - if current_user_menu?(:sign_out)
+    %li
+      = link_to _("Sign out"), destroy_user_session_path, class: "sign-out-link"

app/views/layouts/header/_default.html.haml

@@ -53,22 +53,7 @@
                 = image_tag avatar_icon_for_user(current_user, 23), width: 23, height: 23, class: "header-user-avatar qa-user-avatar"
                 = sprite_icon('angle-down', css_class: 'caret-down')
               .dropdown-menu-nav.dropdown-menu-align-right
-                %ul
-                  %li.current-user
-                    .user-name.bold
-                      = current_user.name
-                    @#{current_user.username}
-                  %li.divider
-                  %li
-                    = link_to "Profile", current_user, class: 'profile-link', data: { user: current_user.username }
-                  %li
-                    = link_to "Settings", profile_path
-                  - if current_user
-                    %li
-                      = link_to "Help", help_path
-                  %li.divider
-                  %li
-                    = link_to "Sign out", destroy_user_session_path, class: "sign-out-link"
+                = render 'layouts/header/current_user_dropdown'
           - if header_link?(:admin_impersonation)
             %li.impersonation
               = link_to admin_impersonation_path, class: 'impersonation-btn', method: :delete, title: "Stop impersonation", aria: { label: 'Stop impersonation' }, data: { toggle: 'tooltip', placement: 'bottom', container: 'body' } do

app/views/layouts/terms.html.haml

+!!! 5
+- @hide_breadcrumbs = true
+%html{ lang: I18n.locale, class: page_class }
+  = render "layouts/head"
+
+  %body{ data: { page: body_data_page } }
+    .layout-page.terms{ class: page_class }
+      .content-wrapper.prepend-top-0
+        .mobile-overlay
+        .alert-wrapper
+          = render "layouts/broadcast"
+          = render 'layouts/header/read_only_banner'
+          = render "layouts/flash", extra_flash_class: 'limit-container-width'
+
+        %div{ class: "#{container_class} limit-container-width" }
+          .content{ id: "content-body" }
+            .panel.panel-default
+              .panel-heading
+                .title
+                  = brand_header_logo
+                  - logo_text = brand_header_logo_type
+                  - if logo_text.present?
+                    %span.logo-text.hidden-xs.prepend-left-8
+                      = logo_text
+                - if header_link?(:user_dropdown)
+                  .navbar-collapse.collapse
+                    %ul.nav.navbar-nav
+                      %li.header-user.dropdown
+                        = link_to current_user, class: user_dropdown_class, data: { toggle: "dropdown" } do
+                          = image_tag avatar_icon_for_user(current_user, 23), width: 23, height: 23, class: "header-user-avatar qa-user-avatar"
+                          = sprite_icon('angle-down', css_class: 'caret-down')
+                        .dropdown-menu-nav.dropdown-menu-align-right
+                          = render 'layouts/header/current_user_dropdown'
+              = yield

app/views/projects/_import_project_pane.html.haml

+- active_tab = local_assigns.fetch(:active_tab, 'blank')
+- f = local_assigns.fetch(:f)
+
+.project-import.row
+  .col-lg-12
+    .form-group.import-btn-container.clearfix
+      = f.label :visibility_level, class: 'label-light' do #the label here seems wrong
+        Import project from
+      .import-buttons
+        - if gitlab_project_import_enabled?
+          .import_gitlab_project.has-tooltip{ data: { container: 'body' } }
+            = link_to new_import_gitlab_project_path, class: 'btn btn_import_gitlab_project project-submit' do
+              = icon('gitlab', text: 'GitLab export')
+        %div
+          - if github_import_enabled?
+            = link_to new_import_github_path, class: 'btn js-import-github' do
+              = icon('github', text: 'GitHub')
+        %div
+          - if bitbucket_import_enabled?
+            = link_to status_import_bitbucket_path, class: "btn import_bitbucket #{'how_to_import_link' unless bitbucket_import_configured?}" do
+              = icon('bitbucket', text: 'Bitbucket')
+            - unless bitbucket_import_configured?
+              = render 'bitbucket_import_modal'
+        %div
+          - if gitlab_import_enabled?
+            = link_to status_import_gitlab_path, class: "btn import_gitlab #{'how_to_import_link' unless gitlab_import_configured?}" do
+              = icon('gitlab', text: 'GitLab.com')
+            - unless gitlab_import_configured?
+              = render 'gitlab_import_modal'
+        %div
+          - if google_code_import_enabled?
+            = link_to new_import_google_code_path, class: 'btn import_google_code' do
+              = icon('google', text: 'Google Code')
+        %div
+          - if fogbugz_import_enabled?
+            = link_to new_import_fogbugz_path, class: 'btn import_fogbugz' do
+              = icon('bug', text: 'Fogbugz')
+        %div
+          - if gitea_import_enabled?
+            = link_to new_import_gitea_path, class: 'btn import_gitea' do
+              = custom_icon('go_logo')
+              Gitea
+        %div
+          - if git_import_enabled?
+            %button.btn.js-toggle-button.js-import-git-toggle-button{ type: "button", data: { toggle_open_class: 'active' } }
+              = icon('git', text: 'Repo by URL')
+  .col-lg-12
+    .js-toggle-content.toggle-import-form{ class: ('hide' if active_tab != 'import') }
+      %hr
+        = render "shared/import_form", f: f
+        = render 'new_project_fields', f: f, project_name_id: "import-url-name"

app/views/projects/new.html.haml

@@ -57,54 +57,11 @@
         .tab-pane.import-project-pane.js-toggle-container{ id: 'import-project-pane', class: active_when(active_tab == 'import'), role: 'tabpanel' }
           = form_for @project, html: { class: 'new_project' } do |f|
             - if import_sources_enabled?
-              .project-import.row
-                .col-lg-12
-                  .form-group.import-btn-container.clearfix
-                    = f.label :visibility_level, class: 'label-light' do #the label here seems wrong
-                      Import project from
-                    .import-buttons
-                      - if gitlab_project_import_enabled?
-                        .import_gitlab_project.has-tooltip{ data: { container: 'body' } }
-                          = link_to new_import_gitlab_project_path, class: 'btn btn_import_gitlab_project project-submit' do
-                            = icon('gitlab', text: 'GitLab export')
-                      %div
-                        - if github_import_enabled?
-                          = link_to new_import_github_path, class: 'btn js-import-github' do
-                            = icon('github', text: 'GitHub')
-                      %div
-                        - if bitbucket_import_enabled?
-                          = link_to status_import_bitbucket_path, class: "btn import_bitbucket #{'how_to_import_link' unless bitbucket_import_configured?}" do
-                            = icon('bitbucket', text: 'Bitbucket')
-                          - unless bitbucket_import_configured?
-                            = render 'bitbucket_import_modal'
-                      %div
-                        - if gitlab_import_enabled?
-                          = link_to status_import_gitlab_path, class: "btn import_gitlab #{'how_to_import_link' unless gitlab_import_configured?}" do
-                            = icon('gitlab', text: 'GitLab.com')
-                          - unless gitlab_import_configured?
-                            = render 'gitlab_import_modal'
-                      %div
-                        - if google_code_import_enabled?
-                          = link_to new_import_google_code_path, class: 'btn import_google_code' do
-                            = icon('google', text: 'Google Code')
-                      %div
-                        - if fogbugz_import_enabled?
-                          = link_to new_import_fogbugz_path, class: 'btn import_fogbugz' do
-                            = icon('bug', text: 'Fogbugz')
-                      %div
-                        - if gitea_import_enabled?
-                          = link_to new_import_gitea_path, class: 'btn import_gitea' do
-                            = custom_icon('go_logo')
-                            Gitea
-                      %div
-                        - if git_import_enabled?
-                          %button.btn.js-toggle-button.js-import-git-toggle-button{ type: "button", data: { toggle_open_class: 'active' } }
-                            = icon('git', text: 'Repo by URL')
-                .col-lg-12
-                  .js-toggle-content.toggle-import-form{ class: ('hide' if active_tab != 'import') }
-                    %hr
-                      = render "shared/import_form", f: f
-                      = render 'new_project_fields', f: f, project_name_id: "import-url-name"
+              = render 'import_project_pane', f: f, active_tab: active_tab
+            - else
+              .nothing-here-block
+                %h4 No import options available
+                %p Contact an administrator to enable options for importing your project.
 
 .save-project-loader.hide
   .center

app/views/projects/pipelines/new.html.haml

 - breadcrumb_title "Pipelines"
 - page_title = s_("Pipeline|Run Pipeline")
+- settings_link = link_to _('CI/CD settings'), project_settings_ci_cd_path(@project)
 
 %h3.page-title
   = s_("Pipeline|Run Pipeline")
@@ -8,17 +9,26 @@
 = form_for @pipeline, as: :pipeline, url: project_pipelines_path(@project), html: { id: "new-pipeline-form", class: "form-horizontal js-new-pipeline-form js-requires-input" } do |f|
   = form_errors(@pipeline)
   .form-group
-    = f.label :ref, s_('Pipeline|Run on'), class: 'control-label'
-    .col-sm-10
+    .col-sm-12
+      = f.label :ref, s_('Pipeline|Create for')
       = hidden_field_tag 'pipeline[ref]', params[:ref] || @project.default_branch
       = dropdown_tag(params[:ref] || @project.default_branch,
                      options: { toggle_class: 'js-branch-select wide git-revision-dropdown-toggle',
                                 filter: true, dropdown_class: "dropdown-menu-selectable git-revision-dropdown", placeholder: s_("Pipeline|Search branches"),
                                 data: { selected: params[:ref] || @project.default_branch, field_name: 'pipeline[ref]' } })
       .help-block
-        = s_("Pipeline|Existing branch name, tag")
+        = s_("Pipeline|Existing branch name or tag")
+
+    .col-sm-12.prepend-top-10.js-ci-variable-list-section
+      %label
+        = s_('Pipeline|Variables')
+      %ul.ci-variable-list
+        = render 'ci/variables/variable_row', form_field: 'pipeline', only_key_value: true
+      .help-block
+        = (s_("Pipeline|Specify variable values to be used in this run. The values specified in %{settings_link} will be used by default.") % {settings_link: settings_link}).html_safe
+
   .form-actions
-    = f.submit s_('Pipeline|Run pipeline'), class: 'btn btn-success', tabindex: 3
+    = f.submit s_('Pipeline|Create pipeline'), class: 'btn btn-success js-variables-save-button', tabindex: 3
     = link_to 'Cancel', project_pipelines_path(@project), class: 'btn btn-default pull-right'
 
 -# haml-lint:disable InlineJavaScript

app/views/projects/runners/_group_runners.html.haml

+%h3 Group Runners
+
+.bs-callout.bs-callout-warning
+  GitLab Group Runners can execute code for all the projects in this group.
+  They can be managed using the #{link_to 'Runners API', help_page_path('api/runners.md')}.
+
+  - if @project.group
+    %hr
+    - if @project.group_runners_enabled?
+      = link_to toggle_group_runners_project_runners_path(@project), class: 'btn btn-warning', method: :post do
+        Disable group Runners
+    - else
+      = link_to toggle_group_runners_project_runners_path(@project), class: 'btn btn-success', method: :post do
+        Enable group Runners
+      for this project
+
+- if !@project.group
+  This project does not belong to a group and can therefore not make use of group Runners.
+
+- elsif @group_runners.empty?
+  This group does not provide any group Runners yet.
+
+  - if can?(current_user, :admin_pipeline, @project.group)
+    = render partial: 'ci/runner/how_to_setup_runner',
+             locals: { registration_token: @project.group.runners_token, type: 'group' }
+  - else
+    Ask your group master to setup a group Runner.
+
+- else
+  %h4.underlined-title Available group Runners : #{@group_runners.count}
+  %ul.bordered-list
+    = render partial: 'projects/runners/runner', collection: @group_runners, as: :runner

app/views/projects/runners/_index.html.haml

@@ -23,3 +23,7 @@
     = render 'projects/runners/specific_runners'
   .col-sm-6
     = render 'projects/runners/shared_runners'
+.row
+  .col-sm-6
+  .col-sm-6
+    = render 'projects/runners/group_runners'

app/views/projects/runners/_runner.html.haml

@@ -26,7 +26,7 @@
         - else
           - runner_project = @project.runner_projects.find_by(runner_id: runner)
           = link_to 'Disable for this project', project_runner_project_path(@project, runner_project), data: { confirm: "Are you sure?" }, method: :delete, class: 'btn btn-danger btn-sm'
-      - elsif runner.specific?
+      - elsif !(runner.is_shared? || runner.group_type?) # We can simplify this to `runner.project_type?` when migrating #runner_type is complete
         = form_for [@project.namespace.becomes(Namespace), @project, @project.runner_projects.new] do |f|
           = f.hidden_field :runner_id, value: runner.id
           = f.submit 'Enable for this project', class: 'btn btn-sm'

app/views/projects/runners/show.html.haml

@@ -62,6 +62,6 @@
       %td Last contact
       %td
         - if @runner.contacted_at
-          = time_ago_with_tooltip @runner.contacted_at
+          #{time_ago_in_words(@runner.contacted_at)} ago
         - else
           Never

app/views/users/terms/index.html.haml

+- redirect_params = { redirect: @redirect } if @redirect
+
+.panel-content.rendered-terms
+  = markdown_field(@term, :terms)
+.row-content-block.footer-block.clearfix
+  - if can?(current_user, :accept_terms, @term)
+    .pull-right
+      = button_to accept_term_path(@term, redirect_params), class: 'btn btn-success prepend-left-8' do
+        = _('Accept terms')
+  - if can?(current_user, :decline_terms, @term)
+    .pull-right
+      = button_to decline_term_path(@term, redirect_params), class: 'btn btn-default prepend-left-8' do
+        =  _('Decline and sign out')

app/workers/all_queues.yml

@@ -52,6 +52,7 @@
 - pipeline_creation:create_pipeline
 - pipeline_creation:run_pipeline_schedule
 - pipeline_background:archive_trace
+- pipeline_background:ci_build_trace_chunk_flush
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
 - pipeline_default:pipeline_metrics
@@ -66,7 +67,6 @@
 - pipeline_processing:pipeline_update
 - pipeline_processing:stage_update
 - pipeline_processing:update_head_pipeline_for_merge_request
-- pipeline_processing:ci_build_trace_chunk_flush
 
 - repository_check:repository_check_clear
 - repository_check:repository_check_single_repository

app/workers/ci/build_trace_chunk_flush_worker.rb

 module Ci
   class BuildTraceChunkFlushWorker
     include ApplicationWorker
-
-    queue_namespace :pipeline_processing
+    include PipelineBackgroundQueue
 
     def perform(build_trace_chunk_id)
       ::Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk|

app/workers/gitlab/github_import/advance_stage_worker.rb

@@ -63,11 +63,10 @@ module Gitlab
       end
 
       def find_project(id)
-        # We only care about the import JID so we can refresh it. We also only
-        # want the project if it hasn't been marked as failed yet. It's possible
-        # the import gets marked as stuck when jobs of the current stage failed
-        # somehow.
-        Project.select(:import_jid).import_started.find_by(id: id)
+        # TODO: Only select the JID
+        # This is due to the fact that the JID could be present in either the project record or
+        # its associated import_state record
+        Project.import_started.find_by(id: id)
       end
     end
   end

app/workers/gitlab/github_import/refresh_import_jid_worker.rb

@@ -31,7 +31,10 @@ module Gitlab
       end
 
       def find_project(id)
-        Project.select(:import_jid).import_started.find_by(id: id)
+        # TODO: Only select the JID
+        # This is due to the fact that the JID could be present in either the project record or
+        # its associated import_state record
+        Project.import_started.find_by(id: id)
       end
     end
   end

app/workers/stuck_import_jobs_worker.rb

@@ -22,7 +22,8 @@ class StuckImportJobsWorker
   end
 
   def mark_projects_with_jid_as_failed!
-    jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h
+    # TODO: Rollback this change to use SQL through #pluck
+    jids_and_ids = enqueued_projects_with_jid.map { |project| [project.import_jid, project.id] }.to_h
 
     # Find the jobs that aren't currently running or that exceeded the threshold.
     completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
@@ -42,15 +43,15 @@ class StuckImportJobsWorker
   end
 
   def enqueued_projects
-    Project.with_import_status(:scheduled, :started)
+    Project.joins_import_state.where("(import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started')")
   end
 
   def enqueued_projects_with_jid
-    enqueued_projects.where.not(import_jid: nil)
+    enqueued_projects.where.not("import_state.jid IS NULL AND projects.import_jid IS NULL")
   end
 
   def enqueued_projects_without_jid
-    enqueued_projects.where(import_jid: nil)
+    enqueued_projects.where("import_state.jid IS NULL AND projects.import_jid IS NULL")
   end
 
   def error_message

changelogs/unreleased/36762-reconcile-project-templates-with-auto-devops.yml

+---
+title: Reconcile project templates with Auto DevOps
+merge_request: 18737
+author: 
+type: changed

changelogs/unreleased/44059-specify-variables-when-executing-a-manual-pipeline-from-the-ui.yml

+---
+title: Enable specifying variables when executing a manual pipeline
+merge_request: 18440
+author:
+type: changed

changelogs/unreleased/46049-import-export-import-is-broken-due-to-the-addition-of-a-ci-table.yml

+---
+title: Resolve Import/Export ci_cd_settings error updating the project
+merge_request: 46049
+author:
+type: fixed

changelogs/unreleased/bvl-enforce-terms.yml

+---
+title: Allow admins to enforce accepting Terms of Service on an instance
+merge_request: 18570
+author:
+type: added

changelogs/unreleased/dm-webhook-catch-blocked-url-exception.yml

+---
+title: Ensure web hook 'blocked URL' errors are stored in web hook logs and properly
+  surfaced to the user
+merge_request:
+author:
+type: fixed

changelogs/unreleased/feature-runner-per-group.yml

+---
+title: Allow group masters to configure runners for groups
+merge_request: 9646
+author: Alexis Reigel
+type: added

changelogs/unreleased/inform-the-user-when-there-are-no-project-import-options-available.yml

+---
+title: Inform the user when there are no project import options available
+merge_request: 18716
+author: George Tsiolis
+type: changed

config/initializers/forbid_sidekiq_in_transactions.rb

@@ -27,7 +27,7 @@ module Sidekiq
                 Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead.
                 MSG
               rescue Sidekiq::Worker::EnqueueFromTransactionError => e
-                Rails.logger.error(e.message) if Rails.env.production?
+                ::Rails.logger.error(e.message) if ::Rails.env.production?
                 Gitlab::Sentry.track_exception(e)
               end
             end

config/routes/project.rb

@@ -409,6 +409,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
 
         collection do
           post :toggle_shared_runners
+          post :toggle_group_runners
         end
       end
 

config/routes/user.rb

@@ -27,6 +27,13 @@ devise_scope :user do
   get '/users/almost_there' => 'confirmations#almost_there'
 end
 
+scope '-/users', module: :users do
+  resources :terms, only: [:index] do
+    post :accept, on: :member
+    post :decline, on: :member
+  end
+end
+
 scope(constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }) do
   scope(path: 'users/:username',
         as: :user,

db/migrate/20170301101006_add_ci_runner_namespaces.rb

+class AddCiRunnerNamespaces < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :ci_runner_namespaces do |t|
+      t.integer :runner_id
+      t.integer :namespace_id
+
+      t.index [:runner_id, :namespace_id], unique: true
+      t.index :namespace_id
+      t.foreign_key :ci_runners, column: :runner_id, on_delete: :cascade
+      t.foreign_key :namespaces, column: :namespace_id, on_delete: :cascade
+    end
+  end
+end

db/migrate/20170906133745_add_runners_token_to_groups.rb

+class AddRunnersTokenToGroups < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :namespaces, :runners_token, :string
+  end
+end

db/migrate/20180424090541_add_enforce_terms_to_application_settings.rb

+class AddEnforceTermsToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :enforce_terms, :boolean, default: false
+  end
+end

db/migrate/20180424134533_create_application_setting_terms.rb

+class CreateApplicationSettingTerms < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :application_setting_terms do |t|
+      t.integer :cached_markdown_version
+      t.text :terms, null: false
+      t.text :terms_html
+    end
+  end
+end

db/migrate/20180425075446_create_term_agreements.rb

+class CreateTermAgreements < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :term_agreements do |t|
+      t.references :term, index: true, null: false
+      t.foreign_key :application_setting_terms, column: :term_id
+      t.references :user, index: true, null: false, foreign_key: { on_delete: :cascade }
+      t.boolean :accepted, default: false, null: false
+
+      t.timestamps_with_timezone null: false
+    end
+
+    add_index :term_agreements, [:user_id, :term_id],
+              unique: true,
+              name: 'term_agreements_unique_index'
+  end
+
+  def down
+    remove_index :term_agreements, name: 'term_agreements_unique_index'
+
+    drop_table :term_agreements
+  end
+end

db/migrate/20180426102016_add_accepted_term_to_users.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAcceptedTermToUsers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    change_table :users do |t|
+      t.references :accepted_term,
+                   null: true
+    end
+    add_concurrent_foreign_key :users, :application_setting_terms, column: :accepted_term_id
+  end
+
+  def down
+    remove_foreign_key :users, column: :accepted_term_id
+    remove_column :users, :accepted_term_id
+  end
+end

db/migrate/20180430101916_add_runner_type_to_ci_runners.rb

+class AddRunnerTypeToCiRunners < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_runners, :runner_type, :smallint
+  end
+end

db/migrate/20180502122856_create_project_mirror_data.rb

+class CreateProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    return if table_exists?(:project_mirror_data)
+
+    create_table :project_mirror_data do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.string :status
+      t.string :jid
+      t.text :last_error
+    end
+  end
+
+  def down
+    drop_table(:project_mirror_data) if table_exists?(:project_mirror_data)
+  end
+end

db/migrate/20180503150427_add_index_to_namespaces_runners_token.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexToNamespacesRunnersToken < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :namespaces, :runners_token, unique: true
+  end
+
+  def down
+    if index_exists?(:namespaces, :runners_token, unique: true)
+      remove_index :namespaces, :runners_token
+    end
+  end
+end

db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb

+class AddIndexesToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_mirror_data, :jid
+    add_concurrent_index :project_mirror_data, :status
+  end
+
+  def down
+    remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid
+    remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status
+  end
+end

db/post_migrate/20180430143705_backfill_runner_type_for_ci_runners_post_migrate.rb

+class BackfillRunnerTypeForCiRunnersPostMigrate < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INSTANCE_RUNNER_TYPE = 1
+  PROJECT_RUNNER_TYPE = 3
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:ci_runners, :runner_type, INSTANCE_RUNNER_TYPE) do |table, query|
+      query.where(table[:is_shared].eq(true)).where(table[:runner_type].eq(nil))
+    end
+
+    update_column_in_batches(:ci_runners, :runner_type, PROJECT_RUNNER_TYPE) do |table, query|
+      query.where(table[:is_shared].eq(false)).where(table[:runner_type].eq(nil))
+    end
+  end
+
+  def down
+  end
+end

db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb

+class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  UP_MIGRATION = 'PopulateImportState'.freeze
+  DOWN_MIGRATION = 'RollbackImportStateData'.freeze
+
+  BATCH_SIZE = 1000
+  DELAY_INTERVAL = 5.minutes
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'projects'
+  end
+
+  class ProjectImportState < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'project_mirror_data'
+  end
+
+  def up
+    projects = Project.where.not(import_status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(projects, UP_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    import_state = ProjectImportState.where.not(status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(import_state, DOWN_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180425131009) do
+ActiveRecord::Schema.define(version: 20180503175054) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -40,6 +40,12 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.text "new_project_guidelines_html"
   end
 
+  create_table "application_setting_terms", force: :cascade do |t|
+    t.integer "cached_markdown_version"
+    t.text "terms", null: false
+    t.text "terms_html"
+  end
+
   create_table "application_settings", force: :cascade do |t|
     t.integer "default_projects_limit"
     t.boolean "signup_enabled"
@@ -158,6 +164,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.string "auto_devops_domain"
     t.boolean "pages_domain_verification_enabled", default: true, null: false
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
+    t.boolean "enforce_terms", default: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -453,6 +460,14 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
   add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree
 
+  create_table "ci_runner_namespaces", force: :cascade do |t|
+    t.integer "runner_id"
+    t.integer "namespace_id"
+  end
+
+  add_index "ci_runner_namespaces", ["namespace_id"], name: "index_ci_runner_namespaces_on_namespace_id", using: :btree
+  add_index "ci_runner_namespaces", ["runner_id", "namespace_id"], name: "index_ci_runner_namespaces_on_runner_id_and_namespace_id", unique: true, using: :btree
+
   create_table "ci_runner_projects", force: :cascade do |t|
     t.integer "runner_id", null: false
     t.datetime "created_at"
@@ -481,6 +496,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.integer "access_level", default: 0, null: false
     t.string "ip_address"
     t.integer "maximum_timeout"
+    t.integer "runner_type", limit: 2
   end
 
   add_index "ci_runners", ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree
@@ -1270,6 +1286,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.boolean "require_two_factor_authentication", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false
     t.integer "cached_markdown_version"
+    t.string "runners_token"
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
@@ -1280,6 +1297,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_index "namespaces", ["path"], name: "index_namespaces_on_path", using: :btree
   add_index "namespaces", ["path"], name: "index_namespaces_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"}
   add_index "namespaces", ["require_two_factor_authentication"], name: "index_namespaces_on_require_two_factor_authentication", using: :btree
+  add_index "namespaces", ["runners_token"], name: "index_namespaces_on_runners_token", unique: true, using: :btree
   add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree
 
   create_table "notes", force: :cascade do |t|
@@ -1509,6 +1527,17 @@ ActiveRecord::Schema.define(version: 20180425131009) do
 
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
+  create_table "project_mirror_data", force: :cascade do |t|
+    t.integer "project_id"
+    t.string "status"
+    t.string "jid"
+    t.text "last_error"
+  end
+
+  add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
+  add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", using: :btree
+  add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree
+
   create_table "project_statistics", force: :cascade do |t|
     t.integer "project_id", null: false
     t.integer "namespace_id", null: false
@@ -1815,6 +1844,18 @@ ActiveRecord::Schema.define(version: 20180425131009) do
 
   add_index "tags", ["name"], name: "index_tags_on_name", unique: true, using: :btree
 
+  create_table "term_agreements", force: :cascade do |t|
+    t.integer "term_id", null: false
+    t.integer "user_id", null: false
+    t.boolean "accepted", default: false, null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+  end
+
+  add_index "term_agreements", ["term_id"], name: "index_term_agreements_on_term_id", using: :btree
+  add_index "term_agreements", ["user_id", "term_id"], name: "term_agreements_unique_index", unique: true, using: :btree
+  add_index "term_agreements", ["user_id"], name: "index_term_agreements_on_user_id", using: :btree
+
   create_table "timelogs", force: :cascade do |t|
     t.integer "time_spent", null: false
     t.integer "user_id"
@@ -2003,6 +2044,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.string "preferred_language"
     t.string "rss_token"
     t.integer "theme_id", limit: 2
+    t.integer "accepted_term_id"
   end
 
   add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
@@ -2097,6 +2139,8 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
   add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
+  add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade
   add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
   add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
   add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
@@ -2188,6 +2232,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
+  add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
   add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
@@ -2202,6 +2247,8 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade
   add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade
+  add_foreign_key "term_agreements", "application_setting_terms", column: "term_id"
+  add_foreign_key "term_agreements", "users", on_delete: :cascade
   add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade
   add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
   add_foreign_key "todos", "notes", name: "fk_91d1f47b13", on_delete: :cascade
@@ -2215,6 +2262,7 @@ ActiveRecord::Schema.define(version: 20180425131009) do
   add_foreign_key "user_interacted_projects", "projects", name: "fk_722ceba4f7", on_delete: :cascade
   add_foreign_key "user_interacted_projects", "users", name: "fk_0894651f08", on_delete: :cascade
   add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade
+  add_foreign_key "users", "application_setting_terms", column: "accepted_term_id", name: "fk_789cd90b35", on_delete: :cascade
   add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
   add_foreign_key "web_hooks", "projects", name: "fk_0c8ca6d9d1", on_delete: :cascade

doc/administration/index.md

@@ -40,6 +40,7 @@ Learn how to install, configure, update, and maintain your GitLab instance.
 [source installations](../install/installation.md#installation-from-source).
 - [Environment variables](environment_variables.md): Supported environment variables that can be used to override their defaults values in order to configure GitLab.
 - [Plugins](plugins.md): With custom plugins, GitLab administrators can introduce custom integrations without modifying GitLab's source code.
+- [Enforcing Terms of Service](../user/admin_area/settings/terms.md)
 
 #### Customizing GitLab's appearance
 

doc/api/settings.md

@@ -53,6 +53,8 @@ Example response:
    "dsa_key_restriction": 0,
    "ecdsa_key_restriction": 0,
    "ed25519_key_restriction": 0,
+   "enforce_terms": true,
+   "terms": "Hello world!",
 }
 ```
 
@@ -153,6 +155,8 @@ PUT /application/settings
 | `user_default_external`                  | boolean          | no                                            | Newly registered users will by default be external                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `user_oauth_applications`                | boolean          | no                                            | Allow users to register any application to use GitLab as an OAuth provider                                                                                                                                                                                                                                                                                                                                                                                |
 | `version_check_enabled`                  | boolean          | no                                            | Let GitLab inform you when an update is available.                                                                                                                                                                                                                                                                                                                                                                                                        |
+| `enforce_terms`                          | boolean          | no                                            | Enforce application ToS to all users                                                                                                                                                                                                                                                                                                                                                                                                         |
+| `terms`                                  | text             | yes (if `enforce_terms` is true)              | Markdown content for the ToS                                                                                                                                                                                                                                                                                                                                                                                                                 |
 
 ```bash
 curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/application/settings?signup_enabled=false&default_project_visibility=internal
@@ -195,5 +199,7 @@ Example response:
   "dsa_key_restriction": 0,
   "ecdsa_key_restriction": 0,
   "ed25519_key_restriction": 0,
+  "enforce_terms": true,
+  "terms": "Hello world!",
 }
 ```

doc/development/fe_guide/style_guide_js.md

@@ -310,7 +310,7 @@ Please check this [rules][eslint-plugin-vue-rules] for more documentation.
     }));
   ```
 
-1. Don not use a singleton for the service or the store
+1. Do not use a singleton for the service or the store
   ```javascript
     // bad
     class Store {
@@ -328,9 +328,11 @@ Please check this [rules][eslint-plugin-vue-rules] for more documentation.
       }
     }
   ```
+1. Use `.vue` for Vue templates. Do not use `%template` in HAML.
 
 #### Naming
-1. **Extensions**: Use `.vue` extension for Vue components.
+
+1. **Extensions**: Use `.vue` extension for Vue components. Do not use `.js` as file extension ([#34371]).
 1. **Reference Naming**: Use PascalCase for their instances:
   ```javascript
     // bad
@@ -364,6 +366,8 @@ Please check this [rules][eslint-plugin-vue-rules] for more documentation.
     <component my-prop="prop" />
   ```
 
+[#34371]: https://gitlab.com/gitlab-org/gitlab-ce/issues/34371
+
 #### Alignment
 1. Follow these alignment styles for the template method:
   1. With more than one attribute, all attributes should be on a new line:

doc/topics/autodevops/index.md

@@ -389,7 +389,7 @@ If you have installed GitLab using a different method, you need to:
 1. [Deploy Prometheus](../../user/project/integrations/prometheus.md#configuring-your-own-prometheus-server-within-kubernetes) into your Kubernetes cluster
 1. If you would like response metrics, ensure you are running at least version
    0.9.0 of NGINX Ingress and
-   [enable Prometheus metrics](https://github.com/kubernetes/ingress/blob/master/examples/customization/custom-vts-metrics/nginx/nginx-vts-metrics-conf.yaml).
+   [enable Prometheus metrics](https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/customization/custom-vts-metrics-prometheus/nginx-vts-metrics-conf.yaml).
 1. Finally, [annotate](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
    the NGINX Ingress deployment to be scraped by Prometheus using
    `prometheus.io/scrape: "true"` and `prometheus.io/port: "10254"`.

doc/user/admin_area/settings/terms.md

+# Enforce accepting Terms of Service
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18570)
+> in [GitLab Core](https://about.gitlab.com/pricing/) 10.8
+
+## Configuration
+
+When it is required for all users of the GitLab instance to accept the
+Terms of Service, this can be configured by an admin on the settings
+page:
+
+![Enable enforcing Terms of Service](img/enforce_terms.png).
+
+The terms itself can be entered using Markdown. For each update to the
+terms, a new version is stored. When a user accepts or declines the
+terms, GitLab will keep track of which version they accepted or
+declined.
+
+When an admin enables this feature, they will automattically be
+directed to the page to accept the terms themselves. After they
+accept, they will be directed back to the settings page.
+
+## Accepting terms
+
+When this feature was enabled, the users that have not accepted the
+terms of service will be presented with a screen where they can either
+accept or decline the terms.
+
+![Respond to terms](img/respond_to_terms.png)
+
+When the user accepts the terms, they will be directed to where they
+were going. After a sign-in or sign-up this will most likely be the
+dashboard.
+
+When the user was already logged in when the feature was turned on,
+they will be asked to accept the terms on their next interaction.
+
+When a user declines the terms, they will be signed out.

features/project/commits/commits.feature

-@project_commits
-Feature: Project Commits
-  Background:
-    Given I sign in as a user
-    And I own a project
-    And I visit my project's commits page
-
-  Scenario: I browse commits list for master branch
-    Then I see project commits
-    And I should not see button to create a new merge request
-    Then I click the "Compare" tab
-    And I should not see button to create a new merge request
-
-  Scenario: I browse commits list for feature branch without a merge request
-    Given I visit commits list page for feature branch
-    Then I see feature branch commits
-    And I see button to create a new merge request
-    Then I click the "Compare" tab
-    And I see button to create a new merge request
-
-  Scenario: I browse commits list for feature branch with an open merge request
-    Given project have an open merge request
-    And I visit commits list page for feature branch
-    Then I see feature branch commits
-    And I should not see button to create a new merge request
-    And I should see button to the merge request
-    Then I click the "Compare" tab
-    And I should not see button to create a new merge request
-    And I should see button to the merge request
-
-  Scenario: I browse atom feed of commits list for master branch
-    Given I click atom feed link
-    Then I see commits atom feed
-
-  Scenario: I browse commit from list
-    Given I click on commit link
-    Then I see commit info
-    And I see side-by-side diff button
-
-  Scenario: I browse commit from list and create a new tag
-    Given I click on commit link
-    And I click on tag link
-    Then I see commit SHA pre-filled
-
-  Scenario: I browse commit with ci from list
-    Given commit has ci status
-    And repository contains ".gitlab-ci.yml" file
-    When I click on commit link
-    Then I see commit ci info
-
-  Scenario: I browse commit with side-by-side diff view
-    Given I click on commit link
-    And I click side-by-side diff button
-    Then I see inline diff button
-
-  @javascript
-  Scenario: I compare branches without a merge request
-    Given I visit compare refs page
-    And I fill compare fields with branches
-    Then I see compared branches
-    And I see button to create a new merge request
-
-  @javascript
-  Scenario: I compare branches with an open merge request
-    Given project have an open merge request
-    And I visit compare refs page
-    And I fill compare fields with branches
-    Then I see compared branches
-    And I should not see button to create a new merge request
-    And I should see button to the merge request
-
-  @javascript
-  Scenario: I compare refs
-    Given I visit compare refs page
-    And I fill compare fields with refs
-    Then I see compared refs
-    And I unfold diff
-    Then I should see additional file lines
-
-  Scenario: I browse commits for a specific path
-    Given I visit my project's commits page for a specific path
-    Then I see breadcrumb links
-
-  # TODO: Implement feature in graphs
-  #Scenario: I browse commits stats
-    #Given I visit my project's commits stats page
-    #Then I see commits stats
-
-  Scenario: I browse a commit with an image
-    Given I visit a commit with an image that changed
-    Then The diff links to both the previous and current image
-
-  @javascript
-  Scenario: I filter commits by message
-    When I search "submodules" commits
-    Then I should see only "submodules" commits

features/steps/project/commits/commits.rb

-class Spinach::Features::ProjectCommits < Spinach::FeatureSteps
-  include SharedAuthentication
-  include SharedProject
-  include SharedPaths
-  include SharedDiffNote
-  include RepoHelpers
-
-  step 'I see project commits' do
-    commit = @project.repository.commit
-    expect(page).to have_content(@project.name)
-    expect(page).to have_content(commit.message[0..20])
-    expect(page).to have_content(commit.short_id)
-  end
-
-  step 'I click atom feed link' do
-    click_link "Commits feed"
-  end
-
-  step 'I see commits atom feed' do
-    commit = @project.repository.commit
-    expect(response_headers['Content-Type']).to have_content("application/atom+xml")
-    expect(body).to have_selector("title", text: "#{@project.name}:master commits")
-    expect(body).to have_selector("author email", text: commit.author_email)
-    expect(body).to have_selector("entry summary", text: commit.description[0..10].delete("\r\n"))
-  end
-
-  step 'I click on tag link' do
-    click_link "Tag"
-  end
-
-  step 'I see commit SHA pre-filled' do
-    expect(page).to have_selector("input[value='#{sample_commit.id}']")
-  end
-
-  step 'I click on commit link' do
-    visit project_commit_path(@project, sample_commit.id)
-  end
-
-  step 'I see commit info' do
-    expect(page).to have_content sample_commit.message
-    expect(page).to have_content "Showing #{sample_commit.files_changed_count} changed files"
-  end
-
-  step 'I fill compare fields with branches' do
-    select_using_dropdown('from', 'feature')
-    select_using_dropdown('to', 'master')
-
-    click_button 'Compare'
-  end
-
-  step 'I fill compare fields with refs' do
-    select_using_dropdown('from', sample_commit.parent_id, true)
-    select_using_dropdown('to', sample_commit.id, true)
-
-    click_button "Compare"
-  end
-
-  step 'I unfold diff' do
-    @diff = first('.js-unfold')
-    @diff.click
-    sleep 2
-  end
-
-  step 'I should see additional file lines' do
-    page.within @diff.query_scope do
-      expect(first('.new_line').text).not_to have_content "..."
-    end
-  end
-
-  step 'I see compared refs' do
-    expect(page).to have_content "Commits (1)"
-    expect(page).to have_content "Showing 2 changed files"
-  end
-
-  step 'I visit commits list page for feature branch' do
-    visit project_commits_path(@project, 'feature', { limit: 5 })
-  end
-
-  step 'I see feature branch commits' do
-    commit = @project.repository.commit('0b4bc9a')
-    expect(page).to have_content(@project.name)
-    expect(page).to have_content(commit.message[0..12])
-    expect(page).to have_content(commit.short_id)
-  end
-
-  step 'project have an open merge request' do
-    create(:merge_request,
-           title: 'Feature',
-           source_project: @project,
-           source_branch: 'feature',
-           target_branch: 'master',
-           author: @project.users.first
-          )
-  end
-
-  step 'I click the "Compare" tab' do
-    click_link('Compare')
-  end
-
-  step 'I fill compare fields with branches' do
-    select_using_dropdown('from', 'master')
-    select_using_dropdown('to', 'feature')
-
-    click_button 'Compare'
-  end
-
-  step 'I see compared branches' do
-    expect(page).to have_content 'Commits (1)'
-    expect(page).to have_content 'Showing 1 changed file with 5 additions and 0 deletions'
-  end
-
-  step 'I see button to create a new merge request' do
-    expect(page).to have_link 'Create merge request'
-  end
-
-  step 'I should not see button to create a new merge request' do
-    expect(page).not_to have_link 'Create merge request'
-  end
-
-  step 'I should see button to the merge request' do
-    merge_request = MergeRequest.find_by(title: 'Feature')
-    expect(page).to have_link "View open merge request", href: project_merge_request_path(@project, merge_request)
-  end
-
-  step 'I see breadcrumb links' do
-    expect(page).to have_selector('ul.breadcrumb')
-    expect(page).to have_selector('ul.breadcrumb a', count: 4)
-  end
-
-  step 'I see commits stats' do
-    expect(page).to have_content 'Top 50 Committers'
-    expect(page).to have_content 'Committers'
-    expect(page).to have_content 'Total commits'
-    expect(page).to have_content 'Authors'
-  end
-
-  step 'I visit a commit with an image that changed' do
-    visit project_commit_path(@project, sample_image_commit.id)
-  end
-
-  step 'The diff links to both the previous and current image' do
-    links = page.all('.file-actions a')
-    expect(links[0]['href']).to match %r{blob/#{sample_image_commit.old_blob_id}}
-    expect(links[1]['href']).to match %r{blob/#{sample_image_commit.new_blob_id}}
-  end
-
-  step 'I see inline diff button' do
-    expect(page).to have_content "Inline"
-  end
-
-  step 'I click side-by-side diff button' do
-    find('#parallel-diff-btn').click
-  end
-
-  step 'commit has ci status' do
-    @project.enable_ci
-    @pipeline = create(:ci_pipeline, project: @project, sha: sample_commit.id)
-    create(:ci_build, pipeline: @pipeline)
-  end
-
-  step 'repository contains ".gitlab-ci.yml" file' do
-    allow_any_instance_of(Ci::Pipeline).to receive(:ci_yaml_file).and_return(String.new)
-  end
-
-  step 'I see commit ci info' do
-    expect(page).to have_content "Pipeline ##{@pipeline.id} pending"
-  end
-
-  step 'I search "submodules" commits' do
-    fill_in 'commits-search', with: 'submodules'
-  end
-
-  step 'I should see only "submodules" commits' do
-    expect(page).to have_content "More submodules"
-    expect(page).not_to have_content "Change some files"
-  end
-
-  def select_using_dropdown(dropdown_type, selection, is_commit = false)
-    dropdown = find(".js-compare-#{dropdown_type}-dropdown")
-    dropdown.find(".compare-dropdown-toggle").click
-    dropdown.find('.dropdown-menu', visible: true)
-    dropdown.fill_in("Filter by Git revision", with: selection)
-
-    if is_commit
-      dropdown.find('input[type="search"]').send_keys(:return)
-    else
-      find_link(selection, visible: true).click
-    end
-
-    dropdown.find('.dropdown-menu', visible: false)
-  end
-end