Merge branch '42099-port-push-mirroring-to-ce' into 'master'

Make push mirrors CE

See merge request gitlab-org/gitlab-ee!5484

app/controllers/projects/mirrors_controller.rb

+class Projects::MirrorsController < Projects::ApplicationController
+  include RepositorySettingsRedirect
+
+  prepend EE::Projects::MirrorsController
+
+  # Authorize
+  before_action :remote_mirror, only: [:update]
+  before_action :check_mirror_available!
+  before_action :authorize_admin_project!
+
+  layout "project_settings"
+
+  def show
+    redirect_to_repository_settings(project)
+  end
+
+  def update
+    if project.update_attributes(mirror_params)
+      flash[:notice] = 'Mirroring settings were successfully updated.'
+    else
+      flash[:alert] = project.errors.full_messages.join(', ').html_safe
+    end
+
+    respond_to do |format|
+      format.html { redirect_to_repository_settings(project) }
+      format.json do
+        if project.errors.present?
+          render json: project.errors, status: :unprocessable_entity
+        else
+          render json: ProjectMirrorSerializer.new.represent(project)
+        end
+      end
+    end
+  end
+
+  def update_now
+    if params[:sync_remote]
+      project.update_remote_mirrors
+      flash[:notice] = "The remote repository is being updated..."
+    end
+
+    redirect_to_repository_settings(project)
+  end
+
+  private
+
+  def remote_mirror
+    @remote_mirror = project.remote_mirrors.first_or_initialize
+  end
+
+  def check_mirror_available!
+    Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
+  end
+
+  def mirror_params_attributes
+    [
+      remote_mirrors_attributes: %i[
+        url
+        id
+        enabled
+        only_protected_branches
+      ]
+    ]
+  end
+
+  def mirror_params
+    params.require(:project).permit(mirror_params_attributes)
+  end
+end

app/controllers/projects/settings/repository_controller.rb

@@ -4,6 +4,7 @@ module Projects
       include SafeMirrorParams
 
       before_action :authorize_admin_project!
+      before_action :remote_mirror, only: [:show]
 
       prepend ::EE::Projects::Settings::RepositoryController
 
@@ -29,6 +30,7 @@ module Projects
 
         define_deploy_token
         define_protected_refs
+        remote_mirror
 
         render 'show'
       end
@@ -45,6 +47,10 @@ module Projects
         load_gon_index
       end
 
+      def remote_mirror
+        @remote_mirror = project.remote_mirrors.first_or_initialize
+      end
+
       def access_levels_options
         {
           create_access_levels: levels_for_dropdown,

app/helpers/application_settings_helper.rb

@@ -251,7 +251,8 @@ module ApplicationSettingsHelper
       :version_check_enabled,
       :allow_local_requests_from_hooks_and_services,
       :enforce_terms,
-      :terms
+      :terms,
+      :mirror_available
     ]
   end
 end

app/models/application_setting.rb

@@ -335,7 +335,8 @@ class ApplicationSetting < ActiveRecord::Base
       gitaly_timeout_fast: 10,
       gitaly_timeout_medium: 30,
       gitaly_timeout_default: 55,
-      allow_local_requests_from_hooks_and_services: false
+      allow_local_requests_from_hooks_and_services: false,
+      mirror_available: true
     }
   end
 

app/models/project.rb

@@ -68,8 +68,9 @@ class Project < ActiveRecord::Base
   default_value_for :only_mirror_protected_branches, true
 
   add_authentication_token_field :runners_token
-  before_save :ensure_runners_token
+  before_validation :mark_remote_mirrors_for_removal
 
+  before_save :ensure_runners_token
   after_save :update_project_statistics, if: :namespace_id_changed?
 
   after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? }
@@ -248,11 +249,17 @@ class Project < ActiveRecord::Base
   has_many :project_badges, class_name: 'ProjectBadge'
   has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :remote_mirrors, inverse_of: :project
+
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :import_data
   accepts_nested_attributes_for :auto_devops, update_only: true
 
+  accepts_nested_attributes_for :remote_mirrors,
+                                allow_destroy: true,
+                                reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
+
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
@@ -342,6 +349,7 @@ class Project < ActiveRecord::Base
   scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
   scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
   scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
+  scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
 
   scope :with_group_runners_enabled, -> do
     joins(:ci_cd_settings)
@@ -771,6 +779,37 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
+  def has_remote_mirror?
+    remote_mirror_available? && remote_mirrors.enabled.exists?
+  end
+
+  def updating_remote_mirror?
+    remote_mirrors.enabled.started.exists?
+  end
+
+  def update_remote_mirrors
+    return unless remote_mirror_available?
+
+    remote_mirrors.enabled.each(&:sync)
+  end
+
+  def mark_stuck_remote_mirrors_as_failed!
+    remote_mirrors.stuck.update_all(
+      update_status: :failed,
+      last_error: 'The remote mirror took to long to complete.',
+      last_update_at: Time.now
+    )
+  end
+
+  def mark_remote_mirrors_for_removal
+    remote_mirrors.each(&:mark_for_delete_if_blank_url)
+  end
+
+  def remote_mirror_available?
+    remote_mirror_available_overridden ||
+      ::Gitlab::CurrentSettings.mirror_available
+  end
+
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit

ee/app/models/remote_mirror.rb → app/models/remote_mirror.rb

 class RemoteMirror < ActiveRecord::Base
   include AfterCommitQueue
 
+  prepend EE::RemoteMirror
+
   PROTECTED_BACKOFF_DELAY   = 1.minute
   UNPROTECTED_BACKOFF_DELAY = 5.minutes
 
@@ -86,9 +88,12 @@ class RemoteMirror < ActiveRecord::Base
     raw.update(options)
   end
 
+  def sync?
+    enabled?
+  end
+
   def sync
-    return unless enabled?
-    return if Gitlab::Geo.secondary?
+    return unless sync?
 
     if recently_scheduled?
       RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
@@ -103,8 +108,7 @@ class RemoteMirror < ActiveRecord::Base
     return false unless project.repository_exists?
     return false if project.pending_delete?
 
-    # Sync is only enabled when the license permits it
-    project.feature_available?(:repository_mirrors)
+    true
   end
   alias_method :enabled?, :enabled
 

app/policies/project_policy.rb

@@ -81,6 +81,11 @@ class ProjectPolicy < BasePolicy
     project.merge_requests_allowing_push_to_user(user).any?
   end
 
+  with_scope :global
+  condition(:mirror_available, score: 0) do
+    ::Gitlab::CurrentSettings.current_application_settings.mirror_available
+  end
+
   # We aren't checking `:read_issue` or `:read_merge_request` in this case
   # because it could be possible for a user to see an issuable-iid
   # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be
@@ -247,6 +252,8 @@ class ProjectPolicy < BasePolicy
     enable :create_cluster
   end
 
+  rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
+
   rule { archived }.policy do
     prevent :push_code
     prevent :push_to_delete_protected_branch

app/serializers/project_mirror_entity.rb

+class ProjectMirrorEntity < Grape::Entity
+  prepend ::EE::ProjectMirrorEntity
+
+  expose :id
+
+  expose :remote_mirrors_attributes do |project|
+    next [] unless project.remote_mirrors.present?
+
+    project.remote_mirrors.map do |remote|
+      remote.as_json(only: %i[id url enabled])
+    end
+  end
+end

ee/app/views/admin/application_settings/_repository_mirrors_form.html.haml → app/views/admin/application_settings/_repository_mirrors_form.html.haml

@@ -13,27 +13,7 @@
           If disabled, only admins will be able to setup mirrors in projects.
           = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring')
 
-  - if Gitlab.com?
-    .form-group
-      = f.label :mirror_max_delay, class: 'control-label col-sm-2' do
-        Maximum delay (Minutes)
-      .col-sm-10
-        = f.number_field :mirror_max_delay, class: 'form-control', min: 0
-        %span.help-block#mirror_max_delay_help_block
-        Maximum time between updates that a mirror can have when scheduled to synchronize.
-    .form-group
-      = f.label :mirror_max_capacity, class: 'control-label col-sm-2' do
-        Maximum capacity
-      .col-sm-10
-        = f.number_field :mirror_max_capacity, class: 'form-control', min: 0
-        %span.help-block#mirror_max_capacity_help_block
-        Maximum number of mirrors that can be synchronizing at the same time.
-    .form-group
-      = f.label :mirror_capacity_threshold, class: 'control-label col-sm-2' do
-        Capacity threshold
-      .col-sm-10
-        = f.number_field :mirror_capacity_threshold, class: 'form-control', min: 0
-        %span.help-block#mirror_capacity_threshold
-        Minimum capacity to be available before we schedule more mirrors preemptively.
+  - if Gitlab.com? && License.feature_available?(:repository_mirrors)
+    = render 'mirror_settings', f: f
 
   = f.submit 'Save changes', class: "btn btn-success"

app/views/admin/application_settings/show.html.haml

@@ -314,6 +314,17 @@
   .settings-content
     = render 'outbound'
 
+%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      = _('Repository mirror settings')
+    %button.btn.js-settings-toggle{ type: 'button' }
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      = _('Configure push and pull mirrors.')
+  .settings-content
+    = render partial: 'repository_mirrors_form'
+
 -# EE-only
 - if Gitlab::Geo.license_allows?
   %section.settings.as-geo.no-animate#js-geo-settings{ class: ('expanded' if expanded) }
@@ -339,18 +350,6 @@
     .settings-content
       = render 'external_authorization_service_form'
 
-- if License.feature_available?(:repository_mirrors)
-  %section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
-    .settings-header
-      %h4
-        = _('Repository mirror settings')
-      %button.btn.js-settings-toggle{ type: 'button' }
-        = expanded ? 'Collapse' : 'Expand'
-      %p
-        = _('Time between updates and capacity settings.')
-    .settings-content
-      = render partial: 'repository_mirrors_form'
-
 - if License.feature_available?(:elastic_search)
   %section.settings.as-elasticsearch.no-animate#js-elasticsearch-settings{ class: ('expanded' if expanded) }
     .settings-header

app/workers/all_queues.yml

@@ -113,6 +113,7 @@
 - update_user_activity
 - upload_checksum
 - web_hook
+- repository_update_remote_mirror
 
 # EE-specific queues
 
@@ -172,5 +173,4 @@
 - project_update_repository_storage
 - rebase
 - repository_update_mirror
-- repository_update_remote_mirror
 - chat_notification

config/routes/project.rb

@@ -200,13 +200,14 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
-      ## EE-specific
       resource :mirror, only: [:show, :update] do
         member do
           get :ssh_host_keys, constraints: { format: :json }
           post :update_now
         end
       end
+
+      ## EE-specific
       resources :push_rules, constraints: { id: /\d+/ }, only: [:update]
       ## EE-specific
 

config/sidekiq_queues.yml

@@ -73,15 +73,15 @@
   - [object_storage, 1]
   - [plugin, 1]
   - [pipeline_background, 1]
+  - [repository_update_remote_mirror, 1]
+  - [repository_remove_remote, 1]
 
   # EE-specific queues
   - [ldap_group_sync, 2]
   - [create_github_webhook, 2]
   - [chat_notification, 2]
   - [geo, 1]
-  - [repository_remove_remote, 1]
   - [repository_update_mirror, 1]
-  - [repository_update_remote_mirror, 1]
   - [project_import_schedule, 1]
   - [project_update_repository_storage, 1]
   - [admin_emails, 1]

ee/app/controllers/ee/projects/mirrors_controller.rb

+module EE
+  module Projects
+    module MirrorsController
+      extend ::Gitlab::Utils::Override
+      extend ActiveSupport::Concern
+
+      prepended do
+        include SafeMirrorParams
+      end
+
+      def ssh_host_keys
+        lookup = SshHostKey.new(project: project, url: params[:ssh_url])
+
+        if lookup.error.present?
+          # Failed to read keys
+          render json: { message: lookup.error }, status: 400
+        elsif lookup.known_hosts.nil?
+          # Still working, come back later
+          render body: nil, status: 204
+        else
+          render json: lookup
+        end
+      rescue ArgumentError => err
+        render json: { message: err.message }, status: 400
+      end
+
+      override :update
+      def update
+        if project.update_attributes(safe_mirror_params)
+          if project.mirror?
+            project.force_import_job!
+
+            flash[:notice] = "Mirroring settings were successfully updated. The project is being updated."
+          elsif project.previous_changes.key?('mirror')
+            flash[:notice] = "Mirroring was successfully disabled."
+          else
+            flash[:notice] = "Mirroring settings were successfully updated."
+          end
+        else
+          flash[:alert] = project.errors.full_messages.join(', ').html_safe
+        end
+
+        respond_to do |format|
+          format.html { redirect_to_repository_settings(project) }
+          format.json do
+            if project.errors.present?
+              render json: project.errors, status: :unprocessable_entity
+            else
+              render json: ProjectMirrorSerializer.new.represent(project)
+            end
+          end
+        end
+      end
+
+      override :update_now
+      def update_now
+        if params[:sync_remote]
+          project.update_remote_mirrors
+          flash[:notice] = "The remote repository is being updated..."
+        else
+          project.force_import_job!
+          flash[:notice] = "The repository is being updated..."
+        end
+
+        redirect_to_repository_settings(project)
+      end
+
+      def mirror_params_attributes
+        if can?(current_user, :admin_mirror, project)
+          super + mirror_params_attributes_ee
+        else
+          super
+        end
+      end
+
+      private
+
+      def mirror_params_attributes_ee
+        [
+          :mirror,
+          :import_url,
+          :username_only_import_url,
+          :mirror_user_id,
+          :mirror_trigger_builds,
+          :only_mirror_protected_branches,
+          :mirror_overwrites_diverged_branches,
+
+          import_data_attributes: %i[
+            id
+            auth_method
+            password
+            ssh_known_hosts
+            regenerate_ssh_private_key
+          ]
+        ]
+      end
+
+      def safe_mirror_params
+        params = mirror_params
+
+        params[:mirror_user_id] = current_user.id unless valid_mirror_user?(params)
+
+        import_data = params[:import_data_attributes]
+        if import_data.present?
+          # Prevent Rails from destroying the existing import data
+          import_data[:id] ||= project.import_data&.id
+
+          # If the known hosts data is being set, store details about who and when
+          if import_data[:ssh_known_hosts].present?
+            import_data[:ssh_known_hosts_verified_at] = Time.now
+            import_data[:ssh_known_hosts_verified_by_id] = current_user.id
+          end
+        end
+
+        params
+      end
+    end
+  end
+end

ee/app/controllers/ee/projects/settings/repository_controller.rb

@@ -6,7 +6,6 @@ module EE
 
         prepended do
           before_action :push_rule, only: [:show]
-          before_action :remote_mirror, only: [:show]
         end
 
         private
@@ -18,12 +17,6 @@ module EE
           @push_rule = project.push_rule # rubocop:disable Gitlab/ModuleWithInstanceVariables
         end
 
-        def remote_mirror
-          return unless project.feature_available?(:repository_mirrors)
-
-          @remote_mirror = project.remote_mirrors.first_or_initialize # rubocop:disable Gitlab/ModuleWithInstanceVariables
-        end
-
         # rubocop:disable Gitlab/ModuleWithInstanceVariables
         def acces_levels_options
           super.merge(

ee/app/controllers/projects/mirrors_controller.rb

-class Projects::MirrorsController < Projects::ApplicationController
-  include RepositorySettingsRedirect
-  include SafeMirrorParams
-  # Authorize
-  before_action :authorize_admin_mirror!
-  before_action :remote_mirror, only: [:update]
-  before_action :check_repository_mirrors_available!
-
-  layout "project_settings"
-
-  def show
-    redirect_to_repository_settings(@project)
-  end
-
-  def ssh_host_keys
-    lookup = SshHostKey.new(project: project, url: params[:ssh_url])
-
-    if lookup.error.present?
-      # Failed to read keys
-      render json: { message: lookup.error }, status: 400
-    elsif lookup.known_hosts.nil?
-      # Still working, come back later
-      render body: nil, status: 204
-    else
-      render json: lookup
-    end
-  rescue ArgumentError => err
-    render json: { message: err.message }, status: 400
-  end
-
-  def update
-    if @project.update_attributes(safe_mirror_params)
-      if @project.mirror?
-        @project.force_import_job!
-
-        flash[:notice] = "Mirroring settings were successfully updated. The project is being updated."
-      elsif project.previous_changes.key?('mirror')
-        flash[:notice] = "Mirroring was successfully disabled."
-      else
-        flash[:notice] = "Mirroring settings were successfully updated."
-      end
-    else
-      flash[:alert] = @project.errors.full_messages.join(', ').html_safe
-    end
-
-    respond_to do |format|
-      format.html { redirect_to_repository_settings(@project) }
-      format.json do
-        if @project.errors.present?
-          render json: @project.errors, status: :unprocessable_entity
-        else
-          render json: ProjectMirrorSerializer.new.represent(@project)
-        end
-      end
-    end
-  end
-
-  def update_now
-    if params[:sync_remote]
-      @project.update_remote_mirrors
-      flash[:notice] = "The remote repository is being updated..."
-    else
-      @project.force_import_job!
-      flash[:notice] = "The repository is being updated..."
-    end
-
-    redirect_to_repository_settings(@project)
-  end
-
-  private
-
-  def remote_mirror
-    @remote_mirror = @project.remote_mirrors.first_or_initialize
-  end
-
-  def mirror_params_attributes
-    [
-      :mirror,
-      :import_url,
-      :username_only_import_url,
-      :mirror_user_id,
-      :mirror_trigger_builds,
-      :only_mirror_protected_branches,
-      :mirror_overwrites_diverged_branches,
-
-      import_data_attributes: %i[
-        id
-        auth_method
-        password
-        ssh_known_hosts
-        regenerate_ssh_private_key
-      ],
-
-      remote_mirrors_attributes: %i[
-        url
-        id
-        enabled
-        only_protected_branches
-      ]
-    ]
-  end
-
-  def mirror_params
-    params.require(:project).permit(mirror_params_attributes)
-  end
-
-  def safe_mirror_params
-    params = mirror_params
-
-    params[:mirror_user_id] = current_user.id unless valid_mirror_user?(params)
-
-    import_data = params[:import_data_attributes]
-    if import_data.present?
-      # Prevent Rails from destroying the existing import data
-      import_data[:id] ||= project.import_data&.id
-
-      # If the known hosts data is being set, store details about who and when
-      if import_data[:ssh_known_hosts].present?
-        import_data[:ssh_known_hosts_verified_at] = Time.now
-        import_data[:ssh_known_hosts_verified_by_id] = current_user.id
-      end
-    end
-
-    params
-  end
-end

ee/app/helpers/ee/application_settings_helper.rb

@@ -55,8 +55,7 @@ module EE
         :slack_app_id,
         :slack_app_secret,
         :slack_app_verification_token,
-        :allow_group_owners_to_manage_ldap,
-        :mirror_available
+        :allow_group_owners_to_manage_ldap
       ]
     end
 

ee/app/models/ee/application_setting.rb

@@ -96,7 +96,6 @@ module EE
           mirror_capacity_threshold: Settings.gitlab['mirror_capacity_threshold'],
           mirror_max_capacity: Settings.gitlab['mirror_max_capacity'],
           mirror_max_delay: Settings.gitlab['mirror_max_delay'],
-          mirror_available: true,
           repository_size_limit: 0,
           slack_app_enabled: false,
           slack_app_id: nil,

ee/app/models/ee/project.rb

@@ -13,8 +13,6 @@ module EE
       include EE::DeploymentPlatform
       include EachBatch
 
-      before_validation :mark_remote_mirrors_for_removal
-
       before_save :set_override_pull_mirror_available, unless: -> { ::Gitlab::CurrentSettings.mirror_available }
       before_save :set_next_execution_timestamp_to_now, if: ->(project) { project.mirror? && project.mirror_changed? && project.import_state }
 
@@ -33,7 +31,6 @@ module EE
       has_many :approvers, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
       has_many :approver_groups, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
       has_many :audit_events, as: :entity
-      has_many :remote_mirrors, inverse_of: :project
       has_many :path_locks
 
       has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_project_id
@@ -54,7 +51,6 @@ module EE
           .where("import_state.retry_count <= ?", ::Gitlab::Mirror::MAX_RETRY)
       end
 
-      scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
       scope :with_wiki_enabled,   -> { with_feature_enabled(:wiki) }
 
       scope :verified_repos, -> { joins(:repository_state).merge(ProjectRepositoryState.verified_repos) }
@@ -73,10 +69,6 @@ module EE
 
       validates :approvals_before_merge, numericality: true, allow_blank: true
 
-      accepts_nested_attributes_for :remote_mirrors,
-        allow_destroy: true,
-        reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
-
       with_options if: :mirror? do
         validates :import_url, presence: true
         validates :mirror_user, presence: true
@@ -208,30 +200,6 @@ module EE
       self.import_state.retry_limit_exceeded?
     end
 
-    def has_remote_mirror?
-      feature_available?(:repository_mirrors) &&
-        remote_mirror_available? &&
-        remote_mirrors.enabled.exists?
-    end
-
-    def updating_remote_mirror?
-      remote_mirrors.enabled.started.exists?
-    end
-
-    def update_remote_mirrors
-      return unless feature_available?(:repository_mirrors) && remote_mirror_available?
-
-      remote_mirrors.enabled.each(&:sync)
-    end
-
-    def mark_stuck_remote_mirrors_as_failed!
-      remote_mirrors.stuck.update_all(
-        update_status: :failed,
-        last_error: 'The remote mirror took to long to complete.',
-        last_update_at: Time.now
-      )
-    end
-
     def fetch_mirror
       return unless mirror?
 
@@ -414,10 +382,6 @@ module EE
       username_only_import_url
     end
 
-    def mark_remote_mirrors_for_removal
-      remote_mirrors.each(&:mark_for_delete_if_blank_url)
-    end
-
     def change_repository_storage(new_repository_storage_key)
       return if repository_read_only?
       return if repository_storage == new_repository_storage_key
@@ -508,11 +472,6 @@ module EE
       end
     end
 
-    def remote_mirror_available?
-      remote_mirror_available_overridden ||
-        ::Gitlab::CurrentSettings.mirror_available
-    end
-
     def pull_mirror_available?
       pull_mirror_available_overridden ||
         ::Gitlab::CurrentSettings.mirror_available

ee/app/models/ee/remote_mirror.rb

+module EE
+  module RemoteMirror
+    extend ActiveSupport::Concern
+
+    def sync?
+      super && !::Gitlab::Geo.secondary?
+    end
+  end
+end

ee/app/policies/ee/project_policy.rb

@@ -43,11 +43,6 @@ module EE
         !PushRule.global&.commit_committer_check
       end
 
-      with_scope :global
-      condition(:mirror_available, score: 0) do
-        ::Gitlab::CurrentSettings.current_application_settings.mirror_available
-      end
-
       rule { admin }.enable :change_repository_storage
 
       rule { support_bot }.enable :guest_access

ee/app/serializers/ee/project_mirror_entity.rb

+module EE
+  module ProjectMirrorEntity
+    extend ActiveSupport::Concern
+
+    prepended do
+      expose :mirror
+      expose :import_url
+      expose :username_only_import_url
+      expose :mirror_user_id
+      expose :mirror_trigger_builds
+      expose :only_mirror_protected_branches
+      expose :mirror_overwrites_diverged_branches
+
+      expose :import_data_attributes do |project|
+        import_data = project.import_data
+        next nil unless import_data.present?
+
+        data = import_data.as_json(
+          only: :id,
+          methods: %i[
+            auth_method
+            ssh_known_hosts
+            ssh_known_hosts_verified_at
+            ssh_known_hosts_verified_by_id
+            ssh_public_key
+          ]
+        )
+
+        data[:ssh_known_hosts_fingerprints] = import_data.ssh_known_hosts_fingerprints.as_json
+
+        data
+      end
+    end
+  end
+end

ee/app/serializers/project_mirror_entity.rb

-class ProjectMirrorEntity < Grape::Entity
-  expose :id
-  expose :mirror
-  expose :import_url
-  expose :username_only_import_url
-  expose :mirror_user_id
-  expose :mirror_trigger_builds
-  expose :only_mirror_protected_branches
-  expose :mirror_overwrites_diverged_branches
-
-  expose :import_data_attributes do |project|
-    import_data = project.import_data
-    next nil unless import_data.present?
-
-    data = import_data.as_json(
-      only: :id,
-      methods: %i[
-        auth_method
-        ssh_known_hosts
-        ssh_known_hosts_verified_at
-        ssh_known_hosts_verified_by_id
-        ssh_public_key
-      ]
-    )
-
-    data[:ssh_known_hosts_fingerprints] = import_data.ssh_known_hosts_fingerprints.as_json
-
-    data
-  end
-
-  expose :remote_mirrors_attributes do |project|
-    next [] unless project.remote_mirrors.present?
-
-    project.remote_mirrors.map do |remote|
-      remote.as_json(only: %i[id url enabled])
-    end
-  end
-end

ee/app/views/admin/application_settings/_mirror_settings.html.haml

+.form-group
+  = f.label :mirror_max_delay, class: 'control-label col-sm-2' do
+    Maximum delay (Minutes)
+  .col-sm-10
+    = f.number_field :mirror_max_delay, class: 'form-control', min: 0
+    %span.help-block#mirror_max_delay_help_block
+    Maximum time between updates that a mirror can have when scheduled to synchronize.
+.form-group
+  = f.label :mirror_max_capacity, class: 'control-label col-sm-2' do
+    Maximum capacity
+  .col-sm-10
+    = f.number_field :mirror_max_capacity, class: 'form-control', min: 0
+    %span.help-block#mirror_max_capacity_help_block
+    Maximum number of mirrors that can be synchronizing at the same time.
+.form-group
+  = f.label :mirror_capacity_threshold, class: 'control-label col-sm-2' do
+    Capacity threshold
+  .col-sm-10
+    = f.number_field :mirror_capacity_threshold, class: 'form-control', min: 0
+    %span.help-block#mirror_capacity_threshold
+    Minimum capacity to be available before we schedule more mirrors preemptively.

ee/app/views/projects/mirrors/_show.html.haml

 - if can?(current_user, :admin_mirror, @project)
   = render 'projects/mirrors/pull'
+- if can?(current_user, :admin_remote_mirror, @project)
   = render 'projects/mirrors/push'

ee/changelogs/unreleased/42099-port-push-mirroring-to-ce.yml

+---
+title: Adds push mirrors to GitLab Community Edition
+merge_request: 5484
+author:
+type: changed

ee/spec/controllers/projects/mirrors_controller_spec.rb

@@ -7,34 +7,6 @@ describe Projects::MirrorsController do
     set(:project) { create(:project, :repository) }
     let(:url) { 'http://foo.com' }
 
-    context 'when remote mirrors are disabled' do
-      before do
-        stub_application_setting(mirror_available: false)
-      end
-
-      context 'when user is admin' do
-        let(:admin) { create(:user, :admin) }
-
-        it 'creates a new remote mirror' do
-          sign_in(admin)
-
-          expect do
-            do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => url } })
-          end.to change { RemoteMirror.count }.to(1)
-        end
-      end
-
-      context 'when user is not admin' do
-        it 'does not create a new remote mirror' do
-          sign_in(project.owner)
-
-          expect do
-            do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => url } })
-          end.not_to change { RemoteMirror.count }
-        end
-      end
-    end
-
     context 'when the current project is a mirror' do
       let(:project) { create(:project, :repository, :mirror) }
 
@@ -67,16 +39,6 @@ describe Projects::MirrorsController do
       end
     end
 
-    context 'when the current project is not a mirror' do
-      it 'allows to create a remote mirror' do
-        sign_in(project.owner)
-
-        expect do
-          do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
-        end.to change { RemoteMirror.count }.to(1)
-      end
-    end
-
     context 'when the current project has a remote mirror' do
       let(:remote_mirror) { project.remote_mirrors.create!(enabled: 1, url: 'http://local.dev') }
 
@@ -277,40 +239,6 @@ describe Projects::MirrorsController do
         expect(flash[:alert]).to match(/must be a valid URL/)
       end
     end
-
-    context 'With valid URL for a push' do
-      before do
-        @remote_mirror_attributes = { "0" => { "enabled" => "0", url: 'https://updated.example.com' } }
-      end
-
-      it 'processes a successful update' do
-        do_put(project, remote_mirrors_attributes: @remote_mirror_attributes)
-
-        expect(response).to redirect_to(project_settings_repository_path(project))
-        expect(flash[:notice]).to match(/successfully updated/)
-      end
-
-      it 'should create a RemoteMirror object' do
-        expect { do_put(project, remote_mirrors_attributes: @remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1)
-      end
-    end
-
-    context 'With invalid URL for a push' do
-      before do
-        @remote_mirror_attributes = { "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } }
-      end
-
-      it 'processes an unsuccessful update' do
-        do_put(project, remote_mirrors_attributes: @remote_mirror_attributes)
-
-        expect(response).to redirect_to(project_settings_repository_path(project))
-        expect(flash[:alert]).to match(/must be a valid URL/)
-      end
-
-      it 'should not create a RemoteMirror object' do
-        expect { do_put(project, remote_mirrors_attributes: @remote_mirror_attributes) }.not_to change(RemoteMirror, :count)
-      end
-    end
   end
 
   describe '#ssh_host_keys', :use_clean_rails_memory_store_caching do

ee/spec/features/projects/mirror_spec.rb

@@ -13,20 +13,6 @@ feature 'Project mirror', :js do
       sign_in user
     end
 
-    context 'unlicensed' do
-      before do
-        stub_licensed_features(repository_mirrors: false)
-      end
-
-      it 'returns 404' do
-        reqs = inspect_requests do
-          visit project_mirror_path(project)
-        end
-
-        expect(reqs.first.status_code).to eq(404)
-      end
-    end
-
     context 'with Update now button' do
       let(:timestamp) { Time.now }
 

ee/spec/features/projects/settings/ee/repository_mirrors_settings_spec.rb

@@ -24,25 +24,5 @@ describe 'Project settings > [EE] repository' do
       expect(page).to have_no_selector('#project_mirror_user_id', visible: false)
       expect(page).to have_no_selector('#project_mirror_trigger_builds')
     end
-
-    it 'does not show push mirror settings' do
-      expect(page).to have_no_selector('#project_remote_mirrors_attributes_0_enabled')
-      expect(page).to have_no_selector('#project_remote_mirrors_attributes_0_url')
-    end
-  end
-
-  describe 'mirror settings', :js do
-    let(:user2) { create(:user) }
-
-    before do
-      project.add_master(user2)
-
-      visit project_settings_repository_path(project)
-    end
-
-    it 'shows push mirror settings' do
-      expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
-      expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
-    end
   end
 end

ee/spec/models/project_spec.rb

@@ -589,71 +589,6 @@ describe Project do
     end
   end
 
-  describe '#has_remote_mirror?' do
-    let(:project) { create(:project, :remote_mirror, :import_started) }
-    subject { project.has_remote_mirror? }
-
-    before do
-      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
-    end
-
-    it 'returns true when a remote mirror is enabled' do
-      is_expected.to be_truthy
-    end
-
-    it 'returns false when unlicensed' do
-      stub_licensed_features(repository_mirrors: false)
-
-      is_expected.to be_falsy
-    end
-
-    it 'returns false when remote mirror is disabled' do
-      project.remote_mirrors.first.update_attributes(enabled: false)
-
-      is_expected.to be_falsy
-    end
-  end
-
-  describe '#update_remote_mirrors' do
-    let(:project) { create(:project, :remote_mirror, :import_started) }
-    delegate :update_remote_mirrors, to: :project
-
-    before do
-      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
-    end
-
-    it 'syncs enabled remote mirror' do
-      expect_any_instance_of(RemoteMirror).to receive(:sync)
-
-      update_remote_mirrors
-    end
-
-    it 'does nothing when remote mirror is disabled globally and not overridden' do
-      stub_application_setting(mirror_available: false)
-      project.remote_mirror_available_overridden = false
-
-      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
-
-      update_remote_mirrors
-    end
-
-    it 'does nothing when unlicensed' do
-      stub_licensed_features(repository_mirrors: false)
-
-      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
-
-      update_remote_mirrors
-    end
-
-    it 'does not sync disabled remote mirrors' do
-      project.remote_mirrors.first.update_attributes(enabled: false)
-
-      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
-
-      update_remote_mirrors
-    end
-  end
-
   describe '#any_runners_limit' do
     let(:project) { create(:project, shared_runners_enabled: shared_runners_enabled) }
     let(:specific_runner) { create(:ci_runner) }
@@ -1205,32 +1140,6 @@ describe Project do
     end
   end
 
-  describe '#remote_mirror_available?' do
-    let(:project) { create(:project) }
-
-    context 'when remote mirror global setting is enabled' do
-      it 'returns true' do
-        expect(project.remote_mirror_available?).to be(true)
-      end
-    end
-
-    context 'when remote mirror global setting is disabled' do
-      before do
-        stub_application_setting(mirror_available: false)
-      end
-
-      it 'returns true when overridden' do
-        project.remote_mirror_available_overridden = true
-
-        expect(project.remote_mirror_available?).to be(true)
-      end
-
-      it 'returns false when not overridden' do
-        expect(project.remote_mirror_available?).to be(false)
-      end
-    end
-  end
-
   describe '#pull_mirror_available?' do
     let(:project) { create(:project) }
 

ee/spec/serializers/project_mirror_entity_spec.rb

@@ -66,30 +66,4 @@ describe ProjectMirrorEntity do
       end
     end
   end
-
-  describe 'push mirror' do
-    let(:project) { create(:project, :repository, :remote_mirror) }
-    let(:remote_mirror) { project.remote_mirrors.first }
-
-    it 'represents the push mirror' do
-      is_expected.to eq(
-        id: project.id,
-        mirror: false,
-        import_url: nil,
-        username_only_import_url: nil,
-        mirror_user_id: nil,
-        mirror_trigger_builds: false,
-        only_mirror_protected_branches: true,
-        mirror_overwrites_diverged_branches: nil,
-        import_data_attributes: nil,
-        remote_mirrors_attributes: [
-          {
-            id: remote_mirror.id,
-            url: remote_mirror.url,
-            enabled: true
-          }
-        ]
-      )
-    end
-  end
 end

spec/controllers/projects/mirrors_controller_spec.rb

+require 'spec_helper'
+
+describe Projects::MirrorsController do
+  include ReactiveCachingHelpers
+
+  describe 'setting up a remote mirror' do
+    set(:project) { create(:project, :repository) }
+
+    context 'when the current project is not a mirror' do
+      it 'allows to create a remote mirror' do
+        sign_in(project.owner)
+
+        expect do
+          do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
+        end.to change { RemoteMirror.count }.to(1)
+      end
+    end
+  end
+
+  describe '#update' do
+    let(:project) { create(:project, :repository, :remote_mirror) }
+
+    before do
+      sign_in(project.owner)
+    end
+
+    around do |example|
+      Sidekiq::Testing.fake! { example.run }
+    end
+
+    context 'With valid URL for a push' do
+      let(:remote_mirror_attributes) do
+        { "0" => { "enabled" => "0", url: 'https://updated.example.com' } }
+      end
+
+      it 'processes a successful update' do
+        do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:notice]).to match(/successfully updated/)
+      end
+
+      it 'should create a RemoteMirror object' do
+        expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1)
+      end
+    end
+
+    context 'With invalid URL for a push' do
+      let(:remote_mirror_attributes) do
+        { "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } }
+      end
+
+      it 'processes an unsuccessful update' do
+        do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
+
+        expect(response).to redirect_to(project_settings_repository_path(project))
+        expect(flash[:alert]).to match(/must be a valid URL/)
+      end
+
+      it 'should not create a RemoteMirror object' do
+        expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.not_to change(RemoteMirror, :count)
+      end
+    end
+  end
+
+  def do_put(project, options, extra_attrs = {})
+    attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
+    attrs[:project] = options
+
+    put :update, attrs
+  end
+end

spec/features/projects/settings/repository_settings_spec.rb

@@ -115,5 +115,20 @@ describe 'Projects > Settings > Repository settings' do
         expect(page).to have_content('Your new project deploy token has been created')
       end
     end
+
+    context 'remote mirror settings' do
+      let(:user2) { create(:user) }
+
+      before do
+        project.add_master(user2)
+
+        visit project_settings_repository_path(project)
+      end
+
+      it 'shows push mirror settings' do
+        expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
+        expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
+      end
+    end
   end
 end

spec/models/project_spec.rb

@@ -2041,6 +2041,83 @@ describe Project do
     it { expect(project.gitea_import?).to be true }
   end
 
+  describe '#has_remote_mirror?' do
+    let(:project) { create(:project, :remote_mirror, :import_started) }
+    subject { project.has_remote_mirror? }
+
+    before do
+      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+    end
+
+    it 'returns true when a remote mirror is enabled' do
+      is_expected.to be_truthy
+    end
+
+    it 'returns false when remote mirror is disabled' do
+      project.remote_mirrors.first.update_attributes(enabled: false)
+
+      is_expected.to be_falsy
+    end
+  end
+
+  describe '#update_remote_mirrors' do
+    let(:project) { create(:project, :remote_mirror, :import_started) }
+    delegate :update_remote_mirrors, to: :project
+
+    before do
+      allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
+    end
+
+    it 'syncs enabled remote mirror' do
+      expect_any_instance_of(RemoteMirror).to receive(:sync)
+
+      update_remote_mirrors
+    end
+
+    it 'does nothing when remote mirror is disabled globally and not overridden' do
+      stub_application_setting(mirror_available: false)
+      project.remote_mirror_available_overridden = false
+
+      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+      update_remote_mirrors
+    end
+
+    it 'does not sync disabled remote mirrors' do
+      project.remote_mirrors.first.update_attributes(enabled: false)
+
+      expect_any_instance_of(RemoteMirror).not_to receive(:sync)
+
+      update_remote_mirrors
+    end
+  end
+
+  describe '#remote_mirror_available?' do
+    let(:project) { create(:project) }
+
+    context 'when remote mirror global setting is enabled' do
+      it 'returns true' do
+        expect(project.remote_mirror_available?).to be(true)
+      end
+    end
+
+    context 'when remote mirror global setting is disabled' do
+      before do
+        stub_application_setting(mirror_available: false)
+      end
+
+      it 'returns true when overridden' do
+        project.remote_mirror_available_overridden = true
+
+        expect(project.remote_mirror_available?).to be(true)
+      end
+
+      it 'returns false when not overridden' do
+        expect(project.remote_mirror_available?).to be(false)
+      end
+    end
+  end
+
   describe '#ancestors_upto', :nested_groups do
     let(:parent) { create(:group) }
     let(:child) { create(:group, parent: parent) }

ee/spec/models/remote_mirror_spec.rb → spec/models/remote_mirror_spec.rb

@@ -156,18 +156,6 @@ describe RemoteMirror do
       Timecop.freeze { example.run }
     end
 
-    context 'repository mirrors not licensed' do
-      before do
-        stub_licensed_features(repository_mirrors: false)
-      end
-
-      it 'does not schedule RepositoryUpdateRemoteMirrorWorker' do
-        expect(RepositoryUpdateRemoteMirrorWorker).not_to receive(:perform_in)
-
-        remote_mirror.sync
-      end
-    end
-
     context 'with remote mirroring disabled' do
       it 'returns nil' do
         remote_mirror.update_attributes(enabled: false)

ee/spec/services/projects/update_remote_mirror_service_spec.rb → spec/services/projects/update_remote_mirror_service_spec.rb

@@ -18,14 +18,6 @@ describe Projects::UpdateRemoteMirrorService do
       allow(raw_repository).to receive(:push_remote_branches).and_return(true)
     end
 
-    it 'does nothing when unlicensed' do
-      stub_licensed_features(repository_mirrors: false)
-
-      expect(project.repository).not_to receive(:fetch_remote)
-
-      subject.execute(remote_mirror)
-    end
-
     it "fetches the remote repository" do
       expect(repository).to receive(:fetch_remote).with(remote_mirror.remote_name, no_tags: true) do
         sync_remote(repository, remote_mirror.remote_name, local_branch_names)