Commit d0cbef7e authored by Douwe Maan's avatar Douwe Maan

Merge branch '42099-port-push-mirroring-to-ce-ce-port-v-2' into 'master'

CE backport of Backports Push Mirrors to CE

Closes #42099

See merge request gitlab-org/gitlab-ce!18715
parents 32009764 2a279143
class Projects::MirrorsController < Projects::ApplicationController
include RepositorySettingsRedirect
# Authorize
before_action :remote_mirror, only: [:update]
before_action :check_mirror_available!
before_action :authorize_admin_project!
layout "project_settings"
def show
redirect_to_repository_settings(project)
end
def update
if project.update_attributes(mirror_params)
flash[:notice] = 'Mirroring settings were successfully updated.'
else
flash[:alert] = project.errors.full_messages.join(', ').html_safe
end
respond_to do |format|
format.html { redirect_to_repository_settings(project) }
format.json do
if project.errors.present?
render json: project.errors, status: :unprocessable_entity
else
render json: ProjectMirrorSerializer.new.represent(project)
end
end
end
end
def update_now
if params[:sync_remote]
project.update_remote_mirrors
flash[:notice] = "The remote repository is being updated..."
end
redirect_to_repository_settings(project)
end
private
def remote_mirror
@remote_mirror = project.remote_mirrors.first_or_initialize
end
def check_mirror_available!
Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
end
def mirror_params_attributes
[
remote_mirrors_attributes: %i[
url
id
enabled
only_protected_branches
]
]
end
def mirror_params
params.require(:project).permit(mirror_params_attributes)
end
end
......@@ -2,6 +2,7 @@ module Projects
module Settings
class RepositoryController < Projects::ApplicationController
before_action :authorize_admin_project!
before_action :remote_mirror, only: [:show]
def show
render_show
......@@ -25,6 +26,7 @@ module Projects
define_deploy_token
define_protected_refs
remote_mirror
render 'show'
end
......@@ -41,6 +43,10 @@ module Projects
load_gon_index
end
def remote_mirror
@remote_mirror = project.remote_mirrors.first_or_initialize
end
def access_levels_options
{
create_access_levels: levels_for_dropdown,
......
......@@ -250,7 +250,8 @@ module ApplicationSettingsHelper
:version_check_enabled,
:allow_local_requests_from_hooks_and_services,
:enforce_terms,
:terms
:terms,
:mirror_available
]
end
end
......@@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base
gitaly_timeout_fast: 10,
gitaly_timeout_medium: 30,
gitaly_timeout_default: 55,
allow_local_requests_from_hooks_and_services: false
allow_local_requests_from_hooks_and_services: false,
mirror_available: true
}
end
......
......@@ -65,6 +65,9 @@ class Project < ActiveRecord::Base
default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
add_authentication_token_field :runners_token
before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) }
before_save :ensure_runners_token
after_save :update_project_statistics, if: :namespace_id_changed?
......@@ -246,11 +249,17 @@ class Project < ActiveRecord::Base
has_many :project_badges, class_name: 'ProjectBadge'
has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :remote_mirrors, inverse_of: :project
accepts_nested_attributes_for :variables, allow_destroy: true
accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data
accepts_nested_attributes_for :auto_devops, update_only: true
accepts_nested_attributes_for :remote_mirrors,
allow_destroy: true,
reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
delegate :name, to: :owner, allow_nil: true, prefix: true
delegate :members, to: :team, prefix: true
delegate :add_user, :add_users, to: :team
......@@ -340,6 +349,7 @@ class Project < ActiveRecord::Base
scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
scope :with_group_runners_enabled, -> do
joins(:ci_cd_settings)
......@@ -759,6 +769,37 @@ class Project < ActiveRecord::Base
import_type == 'gitea'
end
def has_remote_mirror?
remote_mirror_available? && remote_mirrors.enabled.exists?
end
def updating_remote_mirror?
remote_mirrors.enabled.started.exists?
end
def update_remote_mirrors
return unless remote_mirror_available?
remote_mirrors.enabled.each(&:sync)
end
def mark_stuck_remote_mirrors_as_failed!
remote_mirrors.stuck.update_all(
update_status: :failed,
last_error: 'The remote mirror took to long to complete.',
last_update_at: Time.now
)
end
def mark_remote_mirrors_for_removal
remote_mirrors.each(&:mark_for_delete_if_blank_url)
end
def remote_mirror_available?
remote_mirror_available_overridden ||
::Gitlab::CurrentSettings.mirror_available
end
def check_limit
unless creator.can_create_project? || namespace.kind == 'group'
projects_limit = creator.projects_limit
......
class RemoteMirror < ActiveRecord::Base
include AfterCommitQueue
PROTECTED_BACKOFF_DELAY = 1.minute
UNPROTECTED_BACKOFF_DELAY = 5.minutes
attr_encrypted :credentials,
key: Gitlab::Application.secrets.db_key_base,
marshal: true,
encode: true,
mode: :per_attribute_iv_and_salt,
insecure_mode: true,
algorithm: 'aes-256-cbc'
default_value_for :only_protected_branches, true
belongs_to :project, inverse_of: :remote_mirrors
validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
validates :url, addressable_url: true, if: :url_changed?
before_save :set_new_remote_name, if: :mirror_url_changed?
after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
after_save :refresh_remote, if: :mirror_url_changed?
after_update :reset_fields, if: :mirror_url_changed?
after_commit :remove_remote, on: :destroy
scope :enabled, -> { where(enabled: true) }
scope :started, -> { with_update_status(:started) }
scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
state_machine :update_status, initial: :none do
event :update_start do
transition [:none, :finished, :failed] => :started
end
event :update_finish do
transition started: :finished
end
event :update_fail do
transition started: :failed
end
state :started
state :finished
state :failed
after_transition any => :started do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path)
remote_mirror.update(last_update_started_at: Time.now)
end
after_transition started: :finished do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path)
timestamp = Time.now
remote_mirror.update_attributes!(
last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
)
end
after_transition started: :failed do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path)
remote_mirror.update(last_update_at: Time.now)
end
end
def remote_name
super || fallback_remote_name
end
def update_failed?
update_status == 'failed'
end
def update_in_progress?
update_status == 'started'
end
def update_repository(options)
raw.update(options)
end
def sync?
enabled?
end
def sync
return unless sync?
if recently_scheduled?
RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
else
RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
end
end
def enabled
return false unless project && super
return false unless project.remote_mirror_available?
return false unless project.repository_exists?
return false if project.pending_delete?
true
end
alias_method :enabled?, :enabled
def updated_since?(timestamp)
last_update_started_at && last_update_started_at > timestamp && !update_failed?
end
def mark_for_delete_if_blank_url
mark_for_destruction if url.blank?
end
def mark_as_failed(error_message)
update_fail
update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
end
def url=(value)
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
mirror_url = Gitlab::UrlSanitizer.new(value)
self.credentials = mirror_url.credentials
super(mirror_url.sanitized_url)
end
def url
if super
Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
end
rescue
super
end
def safe_url
return if url.nil?
result = URI.parse(url)
result.password = '*****' if result.password
result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
result.to_s
end
private
def raw
@raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
end
def fallback_remote_name
return unless id
"remote_mirror_#{id}"
end
def recently_scheduled?
return false unless self.last_update_started_at
self.last_update_started_at >= Time.now - backoff_delay
end
def backoff_delay
if self.only_protected_branches
PROTECTED_BACKOFF_DELAY
else
UNPROTECTED_BACKOFF_DELAY
end
end
def reset_fields
update_columns(
last_error: nil,
last_update_at: nil,
last_successful_update_at: nil,
update_status: 'finished'
)
end
def set_override_remote_mirror_available
enabled = read_attribute(:enabled)
project.update(remote_mirror_available_overridden: enabled)
end
def set_new_remote_name
self.remote_name = "remote_mirror_#{SecureRandom.hex}"
end
def refresh_remote
return unless project
# Before adding a new remote we have to delete the data from
# the previous remote name
prev_remote_name = remote_name_was || fallback_remote_name
run_after_commit do
project.repository.async_remove_remote(prev_remote_name)
end
project.repository.add_remote(remote_name, url)
end
def remove_remote
return unless project # could be pending to delete so don't need to touch the git repository
project.repository.async_remove_remote(remote_name)
end
def mirror_url_changed?
url_changed? || encrypted_credentials_changed?
end
end
......@@ -854,13 +854,27 @@ class Repository
add_remote(remote_name, url, mirror_refmap: refmap)
fetch_remote(remote_name, forced: forced, prune: prune)
ensure
remove_remote(remote_name) if tmp_remote_name
async_remove_remote(remote_name) if tmp_remote_name
end
def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true)
gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
end
def async_remove_remote(remote_name)
return unless remote_name
job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name)
if job_id
Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.")
else
Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.")
end
job_id
end
def fetch_source_branch!(source_repository, source_branch, local_ref)
raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
end
......
......@@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy
project.merge_requests_allowing_push_to_user(user).any?
end
with_scope :global
condition(:mirror_available, score: 0) do
::Gitlab::CurrentSettings.current_application_settings.mirror_available
end
# We aren't checking `:read_issue` or `:read_merge_request` in this case
# because it could be possible for a user to see an issuable-iid
# (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be
......@@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy
enable :create_cluster
end
rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
rule { archived }.policy do
prevent :push_code
prevent :push_to_delete_protected_branch
......
class ProjectMirrorEntity < Grape::Entity
expose :id
expose :remote_mirrors_attributes do |project|
next [] unless project.remote_mirrors.present?
project.remote_mirrors.map do |remote|
remote.as_json(only: %i[id url enabled])
end
end
end
#
# Concern that helps with getting an exclusive lease for running a block
# of code.
#
# `#try_obtain_lease` takes a block which will be run if it was able to
# obtain the lease. Implement `#lease_timeout` to configure the timeout
# for the exclusive lease. Optionally override `#lease_key` to set the
# lease key, it defaults to the class name with underscores.
#
module ExclusiveLeaseGuard
extend ActiveSupport::Concern
def try_obtain_lease
lease = exclusive_lease.try_obtain
unless lease
log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.')
return
end
begin
yield lease
ensure
release_lease(lease)
end
end
def exclusive_lease
@lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
end
def lease_key
@lease_key ||= self.class.name.underscore
end
def lease_timeout
raise NotImplementedError,
"#{self.class.name} does not implement #{__method__}"
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
def renew_lease!
exclusive_lease.renew
end
def log_error(message, extra_args = {})
logger.error(message)
end
end
......@@ -55,6 +55,7 @@ class GitPushService < BaseService
execute_related_hooks
perform_housekeeping
update_remote_mirrors
update_caches
update_signatures
......@@ -119,6 +120,13 @@ class GitPushService < BaseService
protected
def update_remote_mirrors
return unless @project.has_remote_mirror?
@project.mark_stuck_remote_mirrors_as_failed!
@project.update_remote_mirrors
end
def execute_related_hooks
# Update merge requests that may be affected by this push. A new branch
# could cause the last commit of a merge request to change.
......
module Projects
class UpdateRemoteMirrorService < BaseService
attr_reader :errors
def execute(remote_mirror)
@errors = []
return success unless remote_mirror.enabled?
begin
repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
opts = {}
if remote_mirror.only_protected_branches?
opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
end
remote_mirror.update_repository(opts)
rescue => e
errors << e.message.strip
end
if errors.present?
error(errors.join("\n\n"))
else
success
end
end
end
end
= form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
= form_errors(@application_setting)
%fieldset
.form-group
= f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2'
.col-sm-10
.checkbox
= f.label :mirror_available do
= f.check_box :mirror_available
Allow mirrors to be setup for projects
%span.help-block
If disabled, only admins will be able to setup mirrors in projects.
= link_to icon('question-circle'), help_page_path('workflow/repository_mirroring')
= f.submit 'Save changes', class: "btn btn-success"
......@@ -313,3 +313,14 @@
= _('Allow requests to the local network from hooks and services.')
.settings-content
= render 'outbound'
%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
.settings-header
%h4
= _('Repository mirror settings')
%button.btn.js-settings-toggle{ type: 'button' }
= expanded ? 'Collapse' : 'Expand'
%p
= _('Configure push mirrors.')
.settings-content
= render partial: 'repository_mirrors_form'
.account-well.prepend-top-default.append-bottom-default
%ul
%li
The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>.
%li
Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>.
%li
The update action will time out after 10 minutes. For big repositories, use a clone/push combination.
%li
The Git LFS objects will <strong>not</strong> be synced.
- expanded = Rails.env.test?
%section.settings.no-animate{ class: ('expanded' if expanded) }
.settings-header
%h4
Push to a remote repository
%button.btn.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
%p
Set up the remote repository that you want to update with the content of the current repository
every time someone pushes to it.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
.settings-content
= form_for @project, url: project_mirror_path(@project) do |f|
%div
= form_errors(@project)
= render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
- if @remote_mirror.last_error.present?
.panel.panel-danger
.panel-heading
- if @remote_mirror.last_update_at
The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
- else
The remote repository failed to update.
- if @remote_mirror.last_successful_update_at
Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
.panel-body
%pre
:preserve
#{h(@remote_mirror.last_error.strip)}
= f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
.form-group
= rm_form.check_box :enabled, class: "pull-left"
.prepend-left-20
= rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
%p.light.append-bottom-0
Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
.form-group.has-feedback
= rm_form.label :url, "Git repository URL", class: "label-light"
= rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
.form-group
= rm_form.check_box :only_protected_branches, class: 'pull-left'
.prepend-left-20
= rm_form.label :only_protected_branches, class: 'label-light'
= link_to icon('question-circle'), help_page_path('user/project/protected_branches')
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
- if can?(current_user, :admin_remote_mirror, @project)
= render 'projects/mirrors/push'
......@@ -2,6 +2,8 @@
- page_title "Repository"
- @content_class = "limit-container-width" unless fluid_layout
= render "projects/mirrors/show"
-# Protected branches & tags use a lot of nested partials.
-# The shared parts of the views can be found in the `shared` directory.
-# Those are used throughout the actual views. These `shared` views are then
......
- if @project.has_remote_mirror?
.append-bottom-default
- if remote_mirror.update_in_progress?
%span.btn.disabled
= icon("refresh spin")
Updating&hellip;
- else
= link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do
= icon("refresh")
Update Now
- if @remote_mirror.last_successful_update_at
%p.inline.prepend-left-10
Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
......@@ -107,9 +107,11 @@
- rebase
- repository_fork
- repository_import
- repository_remove_remote
- storage_migrator
- system_hook_push
- update_merge_requests
- update_user_activity
- upload_checksum
- web_hook
- repository_update_remote_mirror
class RepositoryRemoveRemoteWorker
include ApplicationWorker
include ExclusiveLeaseGuard
LEASE_TIMEOUT = 1.hour
attr_reader :project, :remote_name
def perform(project_id, remote_name)
@remote_name = remote_name
@project = Project.find_by_id(project_id)
return unless @project
logger.info("Removing remote #{remote_name} from project #{project.id}")
try_obtain_lease do
remove_remote = @project.repository.remove_remote(remote_name)
if remove_remote
logger.info("Remote #{remote_name} was successfully removed from project #{project.id}")
else
logger.error("Could not remove remote #{remote_name} from project #{project.id}")
end
end
end
def lease_timeout
LEASE_TIMEOUT
end
def lease_key
"remove_remote_#{project.id}_#{remote_name}"
end
end
class RepositoryUpdateRemoteMirrorWorker
UpdateAlreadyInProgressError = Class.new(StandardError)
UpdateError = Class.new(StandardError)
include ApplicationWorker
include Gitlab::ShellAdapter
sidekiq_options retry: 3, dead: false
sidekiq_retry_in { |count| 30 * count }
sidekiq_retries_exhausted do |msg, _|
Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
end
def perform(remote_mirror_id, scheduled_time)
remote_mirror = RemoteMirror.find(remote_mirror_id)
return if remote_mirror.updated_since?(scheduled_time)
raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
remote_mirror.update_start
project = remote_mirror.project
current_user = project.creator
result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
raise UpdateError, result[:message] if result[:status] == :error
remote_mirror.update_finish
rescue UpdateAlreadyInProgressError
raise
rescue UpdateError => ex
fail_remote_mirror(remote_mirror, ex.message)
raise
rescue => ex
return unless remote_mirror
fail_remote_mirror(remote_mirror, ex.message)
raise UpdateError, "#{ex.class}: #{ex.message}"
end
private
def fail_remote_mirror(remote_mirror, message)
remote_mirror.mark_as_failed(message)
Rails.logger.error(message)
end
end
---
title: Adds push mirrors to GitLab Community Edition
merge_request: 18715
author:
type: changed
......@@ -174,6 +174,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
end
end
resource :mirror, only: [:show, :update] do
member do
post :update_now
end
end
resources :pipelines, only: [:index, :new, :create, :show] do
collection do
resource :pipelines_settings, path: 'settings', only: [:show, :update]
......
......@@ -73,3 +73,6 @@
- [object_storage, 1]
- [plugin, 1]
- [pipeline_background, 1]
- [repository_update_remote_mirror, 1]
- [repository_remove_remote, 1]
class CreateRemoteMirrors < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
return if table_exists?(:remote_mirrors)
create_table :remote_mirrors do |t|
t.references :project, index: true, foreign_key: { on_delete: :cascade }
t.string :url
t.boolean :enabled, default: true
t.string :update_status
t.datetime :last_update_at
t.datetime :last_successful_update_at
t.datetime :last_update_started_at
t.string :last_error
t.boolean :only_protected_branches, default: false, null: false
t.string :remote_name
t.text :encrypted_credentials
t.string :encrypted_credentials_iv
t.string :encrypted_credentials_salt
t.timestamps null: false
end
end
def down
drop_table(:remote_mirrors) if table_exists?(:remote_mirrors)
end
end
class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden)
end
def down
remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden)
end
end
class AddIndexesToRemoteMirror < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at)
end
def down
remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at
end
end
class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available)
end
def down
remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available)
end
end
......@@ -165,6 +165,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.boolean "pages_domain_verification_enabled", default: true, null: false
t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
t.boolean "enforce_terms", default: false
t.boolean "mirror_available", default: true, null: false
end
create_table "audit_events", force: :cascade do |t|
......@@ -1602,6 +1603,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.boolean "merge_requests_rebase_enabled", default: false, null: false
t.integer "jobs_cache_index"
t.boolean "pages_https_only", default: true
t.boolean "remote_mirror_available_overridden"
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......@@ -1707,6 +1709,27 @@ ActiveRecord::Schema.define(version: 20180503200320) do
add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree
create_table "remote_mirrors", force: :cascade do |t|
t.integer "project_id"
t.string "url"
t.boolean "enabled", default: true
t.string "update_status"
t.datetime "last_update_at"
t.datetime "last_successful_update_at"
t.datetime "last_update_started_at"
t.string "last_error"
t.boolean "only_protected_branches", default: false, null: false
t.string "remote_name"
t.text "encrypted_credentials"
t.string "encrypted_credentials_iv"
t.string "encrypted_credentials_salt"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
create_table "routes", force: :cascade do |t|
t.integer "source_id", null: false
t.string "source_type", null: false
......@@ -2243,6 +2266,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
add_foreign_key "remote_mirrors", "projects", on_delete: :cascade
add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
add_foreign_key "subscriptions", "projects", on_delete: :cascade
......
# Repository mirroring
Repository Mirroring is a way to mirror repositories from external sources.
It can be used to mirror all branches, tags, and commits that you have
in your repository.
Your mirror at GitLab will be updated automatically. You can
also manually trigger an update at most once every 5 minutes.
## Overview
Repository mirroring is very useful when, for some reason, you must use a
project from another source.
There are two kinds of repository mirroring features supported by GitLab:
**push** and **pull**, the latter being only available in GitLab Enterprise Edition.
The **push** method mirrors the repository in GitLab to another location.
Once the mirror repository is updated, all new branches,
tags, and commits will be visible in the project's activity feed.
Users with at least [developer access][perms] to the project can also force an
immediate update with the click of a button. This button will not be available if
the mirror is already being updated or 5 minutes still haven't passed since its last update.
A few things/limitations to consider:
- The repository must be accessible over `http://`, `https://`, `ssh://` or `git://`.
- If your HTTP repository is not publicly accessible, add authentication
information to the URL, like: `https://username@gitlab.company.com/group/project.git`.
In some cases, you might need to use a personal access token instead of a
password, e.g., you want to mirror to GitHub and have 2FA enabled.
- The import will time out after 15 minutes. For repositories that take longer
use a clone/push combination.
- The Git LFS objects will not be synced. You'll need to push/pull them
manually.
## Use-case
- You have old projects in another source that you don't use actively anymore,
but don't want to remove for archiving purposes. In that case, you can create
a push mirror so that your active GitLab repository can push its changes to the
old location.
## Pushing to a remote repository **[STARTER]**
>[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in
GitLab Enterprise Edition 8.7. [Moved to GitLab Community Edition][ce-18715] in 10.8.
For an existing project, you can set up push mirror from your project's
**Settings ➔ Repository** and searching for the "Push to a remote repository"
section. Check the "Remote mirror repository" box and fill in the Git URL of
the repository to push to. Click **Save changes** for the changes to take
effect.
![Push settings](repository_mirroring/repository_mirroring_push_settings.png)
When push mirroring is enabled, you are advised not to push commits directly
to the mirrored repository to prevent the mirror diverging.
All changes will end up in the mirrored repository whenever commits
are pushed to GitLab, or when a [forced update](#forcing-an-update) is
initiated.
Pushes into GitLab are automatically pushed to the remote mirror at least once
every 5 minutes after they are received or once every minute if **push only
protected branches** is enabled.
In case of a diverged branch, you will see an error indicated at the **Mirror
repository** settings.
![Diverged branch](
repository_mirroring/repository_mirroring_diverged_branch_push.png)
### Push only protected branches
>[Introduced][ee-3350] in GitLab Enterprise Edition 10.3. [Moved to GitLab Community Edition][ce-18715] in 10.8.
You can choose to only push your protected branches from GitLab to your remote repository.
To use this option go to your project's repository settings page under push mirror.
## Setting up a push mirror from GitLab to GitHub
To set up a mirror from GitLab to GitHub, you need to follow these steps:
1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked:
![edit personal access token GitHub](repository_mirroring/repository_mirroring_github_edit_personal_access_token.png)
1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`:
![push to remote repo](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png)
1. Save
1. And either wait or trigger the "Update Now" button:
![update now](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png)
## Forcing an update
While mirrors are scheduled to update automatically, you can always force an update
by using the **Update now** button which is exposed in various places:
- in the commits page
- in the branches page
- in the tags page
- in the **Mirror repository** settings page
[ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350
[ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715
[perms]: ../user/permissions.md
......@@ -106,6 +106,7 @@ excluded_attributes:
- :last_repository_updated_at
- :last_repository_check_at
- :storage_version
- :remote_mirror_available_overridden
- :description_html
snippets:
- :expired_at
......
......@@ -71,6 +71,7 @@ module Gitlab
projects_imported_from_github: Project.where(import_type: 'github').count,
protected_branches: ProtectedBranch.count,
releases: Release.count,
remote_mirrors: RemoteMirror.count,
snippets: Snippet.count,
todos: Todo.count,
uploads: Upload.count,
......
require 'spec_helper'
describe Projects::MirrorsController do
include ReactiveCachingHelpers
describe 'setting up a remote mirror' do
set(:project) { create(:project, :repository) }
context 'when the current project is not a mirror' do
it 'allows to create a remote mirror' do
sign_in(project.owner)
expect do
do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
end.to change { RemoteMirror.count }.to(1)
end
end
end
describe '#update' do
let(:project) { create(:project, :repository, :remote_mirror) }
before do
sign_in(project.owner)
end
around do |example|
Sidekiq::Testing.fake! { example.run }
end
context 'With valid URL for a push' do
let(:remote_mirror_attributes) do
{ "0" => { "enabled" => "0", url: 'https://updated.example.com' } }
end
it 'processes a successful update' do
do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
expect(response).to redirect_to(project_settings_repository_path(project))
expect(flash[:notice]).to match(/successfully updated/)
end
it 'should create a RemoteMirror object' do
expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.to change(RemoteMirror, :count).by(1)
end
end
context 'With invalid URL for a push' do
let(:remote_mirror_attributes) do
{ "0" => { "enabled" => "0", url: 'ftp://invalid.invalid' } }
end
it 'processes an unsuccessful update' do
do_put(project, remote_mirrors_attributes: remote_mirror_attributes)
expect(response).to redirect_to(project_settings_repository_path(project))
expect(flash[:alert]).to match(/must be a valid URL/)
end
it 'should not create a RemoteMirror object' do
expect { do_put(project, remote_mirrors_attributes: remote_mirror_attributes) }.not_to change(RemoteMirror, :count)
end
end
end
def do_put(project, options, extra_attrs = {})
attrs = extra_attrs.merge(namespace_id: project.namespace.to_param, project_id: project.to_param)
attrs[:project] = options
put :update, attrs
end
end
......@@ -183,6 +183,17 @@ FactoryBot.define do
end
end
trait :remote_mirror do
transient do
remote_name "remote_mirror_#{SecureRandom.hex}"
url "http://foo.com"
enabled true
end
after(:create) do |project, evaluator|
project.remote_mirrors.create!(url: evaluator.url, enabled: evaluator.enabled)
end
end
trait :stubbed_repository do
after(:build) do |project|
allow(project).to receive(:empty_repo?).and_return(false)
......
FactoryBot.define do
factory :remote_mirror, class: 'RemoteMirror' do
association :project, :repository
url "http://foo:bar@test.com"
end
end
require 'spec_helper'
feature 'Project remote mirror', :feature do
let(:project) { create(:project, :repository, :remote_mirror) }
let(:remote_mirror) { project.remote_mirrors.first }
let(:user) { create(:user) }
describe 'On a project', :js do
before do
project.add_master(user)
sign_in user
end
context 'when last_error is present but last_update_at is not' do
it 'renders error message without timstamp' do
remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: nil)
visit project_mirror_path(project)
expect(page).to have_content('The remote repository failed to update.')
end
end
context 'when last_error and last_update_at are present' do
it 'renders error message with timestamp' do
remote_mirror.update_attributes(last_error: 'Some new error', last_update_at: Time.now - 5.minutes)
visit project_mirror_path(project)
expect(page).to have_content('The remote repository failed to update 5 minutes ago.')
end
end
end
end
......@@ -115,5 +115,20 @@ describe 'Projects > Settings > Repository settings' do
expect(page).to have_content('Your new project deploy token has been created')
end
end
context 'remote mirror settings' do
let(:user2) { create(:user) }
before do
project.add_master(user2)
visit project_settings_repository_path(project)
end
it 'shows push mirror settings' do
expect(page).to have_selector('#project_remote_mirrors_attributes_0_enabled')
expect(page).to have_selector('#project_remote_mirrors_attributes_0_url')
end
end
end
end
......@@ -268,6 +268,7 @@ project:
- pages_domains
- authorized_users
- project_authorizations
- remote_mirrors
- route
- redirect_routes
- statistics
......
......@@ -96,6 +96,7 @@ describe Gitlab::UsageData do
pages_domains
protected_branches
releases
remote_mirrors
snippets
todos
uploads
......
......@@ -1852,6 +1852,85 @@ describe Project do
it { expect(project.gitea_import?).to be true }
end
describe '#has_remote_mirror?' do
let(:project) { create(:project, :remote_mirror, :import_started) }
subject { project.has_remote_mirror? }
before do
allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
end
it 'returns true when a remote mirror is enabled' do
is_expected.to be_truthy
end
it 'returns false when remote mirror is disabled' do
project.remote_mirrors.first.update_attributes(enabled: false)
is_expected.to be_falsy
end
end
describe '#update_remote_mirrors' do
let(:project) { create(:project, :remote_mirror, :import_started) }
delegate :update_remote_mirrors, to: :project
before do
allow_any_instance_of(RemoteMirror).to receive(:refresh_remote)
end
it 'syncs enabled remote mirror' do
expect_any_instance_of(RemoteMirror).to receive(:sync)
update_remote_mirrors
end
# TODO: study if remote_mirror_available_overridden is still a necessary attribute considering that
# it is no longer under any license
it 'does nothing when remote mirror is disabled globally and not overridden' do
stub_application_setting(mirror_available: false)
project.remote_mirror_available_overridden = false
expect_any_instance_of(RemoteMirror).not_to receive(:sync)
update_remote_mirrors
end
it 'does not sync disabled remote mirrors' do
project.remote_mirrors.first.update_attributes(enabled: false)
expect_any_instance_of(RemoteMirror).not_to receive(:sync)
update_remote_mirrors
end
end
describe '#remote_mirror_available?' do
let(:project) { create(:project) }
context 'when remote mirror global setting is enabled' do
it 'returns true' do
expect(project.remote_mirror_available?).to be(true)
end
end
context 'when remote mirror global setting is disabled' do
before do
stub_application_setting(mirror_available: false)
end
it 'returns true when overridden' do
project.remote_mirror_available_overridden = true
expect(project.remote_mirror_available?).to be(true)
end
it 'returns false when not overridden' do
expect(project.remote_mirror_available?).to be(false)
end
end
end
describe '#ancestors_upto', :nested_groups do
let(:parent) { create(:group) }
let(:child) { create(:group, parent: parent) }
......
require 'rails_helper'
describe RemoteMirror do
describe 'URL validation' do
context 'with a valid URL' do
it 'should be valid' do
remote_mirror = build(:remote_mirror)
expect(remote_mirror).to be_valid
end
end
context 'with an invalid URL' do
it 'should not be valid' do
remote_mirror = build(:remote_mirror, url: 'ftp://invalid.invalid')
expect(remote_mirror).not_to be_valid
expect(remote_mirror.errors[:url].size).to eq(2)
end
end
end
describe 'encrypting credentials' do
context 'when setting URL for a first time' do
it 'stores the URL without credentials' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
expect(mirror.read_attribute(:url)).to eq('http://test.com')
end
it 'stores the credentials on a separate field' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
end
it 'handles credentials with large content' do
mirror = create_mirror(url: 'http://bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif:9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75@test.com')
expect(mirror.credentials).to eq({
user: 'bxnhm8dote33ct932r3xavslj81wxmr7o8yux8do10oozckkif',
password: '9ne7fuvjn40qjt35dgt8v86q9m9g9essryxj76sumg2ccl2fg26c0krtz2gzfpyq4hf22h328uhq6npuiq6h53tpagtsj7vsrz75'
})
end
end
context 'when updating the URL' do
it 'allows a new URL without credentials' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
mirror.update_attribute(:url, 'http://test.com')
expect(mirror.url).to eq('http://test.com')
expect(mirror.credentials).to eq({ user: nil, password: nil })
end
it 'allows a new URL with credentials' do
mirror = create_mirror(url: 'http://test.com')
mirror.update_attribute(:url, 'http://foo:bar@test.com')
expect(mirror.url).to eq('http://foo:bar@test.com')
expect(mirror.credentials).to eq({ user: 'foo', password: 'bar' })
end
it 'updates the remote config if credentials changed' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
repo = mirror.project.repository
mirror.update_attribute(:url, 'http://foo:baz@test.com')
config = repo.raw_repository.rugged.config
expect(config["remote.#{mirror.remote_name}.url"]).to eq('http://foo:baz@test.com')
end
it 'removes previous remote' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
mirror.update_attributes(url: 'http://test.com')
end
end
end
describe '#remote_name' do
context 'when remote name is persisted in the database' do
it 'returns remote name with random value' do
allow(SecureRandom).to receive(:hex).and_return('secret')
remote_mirror = create(:remote_mirror)
expect(remote_mirror.remote_name).to eq("remote_mirror_secret")
end
end
context 'when remote name is not persisted in the database' do
it 'returns remote name with remote mirror id' do
remote_mirror = create(:remote_mirror)
remote_mirror.remote_name = nil
expect(remote_mirror.remote_name).to eq("remote_mirror_#{remote_mirror.id}")
end
end
context 'when remote is not persisted in the database' do
it 'returns nil' do
remote_mirror = build(:remote_mirror, remote_name: nil)
expect(remote_mirror.remote_name).to be_nil
end
end
end
describe '#safe_url' do
context 'when URL contains credentials' do
it 'masks the credentials' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
expect(mirror.safe_url).to eq('http://*****:*****@test.com')
end
end
context 'when URL does not contain credentials' do
it 'shows the full URL' do
mirror = create_mirror(url: 'http://test.com')
expect(mirror.safe_url).to eq('http://test.com')
end
end
end
context 'when remote mirror gets destroyed' do
it 'removes remote' do
mirror = create_mirror(url: 'http://foo:bar@test.com')
expect(RepositoryRemoveRemoteWorker).to receive(:perform_async).with(mirror.project.id, mirror.remote_name).and_call_original
mirror.destroy!
end
end
context 'stuck mirrors' do
it 'includes mirrors stuck in started with no last_update_at set' do
mirror = create_mirror(url: 'http://cantbeblank',
update_status: 'started',
last_update_at: nil,
updated_at: 25.hours.ago)
expect(described_class.stuck.last).to eq(mirror)
end
end
context '#sync' do
let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
around do |example|
Timecop.freeze { example.run }
end
context 'with remote mirroring disabled' do
it 'returns nil' do
remote_mirror.update_attributes(enabled: false)
expect(remote_mirror.sync).to be_nil
end
end
context 'with remote mirroring enabled' do
context 'with only protected branches enabled' do
context 'when it did not update in the last minute' do
it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
remote_mirror.sync
end
end
context 'when it did update in the last minute' do
it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next minute' do
remote_mirror.last_update_started_at = Time.now - 30.seconds
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::PROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
remote_mirror.sync
end
end
end
context 'with only protected branches disabled' do
before do
remote_mirror.only_protected_branches = false
end
context 'when it did not update in the last 5 minutes' do
it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
remote_mirror.sync
end
end
context 'when it did update within the last 5 minutes' do
it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next 5 minutes' do
remote_mirror.last_update_started_at = Time.now - 30.seconds
expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::UNPROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
remote_mirror.sync
end
end
end
end
end
context '#updated_since?' do
let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
let(:timestamp) { Time.now - 5.minutes }
around do |example|
Timecop.freeze { example.run }
end
before do
remote_mirror.update_attributes(last_update_started_at: Time.now)
end
context 'when remote mirror does not have status failed' do
it 'returns true when last update started after the timestamp' do
expect(remote_mirror.updated_since?(timestamp)).to be true
end
it 'returns false when last update started before the timestamp' do
expect(remote_mirror.updated_since?(Time.now + 5.minutes)).to be false
end
end
context 'when remote mirror has status failed' do
it 'returns false when last update started after the timestamp' do
remote_mirror.update_attributes(update_status: 'failed')
expect(remote_mirror.updated_since?(timestamp)).to be false
end
end
end
context 'no project' do
it 'includes mirror with a project in pending_delete' do
mirror = create_mirror(url: 'http://cantbeblank',
update_status: 'finished',
enabled: true,
last_update_at: nil,
updated_at: 25.hours.ago)
project = mirror.project
project.pending_delete = true
project.save
mirror.reload
expect(mirror.sync).to be_nil
expect(mirror.valid?).to be_truthy
expect(mirror.update_status).to eq('finished')
end
end
def create_mirror(params)
project = FactoryBot.create(:project, :repository)
project.remote_mirrors.create!(params)
end
end
......@@ -758,6 +758,38 @@ describe Repository do
end
end
describe '#async_remove_remote' do
before do
masterrev = repository.find_branch('master').dereferenced_target
create_remote_branch('joe', 'remote_branch', masterrev)
end
context 'when worker is scheduled successfully' do
before do
masterrev = repository.find_branch('master').dereferenced_target
create_remote_branch('remote_name', 'remote_branch', masterrev)
allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return('1234')
end
it 'returns job_id' do
expect(repository.async_remove_remote('joe')).to eq('1234')
end
end
context 'when worker does not schedule successfully' do
before do
allow(RepositoryRemoveRemoteWorker).to receive(:perform_async).and_return(nil)
end
it 'returns nil' do
expect(Rails.logger).to receive(:info).with("Remove remote job failed to create for #{project.id} with remote name joe.")
expect(repository.async_remove_remote('joe')).to be_nil
end
end
end
describe '#fetch_ref' do
let(:broken_repository) { create(:project, :broken_storage).repository }
......@@ -2338,6 +2370,11 @@ describe Repository do
end
end
def create_remote_branch(remote_name, branch_name, target)
rugged = repository.rugged
rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
end
describe '#ancestor?' do
let(:commit) { repository.commit }
let(:ancestor) { commit.parents.first }
......
......@@ -14,6 +14,72 @@ describe GitPushService, services: true do
project.add_master(user)
end
describe 'with remote mirrors' do
let(:project) { create(:project, :repository, :remote_mirror) }
subject do
described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref)
end
context 'when remote mirror feature is enabled' do
it 'fails stuck remote mirrors' do
allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
subject.execute
end
it 'updates remote mirrors' do
expect(project).to receive(:update_remote_mirrors)
subject.execute
end
end
context 'when remote mirror feature is disabled' do
before do
stub_application_setting(mirror_available: false)
end
context 'with remote mirrors global setting overridden' do
before do
project.remote_mirror_available_overridden = true
end
it 'fails stuck remote mirrors' do
allow(project).to receive(:update_remote_mirrors).and_return(project.remote_mirrors)
expect(project).to receive(:mark_stuck_remote_mirrors_as_failed!)
subject.execute
end
it 'updates remote mirrors' do
expect(project).to receive(:update_remote_mirrors)
subject.execute
end
end
context 'without remote mirrors global setting overridden' do
before do
project.remote_mirror_available_overridden = false
end
it 'does not fails stuck remote mirrors' do
expect(project).not_to receive(:mark_stuck_remote_mirrors_as_failed!)
subject.execute
end
it 'does not updates remote mirrors' do
expect(project).not_to receive(:update_remote_mirrors)
subject.execute
end
end
end
end
describe 'Push branches' do
subject do
execute_service(project, user, oldrev, newrev, ref)
......
......@@ -65,6 +65,19 @@ describe Projects::DestroyService do
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
end
context 'when has remote mirrors' do
let!(:project) do
create(:project, :repository, namespace: user.namespace).tap do |project|
project.remote_mirrors.create(url: 'http://test.com')
end
end
let!(:async) { true }
it 'destroys them' do
expect(RemoteMirror.count).to eq(0)
end
end
it_behaves_like 'deleting the project'
it 'invalidates personal_project_count cache' do
......
require 'spec_helper'
describe Projects::UpdateRemoteMirrorService do
let(:project) { create(:project, :repository) }
let(:remote_project) { create(:forked_project_with_submodules) }
let(:repository) { project.repository }
let(:raw_repository) { repository.raw }
let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo, enabled: true, only_protected_branches: false) }
subject { described_class.new(project, project.creator) }
describe "#execute", :skip_gitaly_mock do
before do
create_branch(repository, 'existing-branch')
allow(raw_repository).to receive(:remote_tags) do
generate_tags(repository, 'v1.0.0', 'v1.1.0')
end
allow(raw_repository).to receive(:push_remote_branches).and_return(true)
end
it "fetches the remote repository" do
expect(repository).to receive(:fetch_remote).with(remote_mirror.remote_name, no_tags: true) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
end
subject.execute(remote_mirror)
end
it "succeeds" do
allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
result = subject.execute(remote_mirror)
expect(result[:status]).to eq(:success)
end
describe 'Syncing branches' do
it "push all the branches the first time" do
allow(repository).to receive(:fetch_remote)
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, local_branch_names)
subject.execute(remote_mirror)
end
it "does not push anything is remote is up to date" do
allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
expect(raw_repository).not_to receive(:push_remote_branches)
subject.execute(remote_mirror)
end
it "sync new branches" do
# call local_branch_names early so it is not called after the new branch has been created
current_branches = local_branch_names
allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, current_branches) }
create_branch(repository, 'my-new-branch')
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['my-new-branch'])
subject.execute(remote_mirror)
end
it "sync updated branches" do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
update_branch(repository, 'existing-branch')
end
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
subject.execute(remote_mirror)
end
context 'when push only protected branches option is set' do
let(:unprotected_branch_name) { 'existing-branch' }
let(:protected_branch_name) do
project.repository.branch_names.find { |n| n != unprotected_branch_name }
end
let!(:protected_branch) do
create(:protected_branch, project: project, name: protected_branch_name)
end
before do
project.reload
remote_mirror.only_protected_branches = true
end
it "sync updated protected branches" do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
update_branch(repository, protected_branch_name)
end
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
subject.execute(remote_mirror)
end
it 'does not sync unprotected branches' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
update_branch(repository, unprotected_branch_name)
end
expect(raw_repository).not_to receive(:push_remote_branches).with(remote_mirror.remote_name, [unprotected_branch_name])
subject.execute(remote_mirror)
end
end
context 'when branch exists in local and remote repo' do
context 'when it has diverged' do
it 'syncs branches' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
update_remote_branch(repository, remote_mirror.remote_name, 'markdown')
end
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['markdown'])
subject.execute(remote_mirror)
end
end
end
describe 'for delete' do
context 'when branch exists in local and remote repo' do
it 'deletes the branch from remote repo' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
delete_branch(repository, 'existing-branch')
end
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
subject.execute(remote_mirror)
end
end
context 'when push only protected branches option is set' do
before do
remote_mirror.only_protected_branches = true
end
context 'when branch exists in local and remote repo' do
let!(:protected_branch_name) { local_branch_names.first }
before do
create(:protected_branch, project: project, name: protected_branch_name)
project.reload
end
it 'deletes the protected branch from remote repo' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
delete_branch(repository, protected_branch_name)
end
expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
subject.execute(remote_mirror)
end
it 'does not delete the unprotected branch from remote repo' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
delete_branch(repository, 'existing-branch')
end
expect(raw_repository).not_to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['existing-branch'])
subject.execute(remote_mirror)
end
end
context 'when branch only exists on remote repo' do
let!(:protected_branch_name) { 'remote-branch' }
before do
create(:protected_branch, project: project, name: protected_branch_name)
end
context 'when it has diverged' do
it 'does not delete the remote branch' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
rev = repository.find_branch('markdown').dereferenced_target
create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id)
end
expect(raw_repository).not_to receive(:delete_remote_branches)
subject.execute(remote_mirror)
end
end
context 'when it has not diverged' do
it 'deletes the remote branch' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
masterrev = repository.find_branch('master').dereferenced_target
create_remote_branch(repository, remote_mirror.remote_name, protected_branch_name, masterrev.id)
end
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, [protected_branch_name])
subject.execute(remote_mirror)
end
end
end
end
context 'when branch only exists on remote repo' do
context 'when it has diverged' do
it 'does not delete the remote branch' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
rev = repository.find_branch('markdown').dereferenced_target
create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', rev.id)
end
expect(raw_repository).not_to receive(:delete_remote_branches)
subject.execute(remote_mirror)
end
end
context 'when it has not diverged' do
it 'deletes the remote branch' do
allow(repository).to receive(:fetch_remote) do
sync_remote(repository, remote_mirror.remote_name, local_branch_names)
masterrev = repository.find_branch('master').dereferenced_target
create_remote_branch(repository, remote_mirror.remote_name, 'remote-branch', masterrev.id)
end
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['remote-branch'])
subject.execute(remote_mirror)
end
end
end
end
end
describe 'Syncing tags' do
before do
allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.remote_name, local_branch_names) }
end
context 'when there are not tags to push' do
it 'does not try to push tags' do
allow(repository).to receive(:remote_tags) { {} }
allow(repository).to receive(:tags) { [] }
expect(repository).not_to receive(:push_tags)
subject.execute(remote_mirror)
end
end
context 'when there are some tags to push' do
it 'pushes tags to remote' do
allow(raw_repository).to receive(:remote_tags) { {} }
expect(raw_repository).to receive(:push_remote_branches).with(remote_mirror.remote_name, ['v1.0.0', 'v1.1.0'])
subject.execute(remote_mirror)
end
end
context 'when there are some tags to delete' do
it 'deletes tags from remote' do
remote_tags = generate_tags(repository, 'v1.0.0', 'v1.1.0')
allow(raw_repository).to receive(:remote_tags) { remote_tags }
repository.rm_tag(create(:user), 'v1.0.0')
expect(raw_repository).to receive(:delete_remote_branches).with(remote_mirror.remote_name, ['v1.0.0'])
subject.execute(remote_mirror)
end
end
end
end
def create_branch(repository, branch_name)
rugged = repository.rugged
masterrev = repository.find_branch('master').dereferenced_target
parentrev = repository.commit(masterrev).parent_id
rugged.references.create("refs/heads/#{branch_name}", parentrev)
repository.expire_branches_cache
end
def create_remote_branch(repository, remote_name, branch_name, source_id)
rugged = repository.rugged
rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_id)
end
def sync_remote(repository, remote_name, local_branch_names)
rugged = repository.rugged
local_branch_names.each do |branch|
target = repository.find_branch(branch).try(:dereferenced_target)
rugged.references.create("refs/remotes/#{remote_name}/#{branch}", target.id) if target
end
end
def update_remote_branch(repository, remote_name, branch)
rugged = repository.rugged
masterrev = repository.find_branch('master').dereferenced_target.id
rugged.references.create("refs/remotes/#{remote_name}/#{branch}", masterrev, force: true)
repository.expire_branches_cache
end
def update_branch(repository, branch)
rugged = repository.rugged
masterrev = repository.find_branch('master').dereferenced_target.id
# Updated existing branch
rugged.references.create("refs/heads/#{branch}", masterrev, force: true)
repository.expire_branches_cache
end
def delete_branch(repository, branch)
rugged = repository.rugged
rugged.references.delete("refs/heads/#{branch}")
repository.expire_branches_cache
end
def generate_tags(repository, *tag_names)
tag_names.each_with_object([]) do |name, tags|
tag = repository.find_tag(name)
target = tag.try(:target)
target_commit = tag.try(:dereferenced_target)
tags << Gitlab::Git::Tag.new(repository.raw_repository, name, target, target_commit)
end
end
def local_branch_names
branch_names = repository.branches.map(&:name)
# we want the protected branch to be pushed first
branch_names.unshift(branch_names.delete('master'))
end
end
require 'rails_helper'
describe RepositoryRemoveRemoteWorker do
subject(:worker) { described_class.new }
describe '#perform' do
let(:remote_name) { 'joe'}
let!(:project) { create(:project, :repository) }
context 'when it cannot obtain lease' do
it 'logs error' do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain) { nil }
expect_any_instance_of(Repository).not_to receive(:remove_remote)
expect(worker).to receive(:log_error).with('Cannot obtain an exclusive lease. There must be another instance already in execution.')
worker.perform(project.id, remote_name)
end
end
context 'when it gets the lease' do
before do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
end
context 'when project does not exist' do
it 'returns nil' do
expect(worker.perform(-1, 'remote_name')).to be_nil
end
end
context 'when project exists' do
it 'removes remote from repository' do
masterrev = project.repository.find_branch('master').dereferenced_target
create_remote_branch(remote_name, 'remote_branch', masterrev)
expect_any_instance_of(Repository).to receive(:remove_remote).with(remote_name).and_call_original
worker.perform(project.id, remote_name)
end
end
end
end
def create_remote_branch(remote_name, branch_name, target)
rugged = project.repository.rugged
rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
end
end
require 'rails_helper'
describe RepositoryUpdateRemoteMirrorWorker do
subject { described_class.new }
let(:remote_mirror) { create(:project, :repository, :remote_mirror).remote_mirrors.first }
let(:scheduled_time) { Time.now - 5.minutes }
around do |example|
Timecop.freeze(Time.now) { example.run }
end
describe '#perform' do
context 'with status none' do
before do
remote_mirror.update_attributes(update_status: 'none')
end
it 'sets status as finished when update remote mirror service executes successfully' do
expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
expect { subject.perform(remote_mirror.id, Time.now) }.to change { remote_mirror.reload.update_status }.to('finished')
end
it 'sets status as failed when update remote mirror service executes with errors' do
error_message = 'fail!'
expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :error, message: error_message)
expect do
subject.perform(remote_mirror.id, Time.now)
end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError, error_message)
expect(remote_mirror.reload.update_status).to eq('failed')
end
it 'does nothing if last_update_started_at is higher than the time the job was scheduled in' do
remote_mirror.update_attributes(last_update_started_at: Time.now)
expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(true)
expect_any_instance_of(Projects::UpdateRemoteMirrorService).not_to receive(:execute).with(remote_mirror)
expect(subject.perform(remote_mirror.id, scheduled_time)).to be_nil
end
end
context 'with unexpected error' do
it 'marks mirror as failed' do
allow_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_raise(RuntimeError)
expect do
subject.perform(remote_mirror.id, Time.now)
end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateError)
expect(remote_mirror.reload.update_status).to eq('failed')
end
end
context 'with another worker already running' do
before do
remote_mirror.update_attributes(update_status: 'started')
end
it 'raises RemoteMirrorUpdateAlreadyInProgressError' do
expect do
subject.perform(remote_mirror.id, Time.now)
end.to raise_error(RepositoryUpdateRemoteMirrorWorker::UpdateAlreadyInProgressError)
end
end
context 'with status failed' do
before do
remote_mirror.update_attributes(update_status: 'failed')
end
it 'sets status as finished if last_update_started_at is higher than the time the job was scheduled in' do
remote_mirror.update_attributes(last_update_started_at: Time.now)
expect_any_instance_of(RemoteMirror).to receive(:updated_since?).with(scheduled_time).and_return(false)
expect_any_instance_of(Projects::UpdateRemoteMirrorService).to receive(:execute).with(remote_mirror).and_return(status: :success)
expect { subject.perform(remote_mirror.id, scheduled_time) }.to change { remote_mirror.reload.update_status }.to('finished')
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment