Commit 961255b1 authored by Tiago Botelho's avatar Tiago Botelho

Adds remote mirror table migration

parent 9a130593
......@@ -2,8 +2,9 @@ class Projects::MirrorsController < Projects::ApplicationController
include RepositorySettingsRedirect
# Authorize
before_action :authorize_admin_mirror!
before_action :remote_mirror, only: [:update]
before_action :check_mirror_available!
before_action :authorize_admin_project!
layout "project_settings"
......@@ -45,6 +46,10 @@ class Projects::MirrorsController < Projects::ApplicationController
@remote_mirror = project.remote_mirrors.first_or_initialize
end
def check_mirror_available!
Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
end
def mirror_params_attributes
[
remote_mirrors_attributes: %i[
......
......@@ -44,8 +44,6 @@ module Projects
end
def remote_mirror
return unless project.feature_available?(:repository_mirrors)
@remote_mirror = project.remote_mirrors.first_or_initialize
end
......
......@@ -250,7 +250,8 @@ module ApplicationSettingsHelper
:version_check_enabled,
:allow_local_requests_from_hooks_and_services,
:enforce_terms,
:terms
:terms,
:mirror_available
]
end
end
......@@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base
gitaly_timeout_fast: 10,
gitaly_timeout_medium: 30,
gitaly_timeout_default: 55,
allow_local_requests_from_hooks_and_services: false
allow_local_requests_from_hooks_and_services: false,
mirror_available: true
}
end
......
......@@ -65,7 +65,7 @@ class Project < ActiveRecord::Base
add_authentication_token_field :runners_token
before_validation :mark_remote_mirrors_for_removal
before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) }
before_save :ensure_runners_token
......
......@@ -86,9 +86,12 @@ class RemoteMirror < ActiveRecord::Base
raw.update(options)
end
def sync?
!enabled?
end
def sync
return unless enabled?
return if Gitlab::Geo.secondary?
return if sync?
if recently_scheduled?
RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
......
......@@ -854,7 +854,7 @@ class Repository
add_remote(remote_name, url, mirror_refmap: refmap)
fetch_remote(remote_name, forced: forced, prune: prune)
ensure
remove_remote(remote_name) if tmp_remote_name
async_remove_remote(remote_name) if tmp_remote_name
end
def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true)
......
......@@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy
project.merge_requests_allowing_push_to_user(user).any?
end
with_scope :global
condition(:mirror_available, score: 0) do
::Gitlab::CurrentSettings.current_application_settings.mirror_available
end
# We aren't checking `:read_issue` or `:read_merge_request` in this case
# because it could be possible for a user to see an issuable-iid
# (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be
......@@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy
enable :create_cluster
end
rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
rule { archived }.policy do
prevent :push_code
prevent :push_to_delete_protected_branch
......
class ProjectMirrorEntity < Grape::Entity
prepend ::EE::ProjectMirrorEntity
expose :id
expose :remote_mirrors_attributes do |project|
......
#
# Concern that helps with getting an exclusive lease for running a block
# of code.
#
# `#try_obtain_lease` takes a block which will be run if it was able to
# obtain the lease. Implement `#lease_timeout` to configure the timeout
# for the exclusive lease. Optionally override `#lease_key` to set the
# lease key, it defaults to the class name with underscores.
#
module ExclusiveLeaseGuard
extend ActiveSupport::Concern
def try_obtain_lease
lease = exclusive_lease.try_obtain
unless lease
log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.')
return
end
begin
yield lease
ensure
release_lease(lease)
end
end
def exclusive_lease
@lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout)
end
def lease_key
@lease_key ||= self.class.name.underscore
end
def lease_timeout
raise NotImplementedError,
"#{self.class.name} does not implement #{__method__}"
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
def renew_lease!
exclusive_lease.renew
end
def log_error(message, extra_args = {})
logger.error(message)
end
end
= form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f|
= form_errors(@application_setting)
%fieldset
.form-group
= f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2'
.col-sm-10
.checkbox
= f.label :mirror_available do
= f.check_box :mirror_available
Allow mirrors to be setup for projects
%span.help-block
If disabled, only admins will be able to setup mirrors in projects.
= link_to icon('question-circle'), help_page_path('workflow/repository_mirroring')
= f.submit 'Save changes', class: "btn btn-success"
......@@ -313,3 +313,14 @@
= _('Allow requests to the local network from hooks and services.')
.settings-content
= render 'outbound'
%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) }
.settings-header
%h4
= _('Repository mirror settings')
%button.btn.js-settings-toggle{ type: 'button' }
= expanded ? 'Collapse' : 'Expand'
%p
= _('Configure push and pull mirrors.')
.settings-content
= render partial: 'repository_mirrors_form'
.account-well.prepend-top-default.append-bottom-default
%ul
%li
The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>.
%li
Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>.
%li
The update action will time out after 10 minutes. For big repositories, use a clone/push combination.
%li
The Git LFS objects will <strong>not</strong> be synced.
- if can?(current_user, :admin_mirror, @project)
- if can?(current_user, :admin_remote_mirror, @project)
= render 'projects/mirrors/push'
......@@ -2,6 +2,8 @@
- page_title "Repository"
- @content_class = "limit-container-width" unless fluid_layout
= render "projects/mirrors/show"
-# Protected branches & tags use a lot of nested partials.
-# The shared parts of the views can be found in the `shared` directory.
-# Those are used throughout the actual views. These `shared` views are then
......
......@@ -106,6 +106,7 @@
- rebase
- repository_fork
- repository_import
- repository_remove_remote
- storage_migrator
- system_hook_push
- update_merge_requests
......
......@@ -174,6 +174,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
end
end
resource :mirror, only: [:show, :update] do
member do
post :update_now
end
end
resources :pipelines, only: [:index, :new, :create, :show] do
collection do
resource :pipelines_settings, path: 'settings', only: [:show, :update]
......
class CreateRemoteMirrors < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
return if table_exists?(:remote_mirrors)
create_table :remote_mirrors do |t|
t.references :project, index: true, foreign_key: { on_delete: :cascade }
t.string :url
t.boolean :enabled, default: true
t.string :update_status
t.datetime :last_update_at
t.datetime :last_successful_update_at
t.datetime :last_update_started_at
t.string :last_error
t.boolean :only_protected_branches, default: false, null: false
t.string :remote_name
t.text :encrypted_credentials
t.string :encrypted_credentials_iv
t.string :encrypted_credentials_salt
t.timestamps null: false
end
end
def down
drop_table(:remote_mirrors) if table_exists?(:remote_mirrors)
end
end
class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden)
end
def down
remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden)
end
end
class AddIndexesToRemoteMirror < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at)
end
def down
remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at
end
end
class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available)
end
def down
remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available)
end
end
......@@ -165,6 +165,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.boolean "pages_domain_verification_enabled", default: true, null: false
t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
t.boolean "enforce_terms", default: false
t.boolean "mirror_available", default: true, null: false
end
create_table "audit_events", force: :cascade do |t|
......@@ -314,10 +315,10 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.integer "auto_canceled_by_id"
t.boolean "retried"
t.integer "stage_id"
t.boolean "protected"
t.integer "failure_reason"
t.integer "artifacts_file_store"
t.integer "artifacts_metadata_store"
t.boolean "protected"
t.integer "failure_reason"
end
add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
......@@ -365,13 +366,13 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.integer "project_id", null: false
t.integer "job_id", null: false
t.integer "file_type", null: false
t.integer "file_store"
t.integer "size", limit: 8
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.datetime_with_timezone "expire_at"
t.string "file"
t.binary "file_sha256"
t.integer "file_store"
end
add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
......@@ -1593,6 +1594,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.boolean "merge_requests_rebase_enabled", default: false, null: false
t.integer "jobs_cache_index"
t.boolean "pages_https_only", default: true
t.boolean "remote_mirror_available_overridden"
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......@@ -1698,6 +1700,27 @@ ActiveRecord::Schema.define(version: 20180503200320) do
add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree
add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree
create_table "remote_mirrors", force: :cascade do |t|
t.integer "project_id"
t.string "url"
t.boolean "enabled", default: true
t.string "update_status"
t.datetime "last_update_at"
t.datetime "last_successful_update_at"
t.datetime "last_update_started_at"
t.string "last_error"
t.boolean "only_protected_branches", default: false, null: false
t.string "remote_name"
t.text "encrypted_credentials"
t.string "encrypted_credentials_iv"
t.string "encrypted_credentials_salt"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end
add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
create_table "routes", force: :cascade do |t|
t.integer "source_id", null: false
t.string "source_type", null: false
......@@ -2233,6 +2256,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do
add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
add_foreign_key "remote_mirrors", "projects", on_delete: :cascade
add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
add_foreign_key "subscriptions", "projects", on_delete: :cascade
......
# Repository mirroring
Repository Mirroring is a way to mirror repositories from external sources.
It can be used to mirror all branches, tags, and commits that you have
in your repository.
Your mirror at GitLab will be updated automatically. You can
also manually trigger an update at most once every 5 minutes.
## Overview
Repository mirroring is very useful when, for some reason, you must use a
project from another source.
There are two kinds of repository mirroring features supported by GitLab:
**push** and **pull**, the latter being only available in GitLab Enterprise Edition.
The **push** method mirrors the repository in GitLab to another location.
Once the mirror repository is updated, all new branches,
tags, and commits will be visible in the project's activity feed.
Users with at least [developer access][perms] to the project can also force an
immediate update with the click of a button. This button will not be available if
the mirror is already being updated or 5 minutes still haven't passed since its last update.
A few things/limitations to consider:
- The repository must be accessible over `http://`, `https://`, `ssh://` or `git://`.
- If your HTTP repository is not publicly accessible, add authentication
information to the URL, like: `https://username@gitlab.company.com/group/project.git`.
In some cases, you might need to use a personal access token instead of a
password, e.g., you want to mirror to GitHub and have 2FA enabled.
- The import will time out after 15 minutes. For repositories that take longer
use a clone/push combination.
- The Git LFS objects will not be synced. You'll need to push/pull them
manually.
## Use-case
- You have old projects in another source that you don't use actively anymore,
but don't want to remove for archiving purposes. In that case, you can create
a push mirror so that your active GitLab repository can push its changes to the
old location.
## Pushing to a remote repository **[STARTER]**
>[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in
GitLab Enterprise Edition 8.7. [Moved to GitLab Community Edition][ce-18715] in 10.8.
For an existing project, you can set up push mirror from your project's
**Settings ➔ Repository** and searching for the "Push to a remote repository"
section. Check the "Remote mirror repository" box and fill in the Git URL of
the repository to push to. Click **Save changes** for the changes to take
effect.
![Push settings](repository_mirroring/repository_mirroring_push_settings.png)
When push mirroring is enabled, you are advised not to push commits directly
to the mirrored repository to prevent the mirror diverging.
All changes will end up in the mirrored repository whenever commits
are pushed to GitLab, or when a [forced update](#forcing-an-update) is
initiated.
Pushes into GitLab are automatically pushed to the remote mirror at least once
every 5 minutes after they are received or once every minute if **push only
protected branches** is enabled.
In case of a diverged branch, you will see an error indicated at the **Mirror
repository** settings.
![Diverged branch](
repository_mirroring/repository_mirroring_diverged_branch_push.png)
### Push only protected branches
>[Introduced][ee-3350] in GitLab Enterprise Edition 10.3. [Moved to GitLab Community Edition][ce-18715] in 10.8.
You can choose to only push your protected branches from GitLab to your remote repository.
To use this option go to your project's repository settings page under push mirror.
## Setting up a push mirror from GitLab to GitHub
To set up a mirror from GitLab to GitHub, you need to follow these steps:
1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked:
![edit personal access token GitHub](repository_mirroring/repository_mirroring_github_edit_personal_access_token.png)
1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`:
![push to remote repo](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository.png)
1. Save
1. And either wait or trigger the "Update Now" button:
![update now](repository_mirroring/repository_mirroring_gitlab_push_to_a_remote_repository_update_now.png)
## Forcing an update
While mirrors are scheduled to update automatically, you can always force an update
by using the **Update now** button which is exposed in various places:
- in the commits page
- in the branches page
- in the tags page
- in the **Mirror repository** settings page
[ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350
[ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715
[perms]: ../user/permissions.md
......@@ -164,14 +164,6 @@ describe RemoteMirror do
end
end
context 'as a Geo secondary' do
it 'returns nil' do
allow(Gitlab::Geo).to receive(:secondary?).and_return(true)
expect(remote_mirror.sync).to be_nil
end
end
context 'with remote mirroring enabled' do
context 'with only protected branches enabled' do
context 'when it did not update in the last minute' do
......
......@@ -2370,6 +2370,11 @@ describe Repository do
end
end
def create_remote_branch(remote_name, branch_name, target)
rugged = repository.rugged
rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id)
end
describe '#ancestor?' do
let(:commit) { repository.commit }
let(:ancestor) { commit.parents.first }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment