Commit a87989fb authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Remove satellites

parent 277f311f
...@@ -875,6 +875,3 @@ DEPENDENCIES ...@@ -875,6 +875,3 @@ DEPENDENCIES
virtus virtus
webmock (~> 1.21.0) webmock (~> 1.21.0)
wikicloth (= 0.8.1) wikicloth (= 0.8.1)
BUNDLED WITH
1.10.5
require 'gitlab/satellite/satellite'
class Projects::MergeRequestsController < Projects::ApplicationController class Projects::MergeRequestsController < Projects::ApplicationController
before_action :module_enabled before_action :module_enabled
before_action :merge_request, only: [ before_action :merge_request, only: [
......
...@@ -207,7 +207,7 @@ class MergeRequest < ActiveRecord::Base ...@@ -207,7 +207,7 @@ class MergeRequest < ActiveRecord::Base
def check_if_can_be_merged def check_if_can_be_merged
can_be_merged = can_be_merged =
if for_fork? if for_fork?
Gitlab::Satellite::MergeAction.new(self.author, self).can_be_merged? raise 'Implement me'
else else
project.repository.can_be_merged?(source_branch, target_branch) project.repository.can_be_merged?(source_branch, target_branch)
end end
...@@ -274,14 +274,14 @@ class MergeRequest < ActiveRecord::Base ...@@ -274,14 +274,14 @@ class MergeRequest < ActiveRecord::Base
# #
# see "git diff" # see "git diff"
def to_diff(current_user) def to_diff(current_user)
Gitlab::Satellite::MergeAction.new(current_user, self).diff_in_satellite raise 'Implement me'
end end
# Returns the commit as a series of email patches. # Returns the commit as a series of email patches.
# #
# see "git format-patch" # see "git format-patch"
def to_patch(current_user) def to_patch(current_user)
Gitlab::Satellite::MergeAction.new(current_user, self).format_patch raise 'Implement me'
end end
def hook_attrs def hook_attrs
......
...@@ -115,12 +115,11 @@ class Namespace < ActiveRecord::Base ...@@ -115,12 +115,11 @@ class Namespace < ActiveRecord::Base
def move_dir def move_dir
if gitlab_shell.mv_namespace(path_was, path) if gitlab_shell.mv_namespace(path_was, path)
# If repositories moved successfully we need to remove old satellites # If repositories moved successfully we need to
# and send update instructions to users. # send update instructions to users.
# However we cannot allow rollback since we moved namespace dir # However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions # So we basically we mute exceptions in next actions
begin begin
gitlab_shell.rm_satellites(path_was)
send_update_instructions send_update_instructions
rescue rescue
# Returning false does not rollback after_* transaction but gives # Returning false does not rollback after_* transaction but gives
......
...@@ -520,14 +520,6 @@ class Project < ActiveRecord::Base ...@@ -520,14 +520,6 @@ class Project < ActiveRecord::Base
!repository.exists? || repository.empty? !repository.exists? || repository.empty?
end end
def ensure_satellite_exists
self.satellite.create unless self.satellite.exists?
end
def satellite
@satellite ||= Gitlab::Satellite::Satellite.new(self)
end
def repo def repo
repository.raw repository.raw
end end
...@@ -597,14 +589,11 @@ class Project < ActiveRecord::Base ...@@ -597,14 +589,11 @@ class Project < ActiveRecord::Base
new_path_with_namespace = File.join(namespace_dir, path) new_path_with_namespace = File.join(namespace_dir, path)
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to remove old satellite # If repository moved successfully we need to send update instructions to users.
# and send update instructions to users.
# However we cannot allow rollback since we moved repository # However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions # So we basically we mute exceptions in next actions
begin begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
gitlab_shell.rm_satellites(old_path_with_namespace)
ensure_satellite_exists
send_move_instructions send_move_instructions
reset_events_cache reset_events_cache
rescue rescue
...@@ -698,7 +687,6 @@ class Project < ActiveRecord::Base ...@@ -698,7 +687,6 @@ class Project < ActiveRecord::Base
def create_repository def create_repository
if forked? if forked?
if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path) if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path)
ensure_satellite_exists
true true
else else
errors.add(:base, 'Failed to fork repository') errors.add(:base, 'Failed to fork repository')
......
...@@ -3,9 +3,6 @@ ...@@ -3,9 +3,6 @@
class CompareService class CompareService
def execute(current_user, source_project, source_branch, target_project, target_branch) def execute(current_user, source_project, source_branch, target_project, target_branch)
# Try to compare branches to get commits list and diffs # Try to compare branches to get commits list and diffs
#
# Note: Use satellite only when need to compare between two repos
# because satellites are slower than operations on bare repo
if target_project == source_project if target_project == source_project
Gitlab::CompareResult.new( Gitlab::CompareResult.new(
Gitlab::Git::Compare.new( Gitlab::Git::Compare.new(
...@@ -15,13 +12,7 @@ class CompareService ...@@ -15,13 +12,7 @@ class CompareService
) )
) )
else else
Gitlab::Satellite::CompareAction.new( raise 'Implement me'
current_user,
target_project,
target_branch,
source_project,
source_branch
).result
end end
end end
end end
...@@ -10,16 +10,14 @@ class GitPushService ...@@ -10,16 +10,14 @@ class GitPushService
# #
# Next, this method: # Next, this method:
# 1. Creates the push event # 1. Creates the push event
# 2. Ensures that the project satellite exists # 2. Updates merge requests
# 3. Updates merge requests # 3. Recognizes cross-references from commit messages
# 4. Recognizes cross-references from commit messages # 4. Executes the project's web hooks
# 5. Executes the project's web hooks # 5. Executes the project's services
# 6. Executes the project's services
# #
def execute(project, user, oldrev, newrev, ref) def execute(project, user, oldrev, newrev, ref)
@project, @user = project, user @project, @user = project, user
project.ensure_satellite_exists
project.repository.expire_cache project.repository.expire_cache
project.update_repository_size project.update_repository_size
......
module MergeRequests module MergeRequests
# AutoMergeService class # AutoMergeService class
# #
# Do git merge in satellite and in case of success # Do git merge and in case of success
# mark merge request as merged and execute all hooks and notifications # mark merge request as merged and execute all hooks and notifications
# Called when you do merge via GitLab UI # Called when you do merge via GitLab UI
class AutoMergeService < BaseMergeService class AutoMergeService < BaseMergeService
...@@ -32,9 +32,8 @@ module MergeRequests ...@@ -32,9 +32,8 @@ module MergeRequests
def merge! def merge!
if merge_request.for_fork? if merge_request.for_fork?
Gitlab::Satellite::MergeAction.new(current_user, merge_request).merge!(commit_message) raise 'Implement me'
else else
# Merge local branches using rugged instead of satellites
if sha = commit if sha = commit
after_commit(sha, merge_request.target_branch) after_commit(sha, merge_request.target_branch)
end end
......
...@@ -40,7 +40,6 @@ module MergeRequests ...@@ -40,7 +40,6 @@ module MergeRequests
merge_request.compare_diffs = diffs merge_request.compare_diffs = diffs
elsif diffs == false elsif diffs == false
# satellite timeout return false
merge_request.can_be_created = false merge_request.can_be_created = false
merge_request.compare_failed = true merge_request.compare_failed = true
end end
...@@ -59,9 +58,6 @@ module MergeRequests ...@@ -59,9 +58,6 @@ module MergeRequests
end end
merge_request merge_request
rescue Gitlab::Satellite::BranchesWithoutParent
return build_failed(merge_request, "Selected branches have no common commit so they cannot be merged.")
end end
def build_failed(merge_request, message) def build_failed(merge_request, message)
......
...@@ -27,7 +27,6 @@ module Projects ...@@ -27,7 +27,6 @@ module Projects
end end
end end
project.satellite.destroy
log_info("Project \"#{project.name}\" was removed") log_info("Project \"#{project.name}\" was removed")
system_hook_service.execute_hooks_for(project, :destroy) system_hook_service.execute_hooks_for(project, :destroy)
true true
......
...@@ -33,9 +33,6 @@ module Projects ...@@ -33,9 +33,6 @@ module Projects
raise TransferError.new("Project with same path in target namespace already exists") raise TransferError.new("Project with same path in target namespace already exists")
end end
# Remove old satellite
project.satellite.destroy
# Apply new namespace id # Apply new namespace id
project.namespace = new_namespace project.namespace = new_namespace
project.save! project.save!
...@@ -51,9 +48,6 @@ module Projects ...@@ -51,9 +48,6 @@ module Projects
# Move wiki repo also if present # Move wiki repo also if present
gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki") gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
# Create a new satellite (reload project from DB)
Project.find(project.id).ensure_satellite_exists
# clear project cached events # clear project cached events
project.reset_events_cache project.reset_events_cache
......
...@@ -35,7 +35,7 @@ ...@@ -35,7 +35,7 @@
- if @merge_request.compare_failed - if @merge_request.compare_failed
.alert.alert-danger .alert.alert-danger
%h4 Compare failed %h4 Compare failed
%p We can't compare selected branches. It may be because of huge diff or satellite timeout. Please try again or select different branches. %p We can't compare selected branches. It may be because of huge diff. Please try again or select different branches.
- else - else
.light-well .light-well
.center .center
......
...@@ -3,8 +3,6 @@ ...@@ -3,8 +3,6 @@
.mr-widget-body .mr-widget-body
- if @project.archived? - if @project.archived?
= render 'projects/merge_requests/widget/open/archived' = render 'projects/merge_requests/widget/open/archived'
- elsif !@project.satellite.exists?
= render 'projects/merge_requests/widget/open/no_satellite'
- elsif @merge_request.commits.blank? - elsif @merge_request.commits.blank?
= render 'projects/merge_requests/widget/open/nothing' = render 'projects/merge_requests/widget/open/nothing'
- elsif @merge_request.branch_missing? - elsif @merge_request.branch_missing?
......
%p
%span
%strong This repository does not have a satellite. Please ask an administrator to fix this issue!
...@@ -27,7 +27,6 @@ class RepositoryImportWorker ...@@ -27,7 +27,6 @@ class RepositoryImportWorker
project.import_finish project.import_finish
project.save project.save
project.satellite.create unless project.satellite.exists?
project.update_repository_size project.update_repository_size
Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket' Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket'
end end
......
...@@ -58,7 +58,7 @@ A typical install of GitLab will be on GNU/Linux. It uses Nginx or Apache as a w ...@@ -58,7 +58,7 @@ A typical install of GitLab will be on GNU/Linux. It uses Nginx or Apache as a w
The GitLab web app uses MySQL or PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. `/home/git/gitlab-satellites` keeps checked out repositories when performing actions such as a merge request, editing files in the web interface, etc. The GitLab web app uses MySQL or PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. `/home/git/gitlab-satellites` keeps checked out repositories when performing actions such as a merge request, editing files in the web interface, etc.
The satellite repository is used by the web interface for editing repositories and the wiki which is also a git repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects. [DEPRECATED] The satellite repository is used by the web interface for editing repositories and the wiki which is also a git repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects.
The add-on component gitlab-shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. gitlab-shell accesses the bare repositories directly to serve git objects and communicates with redis to submit jobs to Sidekiq for GitLab to process. gitlab-shell queries the GitLab API to determine authorization and access. The add-on component gitlab-shell serves repositories over SSH. It manages the SSH keys within `/home/git/.ssh/authorized_keys` which should not be manually edited. gitlab-shell accesses the bare repositories directly to serve git objects and communicates with redis to submit jobs to Sidekiq for GitLab to process. gitlab-shell queries the GitLab API to determine authorization and access.
......
...@@ -216,10 +216,6 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da ...@@ -216,10 +216,6 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
sudo chmod -R u+rwX,go-w log/ sudo chmod -R u+rwX,go-w log/
sudo chmod -R u+rwX tmp/ sudo chmod -R u+rwX tmp/
# Create directory for satellites
sudo -u git -H mkdir /home/git/gitlab-satellites
sudo chmod u+rwx,g=rx,o-rwx /home/git/gitlab-satellites
# Make sure GitLab can write to the tmp/pids/ and tmp/sockets/ directories # Make sure GitLab can write to the tmp/pids/ and tmp/sockets/ directories
sudo chmod -R u+rwX tmp/pids/ sudo chmod -R u+rwX tmp/pids/
sudo chmod -R u+rwX tmp/sockets/ sudo chmod -R u+rwX tmp/sockets/
......
...@@ -6,16 +6,14 @@ This is the directory structure you will end up with following the instructions ...@@ -6,16 +6,14 @@ This is the directory structure you will end up with following the instructions
| |-- git | |-- git
| |-- .ssh | |-- .ssh
| |-- gitlab | |-- gitlab
| |-- gitlab-satellites
| |-- gitlab-shell | |-- gitlab-shell
| |-- repositories | |-- repositories
* `/home/git/.ssh` - contains openssh settings. Specifically the `authorized_keys` file managed by gitlab-shell. * `/home/git/.ssh` - contains openssh settings. Specifically the `authorized_keys` file managed by gitlab-shell.
* `/home/git/gitlab` - GitLab core software. * `/home/git/gitlab` - GitLab core software.
* `/home/git/gitlab-satellites` - checked out repositories for merge requests and file editing from web UI. This can be treated as a temporary files directory.
* `/home/git/gitlab-shell` - Core add-on component of GitLab. Maintains SSH cloning and other functionality. * `/home/git/gitlab-shell` - Core add-on component of GitLab. Maintains SSH cloning and other functionality.
* `/home/git/repositories` - bare repositories for all projects organized by namespace. This is where the git repositories which are pushed/pulled are maintained for all projects. **This area is critical data for projects. [Keep a backup](../raketasks/backup_restore.md)** * `/home/git/repositories` - bare repositories for all projects organized by namespace. This is where the git repositories which are pushed/pulled are maintained for all projects. **This area is critical data for projects. [Keep a backup](../raketasks/backup_restore.md)**
*Note: the default locations for gitlab-satellites and repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of gitlab-shell.* *Note: the default locations for repositories can be configured in `config/gitlab.yml` of GitLab and `config.yml` of gitlab-shell.*
To see a more in-depth overview see the [GitLab architecture doc](../development/architecture.md). To see a more in-depth overview see the [GitLab architecture doc](../development/architecture.md).
...@@ -51,16 +51,6 @@ December 03, 2014 13:20 -> ERROR -> Command failed [1]: /usr/bin/git --git-dir=/ ...@@ -51,16 +51,6 @@ December 03, 2014 13:20 -> ERROR -> Command failed [1]: /usr/bin/git --git-dir=/
error: failed to push some refs to '/Users/vsizov/gitlab-development-kit/repositories/gitlabhq/gitlab_git.git' error: failed to push some refs to '/Users/vsizov/gitlab-development-kit/repositories/gitlabhq/gitlab_git.git'
``` ```
#### satellites.log
This file lives in `/var/log/gitlab/gitlab-rails/satellites.log` for omnibus package or in `/home/git/gitlab/log/satellites.log` for installations from the source.
In some cases GitLab should perform write actions to git repository, for example when it is needed to merge the merge request or edit a file with online editor. If something went wrong you can look into this file to find out what exactly happened.
```
October 07, 2014 11:36: Failed to create satellite for Chesley Weimann III / project1817
October 07, 2014 11:36: PID: 1872: git clone /Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/repositories/conrad6841/gitlabhq.git /Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/conrad6841/gitlabhq
October 07, 2014 11:36: PID: 1872: -> fatal: repository '/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/repositories/conrad6841/gitlabhq.git' does not exist
```
#### sidekiq.log #### sidekiq.log
This file lives in `/var/log/gitlab/gitlab-rails/sidekiq.log` for omnibus package or in `/home/git/gitlab/log/sidekiq.log` for installations from the source. This file lives in `/var/log/gitlab/gitlab-rails/sidekiq.log` for omnibus package or in `/home/git/gitlab/log/sidekiq.log` for installations from the source.
......
...@@ -105,24 +105,11 @@ Log directory writable? ... yes ...@@ -105,24 +105,11 @@ Log directory writable? ... yes
Tmp directory writable? ... yes Tmp directory writable? ... yes
Init script exists? ... yes Init script exists? ... yes
Init script up-to-date? ... yes Init script up-to-date? ... yes
Projects have satellites? ... yes
Redis version >= 2.0.0? ... yes Redis version >= 2.0.0? ... yes
Checking GitLab ... Finished Checking GitLab ... Finished
``` ```
## (Re-)Create satellite repositories
This will create satellite repositories for all your projects.
If necessary, remove the `repo_satellites` directory and rerun the commands below.
```
sudo -u git -H mkdir -p /home/git/gitlab-satellites
sudo -u git -H bundle exec rake gitlab:satellites:create RAILS_ENV=production
sudo chmod u+rwx,g=rx,o-rwx /home/git/gitlab-satellites
```
## Rebuild authorized_keys file ## Rebuild authorized_keys file
In some case it is necessary to rebuild the `authorized_keys` file. In some case it is necessary to rebuild the `authorized_keys` file.
......
...@@ -9,7 +9,6 @@ class Spinach::Features::ProjectForkedMergeRequests < Spinach::FeatureSteps ...@@ -9,7 +9,6 @@ class Spinach::Features::ProjectForkedMergeRequests < Spinach::FeatureSteps
@project = Project.find_by(name: "Shop") @project = Project.find_by(name: "Shop")
@project ||= create(:project, name: "Shop") @project ||= create(:project, name: "Shop")
@project.team << [@user, :reporter] @project.team << [@user, :reporter]
@project.ensure_satellite_exists
end end
step 'I have a project forked off of "Shop" called "Forked Shop"' do step 'I have a project forked off of "Shop" called "Forked Shop"' do
......
...@@ -198,15 +198,10 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps ...@@ -198,15 +198,10 @@ class Spinach::Features::ProjectMergeRequests < Spinach::FeatureSteps
end end
step 'merge request "Bug NS-05" is mergeable' do step 'merge request "Bug NS-05" is mergeable' do
merge_request.project.satellite.create
merge_request.mark_as_mergeable merge_request.mark_as_mergeable
end end
step 'I accept this merge request' do step 'I accept this merge request' do
Gitlab::Satellite::MergeAction.any_instance.stub(
merge!: true,
)
page.within '.mr-state-widget' do page.within '.mr-state-widget' do
click_button "Accept Merge Request" click_button "Accept Merge Request"
end end
......
require 'gitlab/git'
module Gitlab
autoload :Satellite, 'gitlab/satellite/satellite'
end
...@@ -217,20 +217,6 @@ module Gitlab ...@@ -217,20 +217,6 @@ module Gitlab
FileUtils.mv(full_path(old_name), full_path(new_name)) FileUtils.mv(full_path(old_name), full_path(new_name))
end end
# Remove GitLab Satellites for provided path (namespace or repo dir)
#
# Ex.
# rm_satellites("gitlab")
#
# rm_satellites("gitlab/gitlab-ci.git")
#
def rm_satellites(path)
raise ArgumentError.new("Path can't be blank") if path.blank?
satellites_path = File.join(Gitlab.config.satellites.path, path)
FileUtils.rm_r(satellites_path, force: true)
end
def url_to_repo(path) def url_to_repo(path)
Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git"
end end
......
module Gitlab
module Satellite
class Action
DEFAULT_OPTIONS = { git_timeout: Gitlab.config.satellites.timeout.seconds }
attr_accessor :options, :project, :user
def initialize(user, project, options = {})
@options = DEFAULT_OPTIONS.merge(options)
@project = project
@user = user
end
protected
# * Sets a 30s timeout for Git
# * Locks the satellite repo
# * Yields the prepared satellite repo
def in_locked_and_timed_satellite
Gitlab::ShellEnv.set_env(user)
Grit::Git.with_timeout(options[:git_timeout]) do
project.satellite.lock do
return yield project.satellite.repo
end
end
rescue Errno::ENOMEM => ex
return handle_exception(ex)
rescue Grit::Git::GitTimeout => ex
return handle_exception(ex)
ensure
Gitlab::ShellEnv.reset_env
end
# * Recreates the satellite
# * Sets up Git variables for the user
#
# Note: use this within #in_locked_and_timed_satellite
def prepare_satellite!(repo)
project.satellite.clear_and_update!
if user
repo.config['user.name'] = user.name
repo.config['user.email'] = user.email
end
end
def default_options(options = {})
{ raise: true, timeout: true }.merge(options)
end
def handle_exception(exception)
Gitlab::GitLogger.error(exception.message)
false
end
end
end
end
module Gitlab
module Satellite
class BranchesWithoutParent < StandardError; end
class CompareAction < Action
def initialize(user, target_project, target_branch, source_project, source_branch)
super user, target_project
@target_project, @target_branch = target_project, target_branch
@source_project, @source_branch = source_project, source_branch
end
# Compare 2 repositories and return Gitlab::CompareResult object
def result
in_locked_and_timed_satellite do |target_repo|
prepare_satellite!(target_repo)
update_satellite_source_and_target!(target_repo)
Gitlab::CompareResult.new(compare(target_repo))
end
rescue Grit::Git::CommandFailed => ex
raise BranchesWithoutParent
end
private
# Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for diffs
def update_satellite_source_and_target!(target_repo)
target_repo.remote_add('source', @source_project.repository.path_to_repo)
target_repo.remote_fetch('source')
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
def compare(repo)
@compare ||= Gitlab::Git::Compare.new(
Gitlab::Git::Repository.new(repo.path),
"origin/#{@target_branch}",
"source/#{@source_branch}"
)
end
end
end
end
module Gitlab
module Satellite
class Logger < Gitlab::Logger
def self.file_name
'satellites.log'
end
def format_message(severity, timestamp, progname, msg)
"#{timestamp.to_s(:long)}: #{msg}\n"
end
end
end
end
module Gitlab
module Satellite
# GitLab server-side merge
class MergeAction < Action
attr_accessor :merge_request
def initialize(user, merge_request)
super user, merge_request.target_project
@merge_request = merge_request
end
# Checks if a merge request can be executed without user interaction
def can_be_merged?
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
merge_in_satellite!(merge_repo)
end
end
# Merges the source branch into the target branch in the satellite and
# pushes it back to the repository.
# It also removes the source branch if requested in the merge request (and this is permitted by the merge request).
#
# Returns false if the merge produced conflicts
# Returns false if pushing from the satellite to the repository failed or was rejected
# Returns true otherwise
def merge!(merge_commit_message = nil)
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
if merge_in_satellite!(merge_repo, merge_commit_message)
# push merge back to bare repo
# will raise CommandFailed when push fails
merge_repo.git.push(default_options, :origin, merge_request.target_branch)
# remove source branch
if merge_request.should_remove_source_branch && !project.root_ref?(merge_request.source_branch) && !merge_request.for_fork?
# will raise CommandFailed when push fails
merge_repo.git.push(default_options, :origin, ":#{merge_request.source_branch}")
end
# merge, push and branch removal successful
true
end
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
def diff_in_satellite
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
# Only show what is new in the source branch compared to the target branch, not the other way around.
# The line below with merge_base is equivalent to diff with three dots (git diff branch1...branch2)
# From the git documentation: "git diff A...B" is equivalent to "git diff $(git-merge-base A B) B"
common_commit = merge_repo.git.native(:merge_base, default_options, ["origin/#{merge_request.target_branch}", "source/#{merge_request.source_branch}"]).strip
merge_repo.git.native(:diff, default_options, common_commit, "source/#{merge_request.source_branch}")
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
def diffs_between_satellite
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
if merge_request.for_fork?
repository = Gitlab::Git::Repository.new(merge_repo.path)
diffs = Gitlab::Git::Diff.between(
repository,
"source/#{merge_request.source_branch}",
"origin/#{merge_request.target_branch}"
)
else
raise "Attempt to determine diffs between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]"
end
return diffs
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
# Get commit as an email patch
def format_patch
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
patch = merge_repo.git.format_patch(default_options({ stdout: true }), "origin/#{merge_request.target_branch}..source/#{merge_request.source_branch}")
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
# Retrieve an array of commits between the source and the target
def commits_between
in_locked_and_timed_satellite do |merge_repo|
prepare_satellite!(merge_repo)
update_satellite_source_and_target!(merge_repo)
if merge_request.for_fork?
repository = Gitlab::Git::Repository.new(merge_repo.path)
commits = Gitlab::Git::Commit.between(
repository,
"origin/#{merge_request.target_branch}",
"source/#{merge_request.source_branch}"
)
else
raise "Attempt to determine commits between for a non forked merge request in satellite MergeRequest.id:[#{merge_request.id}]"
end
return commits
end
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
private
# Merges the source_branch into the target_branch in the satellite.
#
# Note: it will clear out the satellite before doing anything
#
# Returns false if the merge produced conflicts
# Returns true otherwise
def merge_in_satellite!(repo, message = nil)
update_satellite_source_and_target!(repo)
message ||= "Merge branch '#{merge_request.source_branch}' into '#{merge_request.target_branch}'"
# merge the source branch into the satellite
# will raise CommandFailed when merge fails
repo.git.merge(default_options({ no_ff: true }), "-m#{message}", "source/#{merge_request.source_branch}")
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
# Assumes a satellite exists that is a fresh clone of the projects repo, prepares satellite for merges, diffs etc
def update_satellite_source_and_target!(repo)
repo.remote_add('source', merge_request.source_project.repository.path_to_repo)
repo.remote_fetch('source')
repo.git.checkout(default_options({ b: true }), merge_request.target_branch, "origin/#{merge_request.target_branch}")
rescue Grit::Git::CommandFailed => ex
handle_exception(ex)
end
end
end
end
module Gitlab
module Satellite
autoload :DeleteFileAction, 'gitlab/satellite/files/delete_file_action'
autoload :EditFileAction, 'gitlab/satellite/files/edit_file_action'
autoload :FileAction, 'gitlab/satellite/files/file_action'
autoload :NewFileAction, 'gitlab/satellite/files/new_file_action'
class CheckoutFailed < StandardError; end
class CommitFailed < StandardError; end
class PushFailed < StandardError; end
class Satellite
include Gitlab::Popen
PARKING_BRANCH = "__parking_branch"
attr_accessor :project
def initialize(project)
@project = project
end
def log(message)
Gitlab::Satellite::Logger.error(message)
end
def clear_and_update!
project.ensure_satellite_exists
@repo = nil
clear_working_dir!
delete_heads!
remove_remotes!
update_from_source!
end
def create
output, status = popen(%W(git clone -- #{project.repository.path_to_repo} #{path}),
Gitlab.config.satellites.path)
log("PID: #{project.id}: git clone #{project.repository.path_to_repo} #{path}")
log("PID: #{project.id}: -> #{output}")
if status.zero?
true
else
log("Failed to create satellite for #{project.name_with_namespace}")
false
end
end
def exists?
File.exists? path
end
# * Locks the satellite
# * Changes the current directory to the satellite's working dir
# * Yields
def lock
project.ensure_satellite_exists
File.open(lock_file, "w+") do |f|
begin
f.flock File::LOCK_EX
yield
ensure
f.flock File::LOCK_UN
end
end
end
def lock_file
create_locks_dir unless File.exists?(lock_files_dir)
File.join(lock_files_dir, "satellite_#{project.id}.lock")
end
def path
File.join(Gitlab.config.satellites.path, project.path_with_namespace)
end
def repo
project.ensure_satellite_exists
@repo ||= Grit::Repo.new(path)
end
def destroy
FileUtils.rm_rf(path)
end
private
# Clear the working directory
def clear_working_dir!
repo.git.reset(hard: true)
repo.git.clean(f: true, d: true, x: true)
end
# Deletes all branches except the parking branch
#
# This ensures we have no name clashes or issues updating branches when
# working with the satellite.
def delete_heads!
heads = repo.heads.map(&:name)
# update or create the parking branch
repo.git.checkout(default_options({ B: true }), PARKING_BRANCH)
# remove the parking branch from the list of heads ...
heads.delete(PARKING_BRANCH)
# ... and delete all others
heads.each { |head| repo.git.branch(default_options({ D: true }), head) }
end
# Deletes all remotes except origin
#
# This ensures we have no remote name clashes or issues updating branches when
# working with the satellite.
def remove_remotes!
remotes = repo.git.remote.split(' ')
remotes.delete('origin')
remotes.each { |name| repo.git.remote(default_options,'rm', name)}
end
# Updates the satellite from bare repo
#
# Note: this will only update remote branches (i.e. origin/*)
def update_from_source!
repo.git.remote(default_options, 'set-url', :origin, project.repository.path_to_repo)
repo.git.fetch(default_options, :origin)
end
def default_options(options = {})
{ raise: true, timeout: true }.merge(options)
end
# Create directory for storing
# satellites lock files
def create_locks_dir
FileUtils.mkdir_p(lock_files_dir)
end
def lock_files_dir
@lock_files_dir ||= File.join(Gitlab.config.satellites.path, "tmp")
end
end
end
end
...@@ -25,7 +25,6 @@ namespace :gitlab do ...@@ -25,7 +25,6 @@ namespace :gitlab do
check_init_script_exists check_init_script_exists
check_init_script_up_to_date check_init_script_up_to_date
check_projects_have_namespace check_projects_have_namespace
check_satellites_exist
check_redis_version check_redis_version
check_ruby_version check_ruby_version
check_git_version check_git_version
...@@ -238,37 +237,6 @@ namespace :gitlab do ...@@ -238,37 +237,6 @@ namespace :gitlab do
end end
end end
def check_satellites_exist
print "Projects have satellites? ... "
unless Project.count > 0
puts "can't check, you have no projects".magenta
return
end
puts ""
Project.find_each(batch_size: 100) do |project|
print sanitized_message(project)
if project.satellite.exists?
puts "yes".green
elsif project.empty_repo?
puts "can't create, repository is empty".magenta
else
puts "no".red
try_fixing_it(
sudo_gitlab("bundle exec rake gitlab:satellites:create RAILS_ENV=production"),
"If necessary, remove the tmp/repo_satellites directory ...",
"... and rerun the above command"
)
for_more_information(
"doc/raketasks/maintenance.md "
)
fix_and_rerun
end
end
end
def check_log_writable def check_log_writable
print "Log directory writable? ... " print "Log directory writable? ... "
...@@ -339,7 +307,6 @@ namespace :gitlab do ...@@ -339,7 +307,6 @@ namespace :gitlab do
check_repo_base_is_not_symlink check_repo_base_is_not_symlink
check_repo_base_user_and_group check_repo_base_user_and_group
check_repo_base_permissions check_repo_base_permissions
check_satellites_permissions
check_repos_hooks_directory_is_link check_repos_hooks_directory_is_link
check_gitlab_shell_self_test check_gitlab_shell_self_test
...@@ -417,29 +384,6 @@ namespace :gitlab do ...@@ -417,29 +384,6 @@ namespace :gitlab do
end end
end end
def check_satellites_permissions
print "Satellites access is drwxr-x---? ... "
satellites_path = Gitlab.config.satellites.path
unless File.exists?(satellites_path)
puts "can't check because of previous errors".magenta
return
end
if File.stat(satellites_path).mode.to_s(8).ends_with?("0750")
puts "yes".green
else
puts "no".red
try_fixing_it(
"sudo chmod u+rwx,g=rx,o-rwx #{satellites_path}",
)
for_more_information(
see_installation_guide_section "GitLab"
)
fix_and_rerun
end
end
def check_repo_base_user_and_group def check_repo_base_user_and_group
gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user
gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group
......
namespace :gitlab do
namespace :satellites do
desc "GitLab | Create satellite repos"
task create: :environment do
create_satellites
end
end
def create_satellites
warn_user_is_not_gitlab
print "Creating satellites for ..."
unless Project.count > 0
puts "skipping, because you have no projects".magenta
return
end
puts ""
Project.find_each(batch_size: 100) do |project|
print "#{project.name_with_namespace.yellow} ... "
unless project.repo_exists?
puts "skipping, because the repo is empty".magenta
next
end
if project.satellite.exists?
puts "exists already".green
else
print "\n... "
if project.satellite.create
puts "created".green
else
puts "error".red
end
end
end
end
end
require 'spec_helper'
describe 'Gitlab::Satellite::Action' do
let(:project) { create(:project) }
let(:user) { create(:user) }
describe '#prepare_satellite!' do
it 'should be able to fetch timeout from conf' do
expect(Gitlab::Satellite::Action::DEFAULT_OPTIONS[:git_timeout]).to eq(30.seconds)
end
it 'create a repository with a parking branch and one remote: origin' do
repo = project.satellite.repo
#now lets dirty it up
starting_remote_count = repo.git.list_remotes.size
expect(starting_remote_count).to be >= 1
#kind of hookey way to add a second remote
origin_uri = repo.git.remote({ v: true }).split(" ")[1]
repo.git.remote({ raise: true }, 'add', 'another-remote', origin_uri)
repo.git.branch({ raise: true }, 'a-new-branch')
expect(repo.heads.size).to be > (starting_remote_count)
expect(repo.git.remote().split(" ").size).to be > (starting_remote_count)
repo.git.config({}, "user.name", "#{user.name} -- foo")
repo.git.config({}, "user.email", "#{user.email} -- foo")
expect(repo.config['user.name']).to eq("#{user.name} -- foo")
expect(repo.config['user.email']).to eq("#{user.email} -- foo")
#These must happen in the context of the satellite directory...
satellite_action = Gitlab::Satellite::Action.new(user, project)
project.satellite.lock do
#Now clean it up, use send to get around prepare_satellite! being protected
satellite_action.send(:prepare_satellite!, repo)
end
#verify it's clean
heads = repo.heads.map(&:name)
expect(heads.size).to eq(1)
expect(heads.include?(Gitlab::Satellite::Satellite::PARKING_BRANCH)).to eq(true)
remotes = repo.git.remote().split(' ')
expect(remotes.size).to eq(1)
expect(remotes.include?('origin')).to eq(true)
expect(repo.config['user.name']).to eq(user.name)
expect(repo.config['user.email']).to eq(user.email)
end
end
describe '#in_locked_and_timed_satellite' do
it 'should make use of a lockfile' do
repo = project.satellite.repo
called = false
#set assumptions
FileUtils.rm_f(project.satellite.lock_file)
expect(File.exists?(project.satellite.lock_file)).to be_falsey
satellite_action = Gitlab::Satellite::Action.new(user, project)
satellite_action.send(:in_locked_and_timed_satellite) do |sat_repo|
expect(repo).to eq(sat_repo)
expect(File.exists? project.satellite.lock_file).to be_truthy
called = true
end
expect(called).to be_truthy
end
it 'should be able to use the satellite after locking' do
repo = project.satellite.repo
called = false
# Set base assumptions
if File.exists? project.satellite.lock_file
expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_falsey
end
satellite_action = Gitlab::Satellite::Action.new(user, project)
satellite_action.send(:in_locked_and_timed_satellite) do |sat_repo|
called = true
expect(repo).to eq(sat_repo)
expect(File.exists? project.satellite.lock_file).to be_truthy
expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_truthy
end
expect(called).to be_truthy
expect(FileLockStatusChecker.new(project.satellite.lock_file).flocked?).to be_falsey
end
class FileLockStatusChecker < File
def flocked?(&block)
status = flock LOCK_EX|LOCK_NB
case status
when false
return true
when 0
begin
block ? block.call : false
ensure
flock LOCK_UN
end
else
raise SystemCallError, status
end
end
end
end
end
require 'spec_helper'
describe 'Gitlab::Satellite::MergeAction' do
include RepoHelpers
let(:project) { create(:project, namespace: create(:group)) }
let(:fork_project) { create(:project, namespace: create(:group), forked_from_project: project) }
let(:merge_request) { create(:merge_request, source_project: project, target_project: project) }
let(:merge_request_fork) { create(:merge_request, source_project: fork_project, target_project: project) }
let(:merge_request_with_conflict) { create(:merge_request, :conflict, source_project: project, target_project: project) }
let(:merge_request_fork_with_conflict) { create(:merge_request, :conflict, source_project: project, target_project: project) }
describe '#commits_between' do
def verify_commits(commits, first_commit_sha, last_commit_sha)
commits.each { |commit| expect(commit.class).to eq(Gitlab::Git::Commit) }
expect(commits.first.id).to eq(first_commit_sha)
expect(commits.last.id).to eq(last_commit_sha)
end
context 'on fork' do
it 'should get proper commits between' do
commits = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).commits_between
verify_commits(commits, sample_compare.commits.first, sample_compare.commits.last)
end
end
context 'between branches' do
it 'should raise exception -- not expected to be used by non forks' do
expect { Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).commits_between }.to raise_error(RuntimeError)
end
end
end
describe '#format_patch' do
def verify_content(patch)
sample_compare.commits.each do |commit|
expect(patch.include?(commit)).to be_truthy
end
end
context 'on fork' do
it 'should build a format patch' do
patch = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).format_patch
verify_content(patch)
end
end
context 'between branches' do
it 'should build a format patch' do
patch = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request).format_patch
verify_content(patch)
end
end
end
describe '#diffs_between_satellite tested against diff_in_satellite' do
def is_a_matching_diff(diff, diffs)
diff_count = diff.scan('diff --git').size
expect(diff_count).to be >= 1
expect(diffs.size).to eq(diff_count)
diffs.each do |a_diff|
expect(a_diff.class).to eq(Gitlab::Git::Diff)
expect(diff.include? a_diff.diff).to be_truthy
end
end
context 'on fork' do
it 'should get proper diffs' do
diffs = Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).diffs_between_satellite
diff = Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request_fork).diff_in_satellite
is_a_matching_diff(diff, diffs)
end
end
context 'between branches' do
it 'should get proper diffs' do
expect{ Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).diffs_between_satellite }.to raise_error(RuntimeError)
end
end
end
describe '#can_be_merged?' do
context 'on fork' do
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request_fork.author, merge_request_fork).can_be_merged?).to be_truthy
end
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request_fork_with_conflict.author, merge_request_fork_with_conflict).can_be_merged?).to be_falsey
end
end
context 'between branches' do
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request.author, merge_request).can_be_merged?).to be_truthy
end
it do
expect(Gitlab::Satellite::MergeAction.new(merge_request_with_conflict.author, merge_request_with_conflict).can_be_merged?).to be_falsey
end
end
end
end
...@@ -90,7 +90,6 @@ describe Project do ...@@ -90,7 +90,6 @@ describe Project do
describe 'Respond to' do describe 'Respond to' do
it { is_expected.to respond_to(:url_to_repo) } it { is_expected.to respond_to(:url_to_repo) }
it { is_expected.to respond_to(:repo_exists?) } it { is_expected.to respond_to(:repo_exists?) }
it { is_expected.to respond_to(:satellite) }
it { is_expected.to respond_to(:update_merge_requests) } it { is_expected.to respond_to(:update_merge_requests) }
it { is_expected.to respond_to(:execute_hooks) } it { is_expected.to respond_to(:execute_hooks) }
it { is_expected.to respond_to(:name_with_namespace) } it { is_expected.to respond_to(:name_with_namespace) }
......
...@@ -110,7 +110,7 @@ describe API::API, api: true do ...@@ -110,7 +110,7 @@ describe API::API, api: true do
expect(response.status).to eq(400) expect(response.status).to eq(400)
end end
it "should return a 400 if satellite fails to create file" do it "should return a 400 if fails to create file" do
allow_any_instance_of(Repository).to receive(:remove_file).and_return(false) allow_any_instance_of(Repository).to receive(:remove_file).and_return(false)
delete api("/projects/#{project.id}/repository/files", user), valid_params delete api("/projects/#{project.id}/repository/files", user), valid_params
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment