Commit c7fde6a0 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'ce-mirror-backport' into 'master'

Backport relevant changes from gitlab-org/gitlab-ee!51

To do:

- [x] Update gitlab-shell

See merge request !1822
parents a5a57452 7763e61a
......@@ -328,6 +328,10 @@ table {
}
}
.well {
margin-bottom: 0;
}
.search_box {
@extend .well;
text-align: center;
......
......@@ -172,7 +172,7 @@
}
.panel-body {
form {
form, pre {
margin: 0;
}
......
......@@ -28,8 +28,8 @@ class Projects::ImportsController < Projects::ApplicationController
if @project.import_finished?
redirect_to(project_path(@project)) and return
else
redirect_to new_namespace_project_import_path(@project.namespace,
@project) && return
redirect_to(new_namespace_project_import_path(@project.namespace,
@project)) and return
end
end
end
......
......@@ -253,14 +253,6 @@ module ProjectsHelper
filename_path(project, :version)
end
def hidden_pass_url(original_url)
result = URI(original_url)
result.password = '*****' unless result.password.nil?
result
rescue
original_url
end
def project_wiki_path_with_version(proj, page, version, is_newest)
url_params = is_newest ? {} : { version_id: version }
namespace_project_wiki_path(proj.namespace, proj, page, url_params)
......
......@@ -310,15 +310,17 @@ class Project < ActiveRecord::Base
def add_import_job
if forked?
unless RepositoryForkWorker.perform_async(id, forked_from_project.path_with_namespace, self.namespace.path)
import_fail
end
RepositoryForkWorker.perform_async(self.id, forked_from_project.path_with_namespace, self.namespace.path)
else
RepositoryImportWorker.perform_async(id)
RepositoryImportWorker.perform_async(self.id)
end
end
def clear_import_data
update(import_error: nil)
ProjectCacheWorker.perform_async(self.id)
self.import_data.destroy if self.import_data
end
......@@ -346,6 +348,14 @@ class Project < ActiveRecord::Base
import_status == 'finished'
end
def safe_import_url
result = URI.parse(self.import_url)
result.password = '*****' unless result.password.nil?
result.to_s
rescue
original_url
end
def check_limit
unless creator.can_create_project? or namespace.kind == 'group'
errors[:limit_reached] << ("Your project limit is #{creator.projects_limit} projects! Please contact your administrator to increase it")
......
......@@ -6,7 +6,7 @@ class Repository
include Gitlab::ShellAdapter
attr_accessor :raw_repository, :path_with_namespace, :project
attr_accessor :path_with_namespace, :project
def self.clean_old_archives
repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path
......@@ -19,14 +19,18 @@ class Repository
def initialize(path_with_namespace, default_branch = nil, project = nil)
@path_with_namespace = path_with_namespace
@project = project
end
if path_with_namespace
@raw_repository = Gitlab::Git::Repository.new(path_to_repo)
@raw_repository.autocrlf = :input
end
def raw_repository
return nil unless path_with_namespace
rescue Gitlab::Git::Repository::NoRepository
nil
@raw_repository ||= begin
repo = Gitlab::Git::Repository.new(path_to_repo)
repo.autocrlf = :input
repo
rescue Gitlab::Git::Repository::NoRepository
nil
end
end
# Return absolute path to repository
......@@ -105,29 +109,25 @@ class Repository
end
def add_branch(branch_name, ref)
cache.expire(:branch_names)
@branches = nil
expire_branches_cache
gitlab_shell.add_branch(path_with_namespace, branch_name, ref)
end
def add_tag(tag_name, ref, message = nil)
cache.expire(:tag_names)
@tags = nil
expire_tags_cache
gitlab_shell.add_tag(path_with_namespace, tag_name, ref, message)
end
def rm_branch(branch_name)
cache.expire(:branch_names)
@branches = nil
expire_branches_cache
gitlab_shell.rm_branch(path_with_namespace, branch_name)
end
def rm_tag(tag_name)
cache.expire(:tag_names)
@tags = nil
expire_tags_cache
gitlab_shell.rm_tag(path_with_namespace, tag_name)
end
......@@ -169,6 +169,16 @@ class Repository
end
end
def expire_tags_cache
cache.expire(:tag_names)
@tags = nil
end
def expire_branches_cache
cache.expire(:branch_names)
@branches = nil
end
def expire_cache
cache_keys.each do |key|
cache.expire(key)
......@@ -484,7 +494,7 @@ class Repository
root_ref_commit = commit(root_ref)
if branch_commit
rugged.merge_base(root_ref_commit.id, branch_commit.id) == branch_commit.id
is_ancestor?(branch_commit.id, root_ref_commit.id)
else
nil
end
......@@ -494,6 +504,11 @@ class Repository
rugged.merge_base(first_commit_id, second_commit_id)
end
def is_ancestor?(ancestor_id, descendant_id)
merge_base(ancestor_id, descendant_id) == ancestor_id
end
def search_files(query, ref)
offset = 2
args = %W(#{Gitlab.config.git.bin_path} grep -i -n --before-context #{offset} --after-context #{offset} -e #{query} #{ref || root_ref})
......
......@@ -55,7 +55,9 @@ module Projects
@project.save
if @project.persisted? && !@project.import?
raise 'Failed to create repository' unless @project.create_repository
unless @project.create_repository
raise 'Failed to create repository'
end
end
end
......@@ -94,9 +96,7 @@ module Projects
@project.team << [current_user, :master, current_user]
end
if @project.import?
@project.import_start
end
@project.import_start if @project.import?
end
end
end
......@@ -18,17 +18,12 @@
.project-repo-buttons
.split-one
= render 'projects/buttons/star'
= render 'projects/buttons/fork'
- unless empty_repo
= render 'projects/buttons/fork'
= render "shared/clone_panel"
.split-repo-buttons
- unless empty_repo
- if can? current_user, :download_code, @project
= link_to archive_namespace_project_repository_path(@project.namespace, @project, ref: @ref, format: 'zip'), class: 'btn', rel: 'nofollow' do
= icon('download fw')
.split-repo-buttons
= render "projects/buttons/download"
= render 'projects/buttons/dropdown'
= render 'projects/buttons/notifications'
- unless @project.empty_repo?
- if can? current_user, :download_code, @project
= link_to archive_namespace_project_repository_path(@project.namespace, @project, ref: @ref, format: 'zip'), class: 'btn has_tooltip', rel: 'nofollow', title: "Download ZIP" do
= icon('download')
......@@ -32,5 +32,3 @@
= link_to new_namespace_project_tag_path(@project.namespace, @project) do
= icon('tags fw')
New tag
- if current_user && can?(current_user, :fork_project, @project)
- if current_user.already_forked?(@project) && current_user.manageable_namespaces.size < 2
= link_to namespace_project_path(current_user, current_user.fork_of(@project)), title: 'Go to your fork', class: 'btn' do
= icon('code-fork fw')
Fork
%span.count
= @project.forks_count
- else
= link_to new_namespace_project_fork_path(@project.namespace, @project), title: "Fork project", class: 'btn' do
= icon('code-fork fw')
%span.count
= @project.forks_count
- unless @project.empty_repo?
- if current_user && can?(current_user, :fork_project, @project)
- if current_user.already_forked?(@project) && current_user.manageable_namespaces.size < 2
= link_to namespace_project_path(current_user, current_user.fork_of(@project)), title: 'Go to your fork', class: 'btn has_tooltip' do
= icon('code-fork fw')
Fork
%span.count
= @project.forks_count
- else
= link_to new_namespace_project_fork_path(@project.namespace, @project), title: "Fork project", class: 'btn has_tooltip' do
= icon('code-fork fw')
%span.count
= @project.forks_count
- if current_user
= link_to toggle_star_namespace_project_path(@project.namespace, @project), class: 'btn star-btn toggle-star', method: :post, remote: true do
= link_to toggle_star_namespace_project_path(@project.namespace, @project), class: 'btn star-btn toggle-star has_tooltip', method: :post, remote: true, title: "Star project" do
= icon('star fw')
%span.count
= @project.star_count
......
- page_title "Import repository"
%h3.page-title
- if @project.import_failed?
Import failed. Retry?
- else
Import repository
Import repository
%hr
- if @project.import_failed?
.panel.panel-danger
.panel-heading The repository could not be imported.
.panel-body
%pre
:preserve
#{@project.import_error.try(:strip)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
.form-group.import-url-data
= f.label :import_url, class: 'control-label' do
%span Import existing git repo
.col-sm-10
= f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
.well.prepend-top-20
This URL must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
%br
The import will time out after 4 minutes. For big repositories, use a clone/push combination.
For SVN repositories, check #{link_to "this migrating from SVN doc.", "http://doc.gitlab.com/ce/workflow/importing/migrating_from_svn.html"}
= render "shared/import_form", f: f
.form-actions
= f.submit 'Start import', class: "btn btn-create", tabindex: 4
......@@ -8,7 +8,7 @@
- else
Import in progress.
- unless @project.forked?
%p.monospace git clone --bare #{hidden_pass_url(@project.import_url)}
%p.monospace git clone --bare #{@project.safe_import_url}
%p Please wait while we import the repository for you. Refresh at will.
:javascript
new ProjectImport();
......@@ -23,7 +23,6 @@
= f.select :namespace_id, namespaces_options(params[:namespace_id] || :current_user), {}, {class: 'select2', tabindex: 2}
- if import_sources_enabled?
.project-import.js-toggle-container
.form-group
%label.control-label Import project from
......@@ -82,19 +81,7 @@
%span Any repo by URL
.js-toggle-content.hide
.form-group.import-url-data
= f.label :import_url, class: 'control-label' do
%span Git repository URL
.col-sm-10
= f.text_field :import_url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
.well.prepend-top-20
%ul
%li
The repository must be accessible over HTTP(S). If it is not publicly accessible, you can add authentication information to the URL: <code>https://username:password@gitlab.company.com/group/project.git</code>.
%li
The import will time out after 4 minutes. For big repositories, use a clone/push combination.
%li
To migrate an SVN repository, check out #{link_to "this document", "http://doc.gitlab.com/ce/workflow/importing/migrating_from_svn.html"}.
= render "shared/import_form", f: f
.prepend-botton-10
......
......@@ -21,7 +21,6 @@
= gitlab_config.protocol.upcase
= text_field_tag :project_clone, default_url_to_repo(project), class: "js-select-on-focus form-control", readonly: true
- if project.kind_of?(Project)
.input-group-addon
.visibility-level-label.has_tooltip{'data-title' => "#{visibility_level_label(project.visibility_level)} project" }
.input-group-addon.has_tooltip{title: "#{visibility_level_label(project.visibility_level)} project", data: { container: "body" } }
.visibility-level-label
= visibility_level_icon(project.visibility_level)
.form-group.import-url-data
= f.label :import_url, class: 'control-label' do
%span Git repository URL
.col-sm-10
= f.text_field :import_url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git'
.well.prepend-top-20
%ul
%li
The repository must be accessible over <code>http://</code>, <code>https://</code> or <code>git://</code>.
%li
If your HTTP repository is not publicly accessible, add authentication information to the URL: <code>https://username:password@gitlab.company.com/group/project.git</code>.
%li
The import will time out after 4 minutes. For big repositories, use a clone/push combination.
%li
To migrate an SVN repository, check out #{link_to "this document", "http://doc.gitlab.com/ce/workflow/importing/migrating_from_svn.html"}.
......@@ -13,22 +13,20 @@ class RepositoryForkWorker
end
result = gitlab_shell.fork_repository(source_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.update(import_error: "The project could not be forked.")
project.import_fail
project.save
return
end
if project.valid_repo?
ProjectCacheWorker.perform_async(project.id)
project.import_finish
else
project.import_fail
unless project.valid_repo?
logger.error("Project #{id} had an invalid repository after fork")
project.update(import_error: "The forked repository is invalid.")
project.import_fail
return
end
project.save
project.import_finish
end
end
......@@ -7,37 +7,52 @@ class RepositoryImportWorker
def perform(project_id)
project = Project.find(project_id)
unless project.import_url == Project::UNKNOWN_IMPORT_URL
import_result = gitlab_shell.send(:import_repository,
project.path_with_namespace,
project.import_url)
return project.import_fail unless import_result
else
if project.import_url == Project::UNKNOWN_IMPORT_URL
# In this case, we only want to import issues, not a repository.
unless project.create_repository
return project.import_fail
project.update(import_error: "The repository could not be created.")
project.import_fail
return
end
else
begin
gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e
project.update(import_error: e.message)
project.import_fail
return
end
end
data_import_result = case project.import_type
when 'github'
Gitlab::GithubImport::Importer.new(project).execute
when 'gitlab'
Gitlab::GitlabImport::Importer.new(project).execute
when 'bitbucket'
Gitlab::BitbucketImport::Importer.new(project).execute
when 'google_code'
Gitlab::GoogleCodeImport::Importer.new(project).execute
when 'fogbugz'
Gitlab::FogbugzImport::Importer.new(project).execute
else
true
end
return project.import_fail unless data_import_result
Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket'
data_import_result =
case project.import_type
when 'github'
Gitlab::GithubImport::Importer.new(project).execute
when 'gitlab'
Gitlab::GitlabImport::Importer.new(project).execute
when 'bitbucket'
Gitlab::BitbucketImport::Importer.new(project).execute
when 'google_code'
Gitlab::GoogleCodeImport::Importer.new(project).execute
when 'fogbugz'
Gitlab::FogbugzImport::Importer.new(project).execute
else
true
end
unless data_import_result
project.update(import_error: "The remote issue data could not be imported.")
project.import_fail
return
end
if project.import_type == 'bitbucket'
Gitlab::BitbucketImport::KeyDeleter.new(project).execute
end
project.import_finish
project.save
ProjectCacheWorker.perform_async(project.id)
# Explicitly update mirror so that upstream remote is created and fetched
project.update_mirror
end
end
class AddImportErrorToProject < ActiveRecord::Migration
def change
add_column :projects, :import_error, :text
end
end
......@@ -640,7 +640,10 @@ ActiveRecord::Schema.define(version: 20151116144118) do
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.integer "commit_count", default: 0
t.boolean "merge_requests_ff_only_enabled", default: false
t.text "issues_template"
t.text "import_error"
end
add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree
......
......@@ -44,7 +44,6 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps
git_import_instructions = first('.js-toggle-content')
expect(git_import_instructions).to be_visible
expect(git_import_instructions).to have_content "Git repository URL"
expect(git_import_instructions).to have_content "The repository must be accessible over HTTP(S). If it is not publicly accessible, you can add authentication information to the URL:"
end
step 'I click on "Google Code"' do
......
module Gitlab
class Shell
class AccessDenied < StandardError; end
class Error < StandardError; end
class KeyAdder < Struct.new(:io)
def add_key(id, key)
......@@ -36,8 +36,9 @@ module Gitlab
# import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git")
#
def import_repository(name, url)
Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'import-project',
"#{name}.git", url, '240'])
output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240'])
raise Error, output unless status.zero?
true
end
# Move repository
......
......@@ -25,13 +25,6 @@ describe Projects::ForkService do
end
end
context 'fork project failure' do
it "fails due to transaction failure" do
@to_project = fork_project(@from_project, @to_user, false)
expect(@to_project.import_failed?)
end
end
context 'project already exists' do
it "should fail due to validation, not transaction failure" do
@existing_project = create(:project, creator_id: @to_user.id, name: @from_project.name, namespace: @to_namespace)
......@@ -66,7 +59,7 @@ describe Projects::ForkService do
context 'fork project for group' do
it 'group owner successfully forks project into the group' do
to_project = fork_project(@project, @group_owner, true, @opts)
to_project = fork_project(@project, @group_owner, @opts)
expect(to_project.owner).to eq(@group)
expect(to_project.namespace).to eq(@group)
expect(to_project.name).to eq(@project.name)
......@@ -78,7 +71,7 @@ describe Projects::ForkService do
context 'fork project for group when user not owner' do
it 'group developer should fail to fork project into the group' do
to_project = fork_project(@project, @developer, true, @opts)
to_project = fork_project(@project, @developer, @opts)
expect(to_project.errors[:namespace]).to eq(['is not valid'])
end
end
......@@ -87,7 +80,7 @@ describe Projects::ForkService do
it 'should fail due to validation, not transaction failure' do
existing_project = create(:project, name: @project.name,
namespace: @group)
to_project = fork_project(@project, @group_owner, true, @opts)
to_project = fork_project(@project, @group_owner, @opts)
expect(existing_project.persisted?).to be_truthy
expect(to_project.errors[:name]).to eq(['has already been taken'])
expect(to_project.errors[:path]).to eq(['has already been taken'])
......@@ -95,8 +88,8 @@ describe Projects::ForkService do
end
end
def fork_project(from_project, user, fork_success = true, params = {})
allow(RepositoryForkWorker).to receive(:perform_async).and_return(fork_success)
def fork_project(from_project, user, params = {})
allow(RepositoryForkWorker).to receive(:perform_async).and_return(true)
Projects::ForkService.new(from_project, user, params).execute
end
end
......@@ -12,7 +12,6 @@ describe RepositoryForkWorker do
project.path_with_namespace,
fork_project.namespace.path).
and_return(true)
expect(ProjectCacheWorker).to receive(:perform_async)
subject.perform(project.id,
project.path_with_namespace,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment