Commit ad6d6232 authored by Douwe Maan's avatar Douwe Maan

Add Bitbucket importer.

parent 7561b1c2
......@@ -72,6 +72,8 @@ v 7.8.0
- Improve database performance for GitLab
- Add Asana service (Jeremy Benoist)
- Improve project web hooks with extra data
- Add Bitbucket omniauth provider.
- Add Bitbucket importer.
v 7.7.2
- Update GitLab Shell to version 2.4.2 that fixes a bug when developers can push to protected branch
......
......@@ -30,6 +30,7 @@ gem 'omniauth-github'
gem 'omniauth-shibboleth'
gem 'omniauth-kerberos'
gem 'omniauth-gitlab'
gem 'omniauth-bitbucket'
gem 'doorkeeper', '2.1.0'
gem "rack-oauth2", "~> 1.0.5"
......
......@@ -338,6 +338,10 @@ GEM
omniauth (1.1.4)
hashie (>= 1.2, < 3)
rack
omniauth-bitbucket (0.0.2)
multi_json (~> 1.7)
omniauth (~> 1.1)
omniauth-oauth (~> 1.0)
omniauth-github (1.1.1)
omniauth (~> 1.0)
omniauth-oauth2 (~> 1.1)
......@@ -701,6 +705,7 @@ DEPENDENCIES
nprogress-rails
octokit (= 3.7.0)
omniauth (~> 1.1.3)
omniauth-bitbucket
omniauth-github
omniauth-gitlab
omniauth-google-oauth2
......
class Import::BitbucketController < Import::BaseController
before_filter :bitbucket_auth, except: :callback
# rescue_from OAuth::Error, with: :bitbucket_unauthorized
def callback
request_token = session.delete(:oauth_request_token)
raise "Session expired!" if request_token.nil?
request_token.symbolize_keys!
access_token = client.get_token(request_token, params[:oauth_verifier], callback_import_bitbucket_url)
current_user.bitbucket_access_token = access_token.token
current_user.bitbucket_access_token_secret = access_token.secret
current_user.save
redirect_to status_import_bitbucket_url
end
def status
@repos = client.projects
@already_added_projects = current_user.created_projects.where(import_type: "bitbucket")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject!{ |repo| already_added_projects_names.include? "#{repo["owner"]}/#{repo["slug"]}" }
end
def jobs
jobs = current_user.created_projects.where(import_type: "bitbucket").to_json(only: [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id] || ""
repo = client.project(@repo_id.gsub("___", "/"))
@target_namespace = params[:new_namespace].presence || repo["owner"]
@project_name = repo["slug"]
namespace = get_or_create_namespace || (render and return)
unless Gitlab::BitbucketImport::KeyAdder.new(repo, current_user).execute
@access_denied = true
render
return
end
@project = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::BitbucketImport::Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
end
def bitbucket_auth
if current_user.bitbucket_access_token.blank?
go_to_bitbucket_for_permissions
end
end
def go_to_bitbucket_for_permissions
request_token = client.request_token(callback_import_bitbucket_url)
session[:oauth_request_token] = request_token
redirect_to client.authorize_url(request_token, callback_import_bitbucket_url)
end
def bitbucket_unauthorized
go_to_bitbucket_for_permissions
end
end
......@@ -4,7 +4,7 @@ module OauthHelper
end
def default_providers
[:twitter, :github, :gitlab, :google_oauth2, :ldap]
[:twitter, :github, :gitlab, :bitbucket, :google_oauth2, :ldap]
end
def enabled_oauth_providers
......@@ -13,7 +13,7 @@ module OauthHelper
def enabled_social_providers
enabled_oauth_providers.select do |name|
[:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym)
[:twitter, :gitlab, :github, :bitbucket, :google_oauth2].include?(name.to_sym)
end
end
......
......@@ -273,4 +273,8 @@ module ProjectsHelper
def gitlab_import_enabled?
enabled_oauth_providers.include?(:gitlab)
end
def bitbucket_import_enabled?
enabled_oauth_providers.include?(:bitbucket)
end
end
......@@ -130,7 +130,7 @@ class Project < ActiveRecord::Base
validates_uniqueness_of :name, scope: :namespace_id
validates_uniqueness_of :path, scope: :namespace_id
validates :import_url,
format: { with: URI::regexp(%w(git http https)), message: 'should be a valid url' },
format: { with: URI::regexp(%w(ssh git http https)), message: 'should be a valid url' },
if: :import?
validates :star_count, numericality: { greater_than_or_equal_to: 0 }
validate :check_limit, on: :create
......
......@@ -46,6 +46,7 @@
# github_access_token :string(255)
# notification_email :string(255)
# password_automatically_set :boolean default(FALSE)
# bitbucket_access_token :string(255)
#
require 'carrierwave/orm/activerecord'
......
......@@ -10,9 +10,16 @@
target_field.append("/" + project_name)
target_field.data("project_name", project_name)
target_field.find('input').prop("value", origin_namespace)
- elsif @access_denied
:plain
job = $("tr#repo_#{@repo_id}")
job.find(".import-actions").html("<p class='alert alert-danger'>Access denied! Please verify you can add deploy keys to this repository.</p>"")
- else
:plain
job = $("tr#repo_#{@repo_id}")
job.attr("id", "project_#{@project.id}")
target_field = job.find(".import-target")
target_field.empty()
target_field.append('<strong>#{link_to @project.path_with_namespace, @project}</strong>')
$("table.import-jobs tbody").prepend(job)
job.addClass("active").find(".import-actions").html("<i class='fa fa-spinner fa-spin'></i> started")
%h3.page-title
%i.fa.fa-bitbucket
Import repositories from Bitbucket
%p.light
Select projects you want to import.
%hr
%p
= button_tag 'Import all projects', class: "btn btn-success js-import-all"
%table.table.import-jobs
%thead
%tr
%th From Bitbucket
%th To GitLab
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td= project.import_source
%td
%strong= link_to project.path_with_namespace, project
%td.job-status
- if project.import_status == 'finished'
%span.cgreen
%i.fa.fa-check
done
- elsif project.import_status == 'started'
%i.fa.fa-spinner.fa-spin
started
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"}
%td= "#{repo["owner"]}/#{repo["slug"]}"
%td.import-target
= "#{repo["owner"]}/#{repo["slug"]}"
%td.import-actions.job-status
= button_tag "Import", class: "btn js-add-to-import"
:coffeescript
$ ->
new ImporterStatus("#{jobs_import_bitbucket_path}", "#{import_bitbucket_path}")
%h3.page-title
%i.fa.fa-github
Import repositories from GitHub.com
Import repositories from GitHub
%p.light
Select projects you want to import.
......
%h3.page-title
%i.fa.fa-github
%i.fa.fa-heart
Import repositories from GitLab.com
%p.light
......
%div#bitbucket_import_modal.modal.hide
.modal-dialog
.modal-content
.modal-header
%a.close{href: "#", "data-dismiss" => "modal"} ×
%h3 GitHub OAuth import
.modal-body
You need to setup integration with Bitbucket first.
= link_to 'How to setup integration with Bitbucket', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/bitbucket.md'
\ No newline at end of file
......@@ -53,6 +53,19 @@
Import projects from GitHub
= render 'github_import_modal'
.project-import.form-group
.col-sm-2
.col-sm-10
- if bitbucket_import_enabled?
= link_to status_import_bitbucket_path do
%i.fa.fa-bitbucket
Import projects from Bitbucket
- else
= link_to '#', class: 'how_to_import_link light' do
%i.fa.fa-bitbucket
Import projects from Bitbucket
= render 'bitbucket_import_modal'
- unless request.host == 'gitlab.com'
.project-import.form-group
.col-sm-2
......
......@@ -14,6 +14,8 @@ class RepositoryImportWorker
Gitlab::GithubImport::Importer.new(project).execute
elsif project.import_type == 'gitlab'
Gitlab::GitlabImport::Importer.new(project).execute
elsif project.import_type == 'bitbucket'
Gitlab::BitbucketImport::Importer.new(project).execute
else
true
end
......
......@@ -207,17 +207,19 @@ production: &base
# arguments, followed by optional 'args' which can be either a hash or an array.
# Documentation for this is available at http://doc.gitlab.com/ce/integration/omniauth.html
providers:
# - { name: 'google_oauth2', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# - { name: 'google_oauth2', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET',
# args: { access_type: 'offline', approval_prompt: '' } }
# - { name: 'twitter', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET'}
# - { name: 'github', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# - { name: 'twitter', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET'}
# - { name: 'github', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET',
# args: { scope: 'user:email' } }
# - { name: 'gitlab', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# - { name: 'gitlab', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET',
# args: { scope: 'api' } }
# - { name: 'bitbucket', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET'}
......
......@@ -68,6 +68,11 @@ Gitlab::Application.routes.draw do
get :jobs
end
resource :bitbucket, only: [:create, :new], controller: :bitbucket do
get :status
get :callback
get :jobs
end
resource :gitorious, only: [:create, :new], controller: :gitorious do
get :status
get :callback
......
class AddBitbucketAccessTokenAndSecretToUser < ActiveRecord::Migration
def change
add_column :users, :bitbucket_access_token, :string
add_column :users, :bitbucket_access_token_secret, :string
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20150213121042) do
ActiveRecord::Schema.define(version: 20150217123345) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -454,6 +454,8 @@ ActiveRecord::Schema.define(version: 20150213121042) do
t.string "notification_email"
t.boolean "hide_no_password", default: false
t.boolean "password_automatically_set", default: false
t.string "bitbucket_access_token"
t.string "bitbucket_access_token_secret"
end
add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
......
TODO
\ No newline at end of file
module Gitlab
module BitbucketImport
class Client
attr_reader :consumer, :api
def initialize(access_token = nil, access_token_secret = nil)
@consumer = ::OAuth::Consumer.new(
config.app_id,
config.app_secret,
bitbucket_options
)
if access_token && access_token_secret
@api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret)
end
end
def request_token(redirect_uri)
request_token = consumer.get_request_token(oauth_callback: redirect_uri)
{
oauth_token: request_token.token,
oauth_token_secret: request_token.secret,
oauth_callback_confirmed: request_token.callback_confirmed?.to_s
}
end
def authorize_url(request_token, redirect_uri)
request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
if request_token.callback_confirmed?
request_token.authorize_url
else
request_token.authorize_url(oauth_callback: redirect_uri)
end
end
def get_token(request_token, oauth_verifier, redirect_uri)
request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
if request_token.callback_confirmed?
request_token.get_access_token(oauth_verifier: oauth_verifier)
else
request_token.get_access_token(oauth_callback: redirect_uri)
end
end
def user
JSON.parse(api.get("/api/1.0/user").body)
end
def issues(project_identifier)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues").body)
end
def issue_comments(project_identifier, issue_id)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
end
def project(project_identifier)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}").body)
end
def deploy_key(project_identifier)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find { |key| key["label"] =~ /GitLab/ }
end
def add_deploy_key(project_identifier, key)
JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body)
end
def projects
JSON.parse(api.get("/api/1.0/user/repositories").body).
select { |repo| repo["scm"] == "git" }
end
private
def config
Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
end
def bitbucket_options
OmniAuth::Strategies::Bitbucket.default_options[:client_options]
end
end
end
end
module Gitlab
module BitbucketImport
class Importer
attr_reader :project, :client
def initialize(project)
@project = project
@client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret)
@formatter = Gitlab::ImportFormatter.new
end
def execute
project_identifier = project.import_source
return true unless client.project(project_identifier)["has_issues"]
#Issues && Comments
issues = client.issues(project_identifier)
issues["issues"].each do |issue|
body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
comments = client.issue_comments(project_identifier, issue["local_id"])
if comments.any?
body += @formatter.comments_header
end
comments.each do |comment|
body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"])
end
project.issues.create!(
description: body,
title: issue["title"],
state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
author_id: gl_user_id(project, issue["reported_by"]["username"])
)
end
true
end
private
def gl_user_id(project, bitbucket_id)
user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
(user && user.id) || project.creator_id
end
end
end
end
module Gitlab
module BitbucketImport
class KeyAdder
attr_reader :repo, :current_user, :client
def initialize(repo, current_user)
@repo, @current_user = repo, current_user
@client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
end
def execute
project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
return true if client.deploy_key(project_identifier)
# TODO: Point to actual public key.
client.add_deploy_key(project_identifier, File.read("/Users/douwemaan/.ssh/id_rsa.pub"))
true
end
end
end
end
module Gitlab
module BitbucketImport
class ProjectCreator
attr_reader :repo, :namespace, :current_user
def initialize(repo, namespace, current_user)
@repo = repo
@namespace = namespace
@current_user = current_user
end
def execute
@project = Project.new(
name: repo["name"],
path: repo["slug"],
description: repo["description"],
namespace: namespace,
creator: current_user,
visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
import_type: "bitbucket",
import_source: "#{repo["owner"]}/#{repo["slug"]}",
import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git"
)
if @project.save!
@project.reload
if @project.import_failed?
@project.import_retry
else
@project.import_start
end
end
@project
end
end
end
end
......@@ -46,11 +46,7 @@ module Gitlab
end
def github_options
{
site: 'https://api.github.com',
authorize_url: 'https://github.com/login/oauth/authorize',
token_url: 'https://github.com/login/oauth/access_token'
}
OmniAuth::Strategies::GitHub.default_options[:client_options]
end
end
end
......
......@@ -20,7 +20,7 @@ module Gitlab
body += @formatter.comments_header
client.issue_comments(project.import_source, issue.number).each do |c|
body += @formatter.comment_to_md(c.user.login, c.created_at, c.body)
body += @formatter.comment(c.user.login, c.created_at, c.body)
end
end
......
......@@ -9,7 +9,7 @@ module Gitlab
@client = ::OAuth2::Client.new(
config.app_id,
config.app_secret,
github_options
gitlab_options
)
if access_token
......@@ -70,12 +70,8 @@ module Gitlab
Gitlab.config.omniauth.providers.find{|provider| provider.name == "gitlab"}
end
def github_options
{
site: 'https://gitlab.com/',
authorize_url: 'oauth/authorize',
token_url: 'oauth/token'
}
def gitlab_options
OmniAuth::Strategies::GitLab.default_options[:client_options]
end
end
end
......
......@@ -25,7 +25,7 @@ module Gitlab
end
comments.each do |comment|
body += @formatter.comment_to_md(comment["author"]["name"], comment["created_at"], comment["body"])
body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"])
end
project.issues.create!(
......
module Gitlab
class ImportFormatter
def comment_to_md(author, date, body)
def comment(author, date, body)
"\n\n*By #{author} on #{date}*\n\n#{body}"
end
......
require 'spec_helper'
describe Import::BitbucketController do
let(:user) { create(:user, bitbucket_access_token: 'asd123', bitbucket_access_token_secret: "sekret") }
before do
sign_in(user)
end
describe "GET callback" do
before do
session[:oauth_request_token] = {}
end
it "updates access token" do
token = "asdasd12345"
secret = "sekrettt"
access_token = double(token: token, secret: secret)
Gitlab::BitbucketImport::Client.any_instance.stub(:get_token).and_return(access_token)
Gitlab.config.omniauth.providers << OpenStruct.new(app_id: "asd123", app_secret: "asd123", name: "bitbucket")
get :callback
expect(user.reload.bitbucket_access_token).to eq(token)
expect(user.reload.bitbucket_access_token_secret).to eq(secret)
expect(controller).to redirect_to(status_import_bitbucket_url)
end
end
describe "GET status" do
before do
@repo = OpenStruct.new(slug: 'vim', owner: 'asd')
end
it "assigns variables" do
@project = create(:project, import_type: 'bitbucket', creator_id: user.id)
controller.stub_chain(:client, :projects).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([@repo])
end
it "does not show already added project" do
@project = create(:project, import_type: 'bitbucket', creator_id: user.id, import_source: 'asd/vim')
controller.stub_chain(:client, :projects).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([])
end
end
describe "POST create" do
before do
@repo = {
slug: 'vim',
owner: "john"
}.with_indifferent_access
end
it "takes already existing namespace" do
namespace = create(:namespace, name: "john", owner: user)
expect(Gitlab::BitbucketImport::KeyAdder).
to receive(:new).with(@repo, user).
and_return(double(execute: true))
expect(Gitlab::BitbucketImport::ProjectCreator).
to receive(:new).with(@repo, namespace, user).
and_return(double(execute: true))
controller.stub_chain(:client, :project).and_return(@repo)
post :create, format: :js
end
end
end
require 'spec_helper'
describe Gitlab::BitbucketImport::ProjectCreator do
let(:user) { create(:user, bitbucket_access_token: "asdffg", bitbucket_access_token_secret: "sekret") }
let(:repo) { {
name: 'Vim',
slug: 'vim',
is_private: true,
owner: "asd"}.with_indifferent_access
}
let(:namespace){ create(:namespace) }
it 'creates project' do
allow_any_instance_of(Project).to receive(:add_import_job)
project_creator = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, user)
project = project_creator.execute
expect(project.import_url).to eq("ssh://git@bitbucket.org/asd/vim.git")
expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)
end
end
require 'spec_helper'
describe Gitlab::Github::ProjectCreator do
describe Gitlab::GithubImport::ProjectCreator do
let(:user) { create(:user, github_access_token: "asdffg") }
let(:repo) { OpenStruct.new(
login: 'vim',
......@@ -15,9 +15,8 @@ describe Gitlab::Github::ProjectCreator do
it 'creates project' do
allow_any_instance_of(Project).to receive(:add_import_job)
project_creator = Gitlab::Github::ProjectCreator.new(repo, namespace, user)
project_creator.execute
project = Project.last
project_creator = Gitlab::GithubImport::ProjectCreator.new(repo, namespace, user)
project = project_creator.execute
expect(project.import_url).to eq("https://asdffg@gitlab.com/asd/vim.git")
expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)
......
......@@ -2,7 +2,7 @@ require 'spec_helper'
describe Gitlab::GitlabImport::ProjectCreator do
let(:user) { create(:user, gitlab_access_token: "asdffg") }
let(:repo) {{
let(:repo) { {
name: 'vim',
path: 'vim',
visibility_level: Gitlab::VisibilityLevel::PRIVATE,
......@@ -16,8 +16,7 @@ describe Gitlab::GitlabImport::ProjectCreator do
allow_any_instance_of(Project).to receive(:add_import_job)
project_creator = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, user)
project_creator.execute
project = Project.last
project = project_creator.execute
expect(project.import_url).to eq("https://oauth2:asdffg@gitlab.com/asd/vim.git")
expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment