Commit 5194214e authored by Valery Sizov's avatar Valery Sizov

GitLab integration. Importer

parent 0a9cab4e
...@@ -29,6 +29,7 @@ gem 'omniauth-twitter' ...@@ -29,6 +29,7 @@ gem 'omniauth-twitter'
gem 'omniauth-github' gem 'omniauth-github'
gem 'omniauth-shibboleth' gem 'omniauth-shibboleth'
gem 'omniauth-kerberos' gem 'omniauth-kerberos'
gem 'omniauth-gitlab'
gem 'doorkeeper', '2.1.0' gem 'doorkeeper', '2.1.0'
gem "rack-oauth2", "~> 1.0.5" gem "rack-oauth2", "~> 1.0.5"
......
...@@ -332,6 +332,9 @@ GEM ...@@ -332,6 +332,9 @@ GEM
omniauth-github (1.1.1) omniauth-github (1.1.1)
omniauth (~> 1.0) omniauth (~> 1.0)
omniauth-oauth2 (~> 1.1) omniauth-oauth2 (~> 1.1)
omniauth-gitlab (1.0.0)
omniauth (~> 1.0)
omniauth-oauth2 (~> 1.0)
omniauth-google-oauth2 (0.2.5) omniauth-google-oauth2 (0.2.5)
omniauth (> 1.0) omniauth (> 1.0)
omniauth-oauth2 (~> 1.1) omniauth-oauth2 (~> 1.1)
...@@ -689,6 +692,7 @@ DEPENDENCIES ...@@ -689,6 +692,7 @@ DEPENDENCIES
octokit (= 3.7.0) octokit (= 3.7.0)
omniauth (~> 1.1.3) omniauth (~> 1.1.3)
omniauth-github omniauth-github
omniauth-gitlab
omniauth-google-oauth2 omniauth-google-oauth2
omniauth-kerberos omniauth-kerberos
omniauth-shibboleth omniauth-shibboleth
......
class GithubImportsController < ApplicationController class Importers::GithubsController < ApplicationController
before_filter :github_auth, except: :callback before_filter :github_auth, except: :callback
rescue_from Octokit::Unauthorized, with: :github_unauthorized rescue_from Octokit::Unauthorized, with: :github_unauthorized
...@@ -7,7 +7,7 @@ class GithubImportsController < ApplicationController ...@@ -7,7 +7,7 @@ class GithubImportsController < ApplicationController
token = client.auth_code.get_token(params[:code]).token token = client.auth_code.get_token(params[:code]).token
current_user.github_access_token = token current_user.github_access_token = token
current_user.save current_user.save
redirect_to status_github_import_url redirect_to status_importers_github_url
end end
def status def status
...@@ -69,7 +69,7 @@ class GithubImportsController < ApplicationController ...@@ -69,7 +69,7 @@ class GithubImportsController < ApplicationController
def go_to_github_for_permissions def go_to_github_for_permissions
redirect_to client.auth_code.authorize_url({ redirect_to client.auth_code.authorize_url({
redirect_uri: callback_github_import_url, redirect_uri: callback_importers_github_url,
scope: "repo, user, user:email" scope: "repo, user, user:email"
}) })
end end
......
class Importers::GitlabsController < ApplicationController
before_filter :gitlab_auth, except: :callback
rescue_from OAuth2::Error, with: :gitlab_unauthorized
def callback
token = client.get_token(params[:code], callback_importers_gitlab_url)
current_user.gitlab_access_token = token
current_user.save
redirect_to status_importers_gitlab_url
end
def status
@repos = client.projects
@already_added_projects = current_user.created_projects.where(import_type: "gitlab")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject!{|repo| already_added_projects_names.include? repo["path_with_namespace"]}
end
def jobs
jobs = current_user.created_projects.where(import_type: "gitlab").to_json(:only => [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id].to_i
repo = client.project(@repo_id)
target_namespace = params[:new_namespace].presence || repo["namespace"]["path"]
existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace)
if existing_namespace
if existing_namespace.owner == current_user
namespace = existing_namespace
else
@already_been_taken = true
@target_namespace = target_namespace
@project_name = repo["path"]
render and return
end
else
namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user)
namespace.add_owner(current_user)
end
@project = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::GitlabImport::Client.new(current_user.gitlab_access_token)
end
def gitlab_auth
if current_user.gitlab_access_token.blank?
go_to_gitlab_for_permissions
end
end
def go_to_gitlab_for_permissions
redirect_to client.authorize_url(callback_importers_gitlab_url)
end
def gitlab_unauthorized
go_to_gitlab_for_permissions
end
end
...@@ -4,7 +4,7 @@ module OauthHelper ...@@ -4,7 +4,7 @@ module OauthHelper
end end
def default_providers def default_providers
[:twitter, :github, :google_oauth2, :ldap] [:twitter, :github, :gitlab, :google_oauth2, :ldap]
end end
def enabled_oauth_providers def enabled_oauth_providers
...@@ -13,7 +13,7 @@ module OauthHelper ...@@ -13,7 +13,7 @@ module OauthHelper
def enabled_social_providers def enabled_social_providers
enabled_oauth_providers.select do |name| enabled_oauth_providers.select do |name|
[:twitter, :github, :google_oauth2].include?(name.to_sym) [:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym)
end end
end end
......
...@@ -253,4 +253,8 @@ module ProjectsHelper ...@@ -253,4 +253,8 @@ module ProjectsHelper
def github_import_enabled? def github_import_enabled?
enabled_oauth_providers.include?(:github) enabled_oauth_providers.include?(:github)
end end
def gitlab_import_enabled?
enabled_oauth_providers.include?(:gitlab)
end
end end
...@@ -43,11 +43,11 @@ ...@@ -43,11 +43,11 @@
if tr.find(".import-target input").length > 0 if tr.find(".import-target input").length > 0
new_namespace = tr.find(".import-target input").prop("value") new_namespace = tr.find(".import-target input").prop("value")
tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name")) tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
$.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script' $.post "#{importers_github_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
setInterval (-> setInterval (->
$.get "#{jobs_github_import_path}", (data)-> $.get "#{jobs_importers_github_path}", (data)->
$.each data, (i, job) -> $.each data, (i, job) ->
job_item = $("#project_" + job.id) job_item = $("#project_" + job.id)
status_field = job_item.find(".job-status") status_field = job_item.find(".job-status")
......
- if @already_been_taken
:plain
target_field = $("tr#repo_#{@repo_id} .import-target")
origin_target = target_field.text()
project_name = "#{@project_name}"
origin_namespace = "#{@target_namespace}"
target_field.empty()
target_field.append("<p class='alert alert-danger'>This namespace already been taken! Please choose another one</p>")
target_field.append("<input type='text' name='target_namespace' />")
target_field.append("/" + project_name)
target_field.data("project_name", project_name)
target_field.find('input').prop("value", origin_namespace)
- else
:plain
job = $("tr#repo_#{@repo_id}")
job.attr("id", "project_#{@project.id}")
$("table.import-jobs tbody").prepend(job)
job.addClass("active").find(".import-actions").html("<i class='fa fa-spinner fa-spin'></i> started")
%h3.page-title
%i.fa.fa-github
Import repositories from GitLab.com
%p.light
Select projects you want to import.
%hr
%table.table.import-jobs
%thead
%tr
%th From GitLab.com
%th To GitLab private instance
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td= project.import_source
%td
%strong= link_to project.name_with_namespace, project
%td.job-status
- if project.import_status == 'finished'
%span.cgreen
%i.fa.fa-check
done
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo["id"]}"}
%td= repo["path_with_namespace"]
%td.import-target
= repo["path_with_namespace"]
%td.import-actions.job-status
= button_tag "Add", class: "btn btn-add-to-import"
:coffeescript
$(".btn-add-to-import").click () ->
new_namespace = null
tr = $(this).closest("tr")
id = tr.attr("id").replace("repo_", "")
if tr.find(".import-target input").length > 0
new_namespace = tr.find(".import-target input").prop("value")
tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
$.post "#{importers_gitlab_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
setInterval (->
$.get "#{jobs_importers_gitlab_path}", (data)->
$.each data, (i, job) ->
job_item = $("#project_" + job.id)
status_field = job_item.find(".job-status")
if job.import_status == 'finished'
job_item.removeClass("active").addClass("success")
status_field.html('<span class="cgreen"><i class="fa fa-check"></i> done</span>')
else if job.import_status == 'started'
status_field.html("<i class='fa fa-spinner fa-spin'></i> started")
else
status_field.html(job.import_status)
), 4000
%div#gitlab_import_modal.modal.hide
.modal-dialog
.modal-content
.modal-header
%a.close{href: "#", "data-dismiss" => "modal"} ×
%h3 GitLab OAuth import
.modal-body
You need to setup integration with GitLab first.
= link_to 'How to setup integration with GitLab', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/gitlab.md'
:javascript
$(function(){
var import_modal = $('#gitlab_import_modal').modal({modal: true, show:false});
$('.how_to_import_link').bind("click", function(e){
e.preventDefault();
import_modal.show();
});
$('.modal-header .close').bind("click", function(){
import_modal.hide();
})
})
...@@ -44,7 +44,7 @@ ...@@ -44,7 +44,7 @@
.col-sm-2 .col-sm-2
.col-sm-10 .col-sm-10
- if github_import_enabled? - if github_import_enabled?
= link_to status_github_import_path do = link_to status_importers_github_path do
%i.fa.fa-github %i.fa.fa-github
Import projects from GitHub Import projects from GitHub
- else - else
...@@ -53,6 +53,19 @@ ...@@ -53,6 +53,19 @@
Import projects from GitHub Import projects from GitHub
= render 'github_import_modal' = render 'github_import_modal'
.project-import.form-group
.col-sm-2
.col-sm-10
- if gitlab_import_enabled?
= link_to status_importers_gitlab_path do
%i.fa.fa-heart
Import projects from GitLab.com
- else
= link_to '#', class: 'how_to_import_link light' do
%i.fa.fa-heart
Import projects from GitLab.com
= render 'gitlab_import_modal'
%hr.prepend-botton-10 %hr.prepend-botton-10
.form-group .form-group
......
...@@ -12,6 +12,8 @@ class RepositoryImportWorker ...@@ -12,6 +12,8 @@ class RepositoryImportWorker
if project.import_type == 'github' if project.import_type == 'github'
result_of_data_import = Gitlab::Github::Importer.new(project).execute result_of_data_import = Gitlab::Github::Importer.new(project).execute
elsif project.import_type == 'gitlab'
result_of_data_import = Gitlab::GitlabImport::Importer.new(project).execute
else else
result_of_data_import = true result_of_data_import = true
end end
......
...@@ -27,7 +27,7 @@ Doorkeeper.configure do ...@@ -27,7 +27,7 @@ Doorkeeper.configure do
# Access token expiration time (default 2 hours). # Access token expiration time (default 2 hours).
# If you want to disable expiration, set this to nil. # If you want to disable expiration, set this to nil.
# access_token_expires_in 2.hours access_token_expires_in nil
# Reuse access token for the same resource owner within an application (disabled by default) # Reuse access token for the same resource owner within an application (disabled by default)
# Rationale: https://github.com/doorkeeper-gem/doorkeeper/issues/383 # Rationale: https://github.com/doorkeeper-gem/doorkeeper/issues/383
......
...@@ -51,15 +51,26 @@ Gitlab::Application.routes.draw do ...@@ -51,15 +51,26 @@ Gitlab::Application.routes.draw do
end end
get '/s/:username' => 'snippets#user_index', as: :user_snippets, constraints: { username: /.*/ } get '/s/:username' => 'snippets#user_index', as: :user_snippets, constraints: { username: /.*/ }
# #
# Github importer area # Importers
# #
resource :github_import, only: [:create, :new] do namespace :importers do
resource :github, only: [:create, :new] do
get :status get :status
get :callback get :callback
get :jobs get :jobs
end end
resource :gitlab, only: [:create, :new] do
get :status
get :callback
get :jobs
end
end
# #
# Explore area # Explore area
# #
......
# GitLab OAuth2 OmniAuth Provider
To enable the GitLab OmniAuth provider you must register your application with GitLab. GitLab will generate a client ID and secret key for you to use.
1. Sign in to GitLab.
1. Navigate to your settings.
1. Select "Applications" in the left menu.
1. Select "New application".
1. Provide the required details.
- Name: This can be anything. Consider something like "\<Organization\>'s GitLab" or "\<Your Name\>'s GitLab" or something else descriptive.
- Redirect URI:
```
http://gitlab.example.com/importers/gitlab/callback
http://gitlab.example.com/users/auth/gitlab/callback
```
The first link is required for the importer and second for the authorization.
1. Select "Submit".
1. You should now see a Application ID and Secret. Keep this page open as you continue configuration.
1. On your GitLab server, open the configuration file.
```sh
cd /home/git/gitlab
sudo -u git -H editor config/gitlab.yml
```
1. Find the section dealing with OmniAuth. See [Initial OmniAuth Configuration](README.md#initial-omniauth-configuration) for more details.
1. Under `providers:` uncomment (or add) lines that look like the following:
```
- { name: 'gitlab', app_id: 'YOUR APP ID',
app_secret: 'YOUR APP SECRET',
args: { scope: 'api' } }
```
1. Change 'YOUR APP ID' to the Application ID from the GitLab application page.
1. Change 'YOUR APP SECRET' to the secret from the GitLab application page.
1. Save the configuration file.
1. Restart GitLab for the changes to take effect.
On the sign in page there should now be a GitLab icon below the regular sign in form. Click the icon to begin the authentication process. GitLab will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to your GitLab instance and will be signed in.
module Gitlab
module GitlabImport
class Client
attr_reader :client, :api
PER_PAGE = 100
def initialize(access_token)
@client = ::OAuth2::Client.new(
config.app_id,
config.app_secret,
github_options
)
if access_token
@api = OAuth2::AccessToken.from_hash(@client, :access_token => access_token)
end
end
def authorize_url(redirect_uri)
client.auth_code.authorize_url({
redirect_uri: redirect_uri,
scope: "api"
})
end
def get_token(code, redirect_uri)
client.auth_code.get_token(code, redirect_uri: redirect_uri).token
end
def issues(project_identifier)
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
end
end
def issue_comments(project_identifier, issue_id)
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
end
end
def project(id)
api.get("/api/v3/projects/#{id}").parsed
end
def projects
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
end
end
private
def lazy_page_iterator(per_page)
Enumerator.new do |y|
page = 1
loop do
items = yield(page)
items.each do |item|
y << item
end
break if items.empty? || items.size < per_page
page += 1
end
end
end
def config
Gitlab.config.omniauth.providers.select{|provider| provider.name == "gitlab"}.first
end
def github_options
{
site: 'https://gitlab.com/',
authorize_url: 'oauth/authorize',
token_url: 'oauth/token'
}
end
end
end
end
module Gitlab
module GitlabImport
class Importer
attr_reader :project, :client
def initialize(project)
@project = project
@client = Client.new(project.creator.gitlab_access_token)
end
def execute
project_identifier = URI.encode(project.import_source, '/')
#Issues && Comments
issues = client.issues(project_identifier)
issues.each do |issue|
body = "*Created by: #{issue["author"]["name"]}*\n\n#{issue["description"]}"
comments = client.issue_comments(project_identifier, issue["id"])
if comments.any?
body += "\n\n\n**Imported comments:**\n"
end
comments.each do |comment|
body += "\n\n*By #{comment["author"]["name"]} on #{comment["created_at"]}*\n\n#{comment["body"]}"
end
project.issues.create!(
description: body,
title: issue["title"],
state: issue["state"],
author_id: gl_user_id(project, issue["author"]["id"])
)
end
true
end
private
def gl_user_id(project, gitlab_id)
user = User.joins(:identities).find_by("identities.extern_uid = ?", gitlab_id.to_s)
(user && user.id) || project.creator_id
end
end
end
end
module Gitlab
module GitlabImport
class ProjectCreator
attr_reader :repo, :namespace, :current_user
def initialize(repo, namespace, current_user)
@repo = repo
@namespace = namespace
@current_user = current_user
end
def execute
@project = Project.new(
name: repo["name"],
path: repo["path"],
description: repo["description"],
namespace: namespace,
creator: current_user,
visibility_level: repo["visibility_level"],
import_type: "gitlab",
import_source: repo["path_with_namespace"],
import_url: repo["http_url_to_repo"]#.sub("://", "://oauth2@#{current_user.gitlab_access_token}")
)
if @project.save!
@project.reload
if @project.import_failed?
@project.import_retry
else
@project.import_start
end
end
@project
end
end
end
end
require 'spec_helper' require 'spec_helper'
describe GithubImportsController do describe Importers::GithubsController do
let(:user) { create(:user, github_access_token: 'asd123') } let(:user) { create(:user, github_access_token: 'asd123') }
before do before do
...@@ -16,7 +16,7 @@ describe GithubImportsController do ...@@ -16,7 +16,7 @@ describe GithubImportsController do
get :callback get :callback
user.reload.github_access_token.should == token user.reload.github_access_token.should == token
controller.should redirect_to(status_github_import_url) controller.should redirect_to(status_importers_github_url)
end end
end end
......
require 'spec_helper'
describe Importers::GitlabsController do
let(:user) { create(:user, gitlab_access_token: 'asd123') }
before do
sign_in(user)
end
describe "GET callback" do
it "updates access token" do
token = "asdasd12345"
Gitlab::GitlabImport::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token)
Gitlab.config.omniauth.providers << OpenStruct.new(app_id: "asd123", app_secret: "asd123", name: "gitlab")
get :callback
user.reload.gitlab_access_token.should == token
controller.should redirect_to(status_importers_gitlab_url)
end
end
describe "GET status" do
before do
@repo = OpenStruct.new(path: 'vim', path_with_namespace: 'asd/vim')
end
it "assigns variables" do
@project = create(:project, import_type: 'gitlab', creator_id: user.id)
controller.stub_chain(:client, :projects).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([@repo])
end
it "does not show already added project" do
@project = create(:project, import_type: 'gitlab', creator_id: user.id, import_source: 'asd/vim')
controller.stub_chain(:client, :projects).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([])
end
end
describe "POST create" do
before do
@repo = {
path: 'vim',
path_with_namespace: 'asd/vim',
owner: {name: "john"},
namespace: {path: "john"}
}.with_indifferent_access
end
it "takes already existing namespace" do
namespace = create(:namespace, name: "john", owner: user)
Gitlab::GitlabImport::ProjectCreator.should_receive(:new).with(@repo, namespace, user).
and_return(double(execute: true))
controller.stub_chain(:client, :project).and_return(@repo)
post :create, format: :js
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment