Commit 5194214e authored by Valery Sizov's avatar Valery Sizov

GitLab integration. Importer

parent 0a9cab4e
......@@ -29,6 +29,7 @@ gem 'omniauth-twitter'
gem 'omniauth-github'
gem 'omniauth-shibboleth'
gem 'omniauth-kerberos'
gem 'omniauth-gitlab'
gem 'doorkeeper', '2.1.0'
gem "rack-oauth2", "~> 1.0.5"
......
......@@ -332,6 +332,9 @@ GEM
omniauth-github (1.1.1)
omniauth (~> 1.0)
omniauth-oauth2 (~> 1.1)
omniauth-gitlab (1.0.0)
omniauth (~> 1.0)
omniauth-oauth2 (~> 1.0)
omniauth-google-oauth2 (0.2.5)
omniauth (> 1.0)
omniauth-oauth2 (~> 1.1)
......@@ -689,6 +692,7 @@ DEPENDENCIES
octokit (= 3.7.0)
omniauth (~> 1.1.3)
omniauth-github
omniauth-gitlab
omniauth-google-oauth2
omniauth-kerberos
omniauth-shibboleth
......
class GithubImportsController < ApplicationController
class Importers::GithubsController < ApplicationController
before_filter :github_auth, except: :callback
rescue_from Octokit::Unauthorized, with: :github_unauthorized
......@@ -7,7 +7,7 @@ class GithubImportsController < ApplicationController
token = client.auth_code.get_token(params[:code]).token
current_user.github_access_token = token
current_user.save
redirect_to status_github_import_url
redirect_to status_importers_github_url
end
def status
......@@ -69,7 +69,7 @@ class GithubImportsController < ApplicationController
def go_to_github_for_permissions
redirect_to client.auth_code.authorize_url({
redirect_uri: callback_github_import_url,
redirect_uri: callback_importers_github_url,
scope: "repo, user, user:email"
})
end
......
class Importers::GitlabsController < ApplicationController
before_filter :gitlab_auth, except: :callback
rescue_from OAuth2::Error, with: :gitlab_unauthorized
def callback
token = client.get_token(params[:code], callback_importers_gitlab_url)
current_user.gitlab_access_token = token
current_user.save
redirect_to status_importers_gitlab_url
end
def status
@repos = client.projects
@already_added_projects = current_user.created_projects.where(import_type: "gitlab")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject!{|repo| already_added_projects_names.include? repo["path_with_namespace"]}
end
def jobs
jobs = current_user.created_projects.where(import_type: "gitlab").to_json(:only => [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id].to_i
repo = client.project(@repo_id)
target_namespace = params[:new_namespace].presence || repo["namespace"]["path"]
existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace)
if existing_namespace
if existing_namespace.owner == current_user
namespace = existing_namespace
else
@already_been_taken = true
@target_namespace = target_namespace
@project_name = repo["path"]
render and return
end
else
namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user)
namespace.add_owner(current_user)
end
@project = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::GitlabImport::Client.new(current_user.gitlab_access_token)
end
def gitlab_auth
if current_user.gitlab_access_token.blank?
go_to_gitlab_for_permissions
end
end
def go_to_gitlab_for_permissions
redirect_to client.authorize_url(callback_importers_gitlab_url)
end
def gitlab_unauthorized
go_to_gitlab_for_permissions
end
end
......@@ -4,7 +4,7 @@ module OauthHelper
end
def default_providers
[:twitter, :github, :google_oauth2, :ldap]
[:twitter, :github, :gitlab, :google_oauth2, :ldap]
end
def enabled_oauth_providers
......@@ -13,7 +13,7 @@ module OauthHelper
def enabled_social_providers
enabled_oauth_providers.select do |name|
[:twitter, :github, :google_oauth2].include?(name.to_sym)
[:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym)
end
end
......
......@@ -253,4 +253,8 @@ module ProjectsHelper
def github_import_enabled?
enabled_oauth_providers.include?(:github)
end
def gitlab_import_enabled?
enabled_oauth_providers.include?(:gitlab)
end
end
......@@ -43,11 +43,11 @@
if tr.find(".import-target input").length > 0
new_namespace = tr.find(".import-target input").prop("value")
tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
$.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
$.post "#{importers_github_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
setInterval (->
$.get "#{jobs_github_import_path}", (data)->
$.get "#{jobs_importers_github_path}", (data)->
$.each data, (i, job) ->
job_item = $("#project_" + job.id)
status_field = job_item.find(".job-status")
......
- if @already_been_taken
:plain
target_field = $("tr#repo_#{@repo_id} .import-target")
origin_target = target_field.text()
project_name = "#{@project_name}"
origin_namespace = "#{@target_namespace}"
target_field.empty()
target_field.append("<p class='alert alert-danger'>This namespace already been taken! Please choose another one</p>")
target_field.append("<input type='text' name='target_namespace' />")
target_field.append("/" + project_name)
target_field.data("project_name", project_name)
target_field.find('input').prop("value", origin_namespace)
- else
:plain
job = $("tr#repo_#{@repo_id}")
job.attr("id", "project_#{@project.id}")
$("table.import-jobs tbody").prepend(job)
job.addClass("active").find(".import-actions").html("<i class='fa fa-spinner fa-spin'></i> started")
%h3.page-title
%i.fa.fa-github
Import repositories from GitLab.com
%p.light
Select projects you want to import.
%hr
%table.table.import-jobs
%thead
%tr
%th From GitLab.com
%th To GitLab private instance
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td= project.import_source
%td
%strong= link_to project.name_with_namespace, project
%td.job-status
- if project.import_status == 'finished'
%span.cgreen
%i.fa.fa-check
done
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo["id"]}"}
%td= repo["path_with_namespace"]
%td.import-target
= repo["path_with_namespace"]
%td.import-actions.job-status
= button_tag "Add", class: "btn btn-add-to-import"
:coffeescript
$(".btn-add-to-import").click () ->
new_namespace = null
tr = $(this).closest("tr")
id = tr.attr("id").replace("repo_", "")
if tr.find(".import-target input").length > 0
new_namespace = tr.find(".import-target input").prop("value")
tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
$.post "#{importers_gitlab_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
setInterval (->
$.get "#{jobs_importers_gitlab_path}", (data)->
$.each data, (i, job) ->
job_item = $("#project_" + job.id)
status_field = job_item.find(".job-status")
if job.import_status == 'finished'
job_item.removeClass("active").addClass("success")
status_field.html('<span class="cgreen"><i class="fa fa-check"></i> done</span>')
else if job.import_status == 'started'
status_field.html("<i class='fa fa-spinner fa-spin'></i> started")
else
status_field.html(job.import_status)
), 4000
%div#gitlab_import_modal.modal.hide
.modal-dialog
.modal-content
.modal-header
%a.close{href: "#", "data-dismiss" => "modal"} ×
%h3 GitLab OAuth import
.modal-body
You need to setup integration with GitLab first.
= link_to 'How to setup integration with GitLab', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/gitlab.md'
:javascript
$(function(){
var import_modal = $('#gitlab_import_modal').modal({modal: true, show:false});
$('.how_to_import_link').bind("click", function(e){
e.preventDefault();
import_modal.show();
});
$('.modal-header .close').bind("click", function(){
import_modal.hide();
})
})
......@@ -44,7 +44,7 @@
.col-sm-2
.col-sm-10
- if github_import_enabled?
= link_to status_github_import_path do
= link_to status_importers_github_path do
%i.fa.fa-github
Import projects from GitHub
- else
......@@ -53,6 +53,19 @@
Import projects from GitHub
= render 'github_import_modal'
.project-import.form-group
.col-sm-2
.col-sm-10
- if gitlab_import_enabled?
= link_to status_importers_gitlab_path do
%i.fa.fa-heart
Import projects from GitLab.com
- else
= link_to '#', class: 'how_to_import_link light' do
%i.fa.fa-heart
Import projects from GitLab.com
= render 'gitlab_import_modal'
%hr.prepend-botton-10
.form-group
......
......@@ -12,6 +12,8 @@ class RepositoryImportWorker
if project.import_type == 'github'
result_of_data_import = Gitlab::Github::Importer.new(project).execute
elsif project.import_type == 'gitlab'
result_of_data_import = Gitlab::GitlabImport::Importer.new(project).execute
else
result_of_data_import = true
end
......
......@@ -27,7 +27,7 @@ Doorkeeper.configure do
# Access token expiration time (default 2 hours).
# If you want to disable expiration, set this to nil.
# access_token_expires_in 2.hours
access_token_expires_in nil
# Reuse access token for the same resource owner within an application (disabled by default)
# Rationale: https://github.com/doorkeeper-gem/doorkeeper/issues/383
......
......@@ -51,15 +51,26 @@ Gitlab::Application.routes.draw do
end
get '/s/:username' => 'snippets#user_index', as: :user_snippets, constraints: { username: /.*/ }
#
# Github importer area
# Importers
#
resource :github_import, only: [:create, :new] do
namespace :importers do
resource :github, only: [:create, :new] do
get :status
get :callback
get :jobs
end
resource :gitlab, only: [:create, :new] do
get :status
get :callback
get :jobs
end
end
#
# Explore area
#
......
# GitLab OAuth2 OmniAuth Provider
To enable the GitLab OmniAuth provider you must register your application with GitLab. GitLab will generate a client ID and secret key for you to use.
1. Sign in to GitLab.
1. Navigate to your settings.
1. Select "Applications" in the left menu.
1. Select "New application".
1. Provide the required details.
- Name: This can be anything. Consider something like "\<Organization\>'s GitLab" or "\<Your Name\>'s GitLab" or something else descriptive.
- Redirect URI:
```
http://gitlab.example.com/importers/gitlab/callback
http://gitlab.example.com/users/auth/gitlab/callback
```
The first link is required for the importer and second for the authorization.
1. Select "Submit".
1. You should now see a Application ID and Secret. Keep this page open as you continue configuration.
1. On your GitLab server, open the configuration file.
```sh
cd /home/git/gitlab
sudo -u git -H editor config/gitlab.yml
```
1. Find the section dealing with OmniAuth. See [Initial OmniAuth Configuration](README.md#initial-omniauth-configuration) for more details.
1. Under `providers:` uncomment (or add) lines that look like the following:
```
- { name: 'gitlab', app_id: 'YOUR APP ID',
app_secret: 'YOUR APP SECRET',
args: { scope: 'api' } }
```
1. Change 'YOUR APP ID' to the Application ID from the GitLab application page.
1. Change 'YOUR APP SECRET' to the secret from the GitLab application page.
1. Save the configuration file.
1. Restart GitLab for the changes to take effect.
On the sign in page there should now be a GitLab icon below the regular sign in form. Click the icon to begin the authentication process. GitLab will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to your GitLab instance and will be signed in.
module Gitlab
module GitlabImport
class Client
attr_reader :client, :api
PER_PAGE = 100
def initialize(access_token)
@client = ::OAuth2::Client.new(
config.app_id,
config.app_secret,
github_options
)
if access_token
@api = OAuth2::AccessToken.from_hash(@client, :access_token => access_token)
end
end
def authorize_url(redirect_uri)
client.auth_code.authorize_url({
redirect_uri: redirect_uri,
scope: "api"
})
end
def get_token(code, redirect_uri)
client.auth_code.get_token(code, redirect_uri: redirect_uri).token
end
def issues(project_identifier)
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
end
end
def issue_comments(project_identifier, issue_id)
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
end
end
def project(id)
api.get("/api/v3/projects/#{id}").parsed
end
def projects
lazy_page_iterator(PER_PAGE) do |page|
api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
end
end
private
def lazy_page_iterator(per_page)
Enumerator.new do |y|
page = 1
loop do
items = yield(page)
items.each do |item|
y << item
end
break if items.empty? || items.size < per_page
page += 1
end
end
end
def config
Gitlab.config.omniauth.providers.select{|provider| provider.name == "gitlab"}.first
end
def github_options
{
site: 'https://gitlab.com/',
authorize_url: 'oauth/authorize',
token_url: 'oauth/token'
}
end
end
end
end
module Gitlab
module GitlabImport
class Importer
attr_reader :project, :client
def initialize(project)
@project = project
@client = Client.new(project.creator.gitlab_access_token)
end
def execute
project_identifier = URI.encode(project.import_source, '/')
#Issues && Comments
issues = client.issues(project_identifier)
issues.each do |issue|
body = "*Created by: #{issue["author"]["name"]}*\n\n#{issue["description"]}"
comments = client.issue_comments(project_identifier, issue["id"])
if comments.any?
body += "\n\n\n**Imported comments:**\n"
end
comments.each do |comment|
body += "\n\n*By #{comment["author"]["name"]} on #{comment["created_at"]}*\n\n#{comment["body"]}"
end
project.issues.create!(
description: body,
title: issue["title"],
state: issue["state"],
author_id: gl_user_id(project, issue["author"]["id"])
)
end
true
end
private
def gl_user_id(project, gitlab_id)
user = User.joins(:identities).find_by("identities.extern_uid = ?", gitlab_id.to_s)
(user && user.id) || project.creator_id
end
end
end
end
module Gitlab
module GitlabImport
class ProjectCreator
attr_reader :repo, :namespace, :current_user
def initialize(repo, namespace, current_user)
@repo = repo
@namespace = namespace
@current_user = current_user
end
def execute
@project = Project.new(
name: repo["name"],
path: repo["path"],
description: repo["description"],
namespace: namespace,
creator: current_user,
visibility_level: repo["visibility_level"],
import_type: "gitlab",
import_source: repo["path_with_namespace"],
import_url: repo["http_url_to_repo"]#.sub("://", "://oauth2@#{current_user.gitlab_access_token}")
)
if @project.save!
@project.reload
if @project.import_failed?
@project.import_retry
else
@project.import_start
end
end
@project
end
end
end
end
require 'spec_helper'
describe GithubImportsController do
describe Importers::GithubsController do
let(:user) { create(:user, github_access_token: 'asd123') }
before do
......@@ -16,7 +16,7 @@ describe GithubImportsController do
get :callback
user.reload.github_access_token.should == token
controller.should redirect_to(status_github_import_url)
controller.should redirect_to(status_importers_github_url)
end
end
......
require 'spec_helper'
describe Importers::GitlabsController do
let(:user) { create(:user, gitlab_access_token: 'asd123') }
before do
sign_in(user)
end
describe "GET callback" do
it "updates access token" do
token = "asdasd12345"
Gitlab::GitlabImport::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token)
Gitlab.config.omniauth.providers << OpenStruct.new(app_id: "asd123", app_secret: "asd123", name: "gitlab")
get :callback
user.reload.gitlab_access_token.should == token
controller.should redirect_to(status_importers_gitlab_url)
end
end
describe "GET status" do
before do
@repo = OpenStruct.new(path: 'vim', path_with_namespace: 'asd/vim')
end
it "assigns variables" do
@project = create(:project, import_type: 'gitlab', creator_id: user.id)
controller.stub_chain(:client, :projects).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([@repo])
end
it "does not show already added project" do
@project = create(:project, import_type: 'gitlab', creator_id: user.id, import_source: 'asd/vim')
controller.stub_chain(:client, :projects).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([])
end
end
describe "POST create" do
before do
@repo = {
path: 'vim',
path_with_namespace: 'asd/vim',
owner: {name: "john"},
namespace: {path: "john"}
}.with_indifferent_access
end
it "takes already existing namespace" do
namespace = create(:namespace, name: "john", owner: user)
Gitlab::GitlabImport::ProjectCreator.should_receive(:new).with(@repo, namespace, user).
and_return(double(execute: true))
controller.stub_chain(:client, :project).and_return(@repo)
post :create, format: :js
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment