Commit a9f7fd2c authored by Valery Sizov's avatar Valery Sizov Committed by Valery Sizov

Github Importer

parent d02a22ba
...@@ -263,3 +263,5 @@ group :production do ...@@ -263,3 +263,5 @@ group :production do
end end
gem "newrelic_rpm" gem "newrelic_rpm"
gem 'octokit', '3.7.0'
...@@ -318,6 +318,8 @@ GEM ...@@ -318,6 +318,8 @@ GEM
jwt (~> 0.1.4) jwt (~> 0.1.4)
multi_json (~> 1.0) multi_json (~> 1.0)
rack (~> 1.2) rack (~> 1.2)
octokit (3.7.0)
sawyer (~> 0.6.0, >= 0.5.3)
omniauth (1.1.4) omniauth (1.1.4)
hashie (>= 1.2, < 3) hashie (>= 1.2, < 3)
rack rack
...@@ -472,6 +474,9 @@ GEM ...@@ -472,6 +474,9 @@ GEM
sass (~> 3.2.0) sass (~> 3.2.0)
sprockets (~> 2.8, <= 2.11.0) sprockets (~> 2.8, <= 2.11.0)
sprockets-rails (~> 2.0) sprockets-rails (~> 2.0)
sawyer (0.6.0)
addressable (~> 2.3.5)
faraday (~> 0.8, < 0.10)
sdoc (0.3.20) sdoc (0.3.20)
json (>= 1.1.3) json (>= 1.1.3)
rdoc (~> 3.10) rdoc (~> 3.10)
...@@ -671,6 +676,7 @@ DEPENDENCIES ...@@ -671,6 +676,7 @@ DEPENDENCIES
mysql2 mysql2
newrelic_rpm newrelic_rpm
nprogress-rails nprogress-rails
octokit (= 3.7.0)
omniauth (~> 1.1.3) omniauth (~> 1.1.3)
omniauth-github omniauth-github
omniauth-google-oauth2 omniauth-google-oauth2
......
class GithubImportsController < ApplicationController
before_filter :github_auth, except: :callback
rescue_from Octokit::Unauthorized, with: :github_unauthorized
def callback
token = client.auth_code.get_token(params[:code]).token
current_user.github_access_token = token
current_user.save
redirect_to status_github_import_url
end
def status
@repos = octo_client.repos
octo_client.orgs.each do |org|
@repos += octo_client.repos(org.login)
end
@already_added_projects = current_user.created_projects.where(import_type: "github")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject!{|repo| already_added_projects_names.include? repo.full_name}
end
def create
@repo_id = params[:repo_id].to_i
repo = octo_client.repo(@repo_id)
target_namespace = params[:new_namespace].presence || repo.owner.login
existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace)
if existing_namespace
if existing_namespace.owner == current_user
namespace = existing_namespace
else
@already_been_taken = true
@target_namespace = target_namespace
@project_name = repo.name
render and return
end
else
namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user)
namespace.add_owner(current_user)
end
Gitlab::Github::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::Github::Client.new.client
end
def octo_client
Octokit.auto_paginate = true
@octo_client ||= Octokit::Client.new(:access_token => current_user.github_access_token)
end
def github_auth
if current_user.github_access_token.blank?
go_to_gihub_for_permissions
end
end
def go_to_gihub_for_permissions
redirect_to client.auth_code.authorize_url({
redirect_uri: callback_github_import_url,
scope: "repo, user, user:email"
})
end
def github_unauthorized
go_to_gihub_for_permissions
end
end
...@@ -65,7 +65,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController ...@@ -65,7 +65,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController
redirect_to omniauth_error_path(oauth['provider'], error: error_message) and return redirect_to omniauth_error_path(oauth['provider'], error: error_message) and return
end end
end end
rescue ForbiddenAction => e rescue Gitlab::OAuth::ForbiddenAction => e
flash[:notice] = e.message flash[:notice] = e.message
redirect_to new_user_session_path redirect_to new_user_session_path
end end
......
...@@ -237,4 +237,20 @@ module ProjectsHelper ...@@ -237,4 +237,20 @@ module ProjectsHelper
result.password = '*****' if result.password.present? result.password = '*****' if result.password.present?
result result
end end
def project_status_css_class(status)
case status
when "started"
"active"
when "failed"
"danger"
when "finished"
"success"
end
end
def github_import_enabled?
Gitlab.config.omniauth.enabled && enabled_oauth_providers.include?(:github)
end
end end
- if @already_been_taken
:plain
target_field = $("tr#repo_#{@repo_id} .import-target")
origin_target = target_field.text()
project_name = "#{@project_name}"
origin_namespace = "#{@target_namespace}"
target_field.empty()
target_field.append("<p class='alert alert-danger'>This namespace already been taken! Please choose another one</p>")
target_field.append("<input type='text' name='target_namespace' />")
target_field.append("/" + project_name)
target_field.data("project_name", project_name)
target_field.find('input').prop("value", origin_namespace)
- else
:plain
$("table.import-jobs tbody").prepend($("tr#repo_#{@repo_id}"))
$("tr#repo_#{@repo_id}").addClass("active").find(".import-actions").text("started")
\ No newline at end of file
%h3.page-title
Import repositories from github
%hr
%h4
Select projects you want to import.
%table.table.table-bordered.import-jobs
%thead
%tr
%th From GitHub
%th To GitLab
%th Status
%tbody
- @already_added_projects.each do |repo|
%tr{id: "repo_#{repo.id}", class: "#{project_status_css_class(repo.import_status)}"}
%td= repo.import_source
%td= repo.name_with_namespace
%td= repo.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo.id}"}
%td= repo.full_name
%td.import-target
= repo.full_name
%td.import-actions
= button_tag "Add", class: "btn btn-add-to-import"
:coffeescript
$(".btn-add-to-import").click () ->
new_namespace = null
tr = $(this).closest("tr")
id = tr.attr("id").replace("repo_", "")
if tr.find(".import-target input").length > 0
new_namespace = tr.find(".import-target input").prop("value")
tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
$.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
...@@ -39,7 +39,15 @@ ...@@ -39,7 +39,15 @@
%br %br
The import will time out after 4 minutes. For big repositories, use a clone/push combination. The import will time out after 4 minutes. For big repositories, use a clone/push combination.
For SVN repositories, check #{link_to "this migrating from SVN doc.", "http://doc.gitlab.com/ce/workflow/migrating_from_svn.html"} For SVN repositories, check #{link_to "this migrating from SVN doc.", "http://doc.gitlab.com/ce/workflow/migrating_from_svn.html"}
%hr
- if github_import_enabled?
.project-import.form-group
.col-sm-2
.col-sm-10
%i.fa.fa-bars
= link_to "Import projects from github", status_github_import_path
%hr.prepend-botton-10
.form-group .form-group
= f.label :description, class: 'control-label' do = f.label :description, class: 'control-label' do
......
...@@ -10,7 +10,13 @@ class RepositoryImportWorker ...@@ -10,7 +10,13 @@ class RepositoryImportWorker
project.path_with_namespace, project.path_with_namespace,
project.import_url) project.import_url)
if result if project.import_type == 'github'
result_of_data_import = Gitlab::Github::Importer.new(project).execute
else
result_of_data_import = true
end
if result && result_of_data_import
project.import_finish project.import_finish
project.save project.save
project.satellite.create unless project.satellite.exists? project.satellite.create unless project.satellite.exists?
......
...@@ -51,6 +51,14 @@ Gitlab::Application.routes.draw do ...@@ -51,6 +51,14 @@ Gitlab::Application.routes.draw do
end end
get "/s/:username" => "snippets#user_index", as: :user_snippets, constraints: { username: /.*/ } get "/s/:username" => "snippets#user_index", as: :user_snippets, constraints: { username: /.*/ }
#
# Github importer area
#
resource :github_import, only: [:create, :new] do
get :status
get :callback
end
# #
# Explroe area # Explroe area
# #
......
class AddImportDataToProjectTable < ActiveRecord::Migration
def change
add_column :projects, :import_type, :string
add_column :projects, :import_source, :string
add_column :users, :github_access_token, :string
end
end
...@@ -314,6 +314,8 @@ ActiveRecord::Schema.define(version: 20141226080412) do ...@@ -314,6 +314,8 @@ ActiveRecord::Schema.define(version: 20141226080412) do
t.string "import_status" t.string "import_status"
t.float "repository_size", default: 0.0 t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
end end
add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
...@@ -411,6 +413,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do ...@@ -411,6 +413,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do
t.integer "notification_level", default: 1, null: false t.integer "notification_level", default: 1, null: false
t.datetime "password_expires_at" t.datetime "password_expires_at"
t.integer "created_by_id" t.integer "created_by_id"
t.datetime "last_credential_check_at"
t.string "avatar" t.string "avatar"
t.string "confirmation_token" t.string "confirmation_token"
t.datetime "confirmed_at" t.datetime "confirmed_at"
...@@ -418,7 +421,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do ...@@ -418,7 +421,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do
t.string "unconfirmed_email" t.string "unconfirmed_email"
t.boolean "hide_no_ssh_key", default: false t.boolean "hide_no_ssh_key", default: false
t.string "website_url", default: "", null: false t.string "website_url", default: "", null: false
t.datetime "last_credential_check_at" t.string "github_access_token"
end end
add_index "users", ["admin"], name: "index_users_on_admin", using: :btree add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
......
module Gitlab
module Github
class Client
attr_reader :client
def initialize
@client = ::OAuth2::Client.new(
config.app_id,
config.app_secret,
github_options
)
end
private
def config
Gitlab.config.omniauth.providers.select{|provider| provider.name == "github"}.first
end
def github_options
{
:site => 'https://api.github.com',
:authorize_url => 'https://github.com/login/oauth/authorize',
:token_url => 'https://github.com/login/oauth/access_token'
}
end
end
end
end
module Gitlab
module Github
class Importer
attr_reader :project
def initialize(project)
@project = project
end
def execute
client = octo_client(project.creator.github_access_token)
#Issues && Comments
client.list_issues(project.import_source, state: :all).each do |issue|
if issue.pull_request.nil?
body = "*Created by: #{issue.user.login}*\n\n#{issue.body}"
if issue.comments > 0
body += "\n\n\n**Imported comments:**\n"
client.issue_comments(project.import_source, issue.number).each do |c|
body += "\n\n*By #{c.user.login} on #{c.created_at}*\n\n#{c.body}"
end
end
project.issues.create!(
description: body,
title: issue.title,
state: issue.state == 'closed' ? 'closed' : 'opened',
author_id: gl_user_id(project, issue.user.id)
)
end
end
end
private
def octo_client(access_token)
::Octokit.auto_paginate = true
::Octokit::Client.new(:access_token => access_token)
end
def gl_user_id(project, github_id)
user = User.joins(:identities).find_by("identities.extern_uid = ?", github_id.to_s)
(user && user.id) || project.creator_id
end
end
end
end
module Gitlab
module Github
class ProjectCreator
attr_reader :repo, :namespace, :current_user
def initialize(repo, namespace, current_user)
@repo = repo
@namespace = namespace
@current_user = current_user
end
def execute
@project = Project.new(
name: repo.name,
path: repo.name,
description: repo.description,
namespace: namespace,
creator: current_user,
visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
import_type: "github",
import_source: repo.full_name,
import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@")
)
if @project.save!
@project.reload
if @project.import_failed?
@project.import_retry
else
@project.import_start
end
end
end
end
end
end
...@@ -11,7 +11,7 @@ module Gitlab ...@@ -11,7 +11,7 @@ module Gitlab
end end
def project_name_regex def project_name_regex
/\A[a-zA-Z0-9_][a-zA-Z0-9_\-\. ]*\z/ /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\. ]*\z/
end end
def project_regex_message def project_regex_message
......
require 'spec_helper'
describe GithubImportsController do
let(:user) { create(:user, github_access_token: 'asd123') }
before do
sign_in(user)
end
describe "GET callback" do
it "updates access token" do
token = "asdasd12345"
Gitlab::Github::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token)
get :callback
user.reload.github_access_token.should == token
controller.should redirect_to(status_github_import_url)
end
end
describe "GET status" do
before do
@repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim')
end
it "assigns variables" do
@project = create(:project, import_type: 'github', creator_id: user.id)
controller.stub_chain(:octo_client, :repos).and_return([@repo])
controller.stub_chain(:octo_client, :orgs).and_return([])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([@repo])
end
it "does not show already added project" do
@project = create(:project, import_type: 'github', creator_id: user.id, import_source: 'asd/vim')
controller.stub_chain(:octo_client, :repos).and_return([@repo])
controller.stub_chain(:octo_client, :orgs).and_return([])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([])
end
end
describe "POST create" do
before do
@repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim', owner: OpenStruct.new(login: "john"))
end
it "takes already existing namespace" do
namespace = create(:namespace, name: "john", owner: user)
Gitlab::Github::ProjectCreator.should_receive(:new).with(@repo, namespace, user).
and_return(double(execute: true))
controller.stub_chain(:octo_client, :repo).and_return(@repo)
post :create, format: :js
end
end
end
...@@ -20,4 +20,13 @@ describe ProjectsHelper do ...@@ -20,4 +20,13 @@ describe ProjectsHelper do
"<option value=\"gitlab\">GitLab</option>" "<option value=\"gitlab\">GitLab</option>"
end end
end end
describe "#project_status_css_class" do
it "returns appropriate class" do
project_status_css_class("started").should == "active"
project_status_css_class("failed").should == "danger"
project_status_css_class("finished").should == "success"
end
end
end end
require 'spec_helper'
describe Gitlab::Github::ProjectCreator do
let(:user) { create(:user, github_access_token: "asdffg") }
let(:repo) { OpenStruct.new(
login: 'vim',
name: 'vim',
private: true,
full_name: 'asd/vim',
clone_url: "https://gitlab.com/asd/vim.git",
owner: OpenStruct.new(login: "john"))
}
let(:namespace){ create(:namespace) }
it 'creates project' do
Project.any_instance.stub(:add_import_job)
project_creator = Gitlab::Github::ProjectCreator.new(repo, namespace, user)
project_creator.execute
project = Project.last
project.import_url.should == "https://asdffg@gitlab.com/asd/vim.git"
project.visibility_level.should == Gitlab::VisibilityLevel::PRIVATE
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment