GitLab integration. Importer

Gemfile

@@ -29,6 +29,7 @@ gem 'omniauth-twitter'
 gem 'omniauth-github'
 gem 'omniauth-shibboleth'
 gem 'omniauth-kerberos'
+gem 'omniauth-gitlab'
 gem 'doorkeeper', '2.1.0'
 gem "rack-oauth2", "~> 1.0.5"
 

Gemfile.lock

@@ -332,6 +332,9 @@ GEM
     omniauth-github (1.1.1)
       omniauth (~> 1.0)
       omniauth-oauth2 (~> 1.1)
+    omniauth-gitlab (1.0.0)
+      omniauth (~> 1.0)
+      omniauth-oauth2 (~> 1.0)
     omniauth-google-oauth2 (0.2.5)
       omniauth (> 1.0)
       omniauth-oauth2 (~> 1.1)
@@ -689,6 +692,7 @@ DEPENDENCIES
   octokit (= 3.7.0)
   omniauth (~> 1.1.3)
   omniauth-github
+  omniauth-gitlab
   omniauth-google-oauth2
   omniauth-kerberos
   omniauth-shibboleth

app/controllers/github_imports_controller.rb → app/controllers/importers/githubs_controller.rb

-class GithubImportsController < ApplicationController
+class Importers::GithubsController < ApplicationController
   before_filter :github_auth, except: :callback
 
   rescue_from Octokit::Unauthorized, with: :github_unauthorized
@@ -7,7 +7,7 @@ class GithubImportsController < ApplicationController
     token = client.auth_code.get_token(params[:code]).token
     current_user.github_access_token = token
     current_user.save
-    redirect_to status_github_import_url
+    redirect_to status_importers_github_url
   end
 
   def status
@@ -69,7 +69,7 @@ class GithubImportsController < ApplicationController
 
   def go_to_github_for_permissions
     redirect_to client.auth_code.authorize_url({
-      redirect_uri: callback_github_import_url,
+      redirect_uri: callback_importers_github_url,
       scope: "repo, user, user:email"
     })
   end

app/controllers/importers/gitlabs_controller.rb

+class Importers::GitlabsController < ApplicationController
+  before_filter :gitlab_auth, except: :callback
+
+  rescue_from OAuth2::Error, with: :gitlab_unauthorized
+
+  def callback
+    token = client.get_token(params[:code], callback_importers_gitlab_url)
+    current_user.gitlab_access_token = token
+    current_user.save
+    redirect_to status_importers_gitlab_url
+  end
+
+  def status
+    @repos = client.projects
+    
+    @already_added_projects = current_user.created_projects.where(import_type: "gitlab")
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @repos.to_a.reject!{|repo| already_added_projects_names.include? repo["path_with_namespace"]}
+  end
+
+  def jobs
+    jobs = current_user.created_projects.where(import_type: "gitlab").to_json(:only => [:id, :import_status])
+    render json: jobs
+  end
+
+  def create
+    @repo_id = params[:repo_id].to_i
+    repo = client.project(@repo_id)
+    target_namespace = params[:new_namespace].presence || repo["namespace"]["path"]
+    existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace)
+    
+    if existing_namespace
+      if existing_namespace.owner == current_user
+        namespace = existing_namespace
+      else
+        @already_been_taken = true
+        @target_namespace = target_namespace
+        @project_name = repo["path"]
+        render and return
+      end
+    else
+      namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user)
+      namespace.add_owner(current_user)
+    end
+
+    @project = Gitlab::GitlabImport::ProjectCreator.new(repo, namespace, current_user).execute
+  end
+
+  private
+
+  def client
+    @client ||= Gitlab::GitlabImport::Client.new(current_user.gitlab_access_token)
+  end
+
+  def gitlab_auth
+    if current_user.gitlab_access_token.blank?
+      go_to_gitlab_for_permissions
+    end
+  end
+
+  def go_to_gitlab_for_permissions
+    redirect_to client.authorize_url(callback_importers_gitlab_url)
+  end
+
+  def gitlab_unauthorized
+    go_to_gitlab_for_permissions
+  end
+end

app/helpers/oauth_helper.rb

@@ -4,7 +4,7 @@ module OauthHelper
   end
 
   def default_providers
-    [:twitter, :github, :google_oauth2, :ldap]
+    [:twitter, :github, :gitlab, :google_oauth2, :ldap]
   end
 
   def enabled_oauth_providers
@@ -13,7 +13,7 @@ module OauthHelper
 
   def enabled_social_providers
     enabled_oauth_providers.select do |name|
-      [:twitter, :github, :google_oauth2].include?(name.to_sym)
+      [:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym)
     end
   end
 

app/helpers/projects_helper.rb

@@ -253,4 +253,8 @@ module ProjectsHelper
   def github_import_enabled?
     enabled_oauth_providers.include?(:github)
   end
+
+  def gitlab_import_enabled?
+    enabled_oauth_providers.include?(:gitlab)
+  end
 end

app/views/github_imports/status.html.haml → app/views/importers/githubs/status.html.haml

@@ -43,11 +43,11 @@
     if tr.find(".import-target input").length > 0
       new_namespace = tr.find(".import-target input").prop("value")
       tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
-    $.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
+    $.post "#{importers_github_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
   
 
   setInterval (->
-    $.get "#{jobs_github_import_path}", (data)->
+    $.get "#{jobs_importers_github_path}", (data)->
       $.each data, (i, job) ->
         job_item = $("#project_" + job.id)
         status_field = job_item.find(".job-status")

app/views/importers/gitlabs/create.js.haml

+- if @already_been_taken
+  :plain
+    target_field = $("tr#repo_#{@repo_id} .import-target")
+    origin_target = target_field.text()
+    project_name = "#{@project_name}"
+    origin_namespace = "#{@target_namespace}"
+    target_field.empty()
+    target_field.append("<p class='alert alert-danger'>This namespace already been taken! Please choose another one</p>")
+    target_field.append("<input type='text' name='target_namespace' />")
+    target_field.append("/" + project_name)
+    target_field.data("project_name", project_name)
+    target_field.find('input').prop("value", origin_namespace)
+- else
+  :plain
+    job = $("tr#repo_#{@repo_id}")
+    job.attr("id", "project_#{@project.id}")
+    $("table.import-jobs tbody").prepend(job)
+    job.addClass("active").find(".import-actions").html("<i class='fa fa-spinner fa-spin'></i> started")

app/views/importers/gitlabs/status.html.haml

+%h3.page-title
+  %i.fa.fa-github
+  Import repositories from GitLab.com
+
+%p.light
+  Select projects you want to import.  
+
+%hr
+%table.table.import-jobs
+  %thead
+    %tr
+      %th From GitLab.com
+      %th To GitLab private instance
+      %th Status
+  %tbody
+    - @already_added_projects.each do |project|
+      %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
+        %td= project.import_source
+        %td
+          %strong= link_to project.name_with_namespace, project
+        %td.job-status
+          - if project.import_status == 'finished'
+            %span.cgreen
+              %i.fa.fa-check
+              done
+          - else
+            = project.human_import_status_name
+
+    - @repos.each do |repo|
+      %tr{id: "repo_#{repo["id"]}"}
+        %td= repo["path_with_namespace"]
+        %td.import-target
+          = repo["path_with_namespace"]
+        %td.import-actions.job-status
+          = button_tag "Add", class: "btn btn-add-to-import"
+
+
+:coffeescript
+  $(".btn-add-to-import").click () ->
+    new_namespace = null
+    tr = $(this).closest("tr")
+    id = tr.attr("id").replace("repo_", "")
+    if tr.find(".import-target input").length > 0
+      new_namespace = tr.find(".import-target input").prop("value")
+      tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
+    $.post "#{importers_gitlab_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
+  
+
+  setInterval (->
+    $.get "#{jobs_importers_gitlab_path}", (data)->
+      $.each data, (i, job) ->
+        job_item = $("#project_" + job.id)
+        status_field = job_item.find(".job-status")
+        
+        if job.import_status == 'finished'
+          job_item.removeClass("active").addClass("success")
+          status_field.html('<span class="cgreen"><i class="fa fa-check"></i> done</span>')
+        else if job.import_status == 'started'
+          status_field.html("<i class='fa fa-spinner fa-spin'></i> started")
+        else
+          status_field.html(job.import_status)
+        
+  ), 4000

app/views/projects/_gitlab_import_modal.html.haml

+%div#gitlab_import_modal.modal.hide
+  .modal-dialog
+    .modal-content
+      .modal-header
+        %a.close{href: "#", "data-dismiss" => "modal"} ×
+        %h3 GitLab OAuth import
+      .modal-body
+        You need to setup integration with GitLab first.
+        = link_to 'How to setup integration with GitLab', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/gitlab.md'
+
+
+:javascript
+  $(function(){
+    var import_modal = $('#gitlab_import_modal').modal({modal: true, show:false});
+    $('.how_to_import_link').bind("click", function(e){
+      e.preventDefault();
+      import_modal.show();
+    });
+    $('.modal-header .close').bind("click", function(){
+      import_modal.hide();
+    })
+  })

app/views/projects/new.html.haml

@@ -44,7 +44,7 @@
         .col-sm-2
         .col-sm-10
           - if github_import_enabled?
-            = link_to status_github_import_path do
+            = link_to status_importers_github_path do
               %i.fa.fa-github
               Import projects from GitHub
           - else
@@ -53,6 +53,19 @@
               Import projects from GitHub
             = render 'github_import_modal'
       
+      .project-import.form-group
+        .col-sm-2
+        .col-sm-10
+          - if gitlab_import_enabled?
+            = link_to status_importers_gitlab_path do
+              %i.fa.fa-heart
+              Import projects from GitLab.com
+          - else
+            = link_to '#', class: 'how_to_import_link light' do
+              %i.fa.fa-heart
+              Import projects from GitLab.com
+            = render 'gitlab_import_modal'
+
       %hr.prepend-botton-10
 
       .form-group

app/workers/repository_import_worker.rb

@@ -12,6 +12,8 @@ class RepositoryImportWorker
 
     if project.import_type == 'github'
       result_of_data_import = Gitlab::Github::Importer.new(project).execute
+    elsif project.import_type == 'gitlab'
+      result_of_data_import = Gitlab::GitlabImport::Importer.new(project).execute
     else
       result_of_data_import = true
     end

config/initializers/doorkeeper.rb

@@ -27,7 +27,7 @@ Doorkeeper.configure do
 
   # Access token expiration time (default 2 hours).
   # If you want to disable expiration, set this to nil.
-  # access_token_expires_in 2.hours
+  access_token_expires_in nil
 
   # Reuse access token for the same resource owner within an application (disabled by default)
   # Rationale: https://github.com/doorkeeper-gem/doorkeeper/issues/383

config/routes.rb

@@ -51,15 +51,26 @@ Gitlab::Application.routes.draw do
   end
   get '/s/:username' => 'snippets#user_index', as: :user_snippets, constraints: { username: /.*/ }
 
+  
   #
-  # Github importer area
+  # Importers
   #
-  resource :github_import, only: [:create, :new] do
+  namespace :importers do
+    resource :github, only: [:create, :new] do
       get :status
       get :callback
       get :jobs
     end
 
+    resource :gitlab, only: [:create, :new] do
+      get :status
+      get :callback
+      get :jobs
+    end
+  end
+  
+  
+
   #
   # Explore area
   #

doc/integration/gitlab.md

+# GitLab OAuth2 OmniAuth Provider
+
+To enable the GitLab OmniAuth provider you must register your application with GitLab. GitLab will generate a client ID and secret key for you to use.
+
+1.  Sign in to GitLab.
+
+1.  Navigate to your settings.
+
+1.  Select "Applications" in the left menu.
+
+1.  Select "New application".
+
+1.  Provide the required details.
+    - Name: This can be anything. Consider something like "\<Organization\>'s GitLab" or "\<Your Name\>'s GitLab" or something else descriptive.
+    - Redirect URI: 
+    
+    ```
+    http://gitlab.example.com/importers/gitlab/callback
+    http://gitlab.example.com/users/auth/gitlab/callback
+    ```
+
+    The first link is required for the importer and second for the authorization. 
+
+1.  Select "Submit".
+
+1.  You should now see a Application ID and Secret. Keep this page open as you continue configuration.
+
+1.  On your GitLab server, open the configuration file.
+
+    ```sh
+    cd /home/git/gitlab
+
+    sudo -u git -H editor config/gitlab.yml
+    ```
+
+1.  Find the section dealing with OmniAuth. See [Initial OmniAuth Configuration](README.md#initial-omniauth-configuration) for more details.
+
+1.  Under `providers:` uncomment (or add) lines that look like the following:
+
+    ```
+        - { name: 'gitlab', app_id: 'YOUR APP ID',
+          app_secret: 'YOUR APP SECRET',
+          args: { scope: 'api' } }
+    ```
+
+1.  Change 'YOUR APP ID' to the Application ID from the GitLab application page.
+
+1.  Change 'YOUR APP SECRET' to the secret from the GitLab application page.
+
+1.  Save the configuration file.
+
+1.  Restart GitLab for the changes to take effect.
+
+On the sign in page there should now be a GitLab icon below the regular sign in form. Click the icon to begin the authentication process. GitLab will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to your GitLab instance and will be signed in.

lib/gitlab/gitlab_import/client.rb

+module Gitlab
+  module GitlabImport
+    class Client
+      attr_reader :client, :api
+
+      PER_PAGE = 100
+
+      def initialize(access_token)
+        @client = ::OAuth2::Client.new(
+          config.app_id,
+          config.app_secret,
+          github_options
+        )
+
+        if access_token
+          @api = OAuth2::AccessToken.from_hash(@client, :access_token => access_token)
+        end
+      end
+
+      def authorize_url(redirect_uri)
+        client.auth_code.authorize_url({
+          redirect_uri: redirect_uri,
+          scope: "api"
+        })
+      end
+
+      def get_token(code, redirect_uri)
+        client.auth_code.get_token(code, redirect_uri: redirect_uri).token
+      end
+
+      def issues(project_identifier)
+        lazy_page_iterator(PER_PAGE) do |page|
+          api.get("/api/v3/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed
+        end
+      end
+
+      def issue_comments(project_identifier, issue_id)
+        lazy_page_iterator(PER_PAGE) do |page|
+          api.get("/api/v3/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed
+        end
+      end
+
+      def project(id)
+        api.get("/api/v3/projects/#{id}").parsed
+      end
+
+      def projects
+        lazy_page_iterator(PER_PAGE) do |page|
+          api.get("/api/v3/projects?per_page=#{PER_PAGE}&page=#{page}").parsed
+        end
+      end
+
+      private
+
+      def lazy_page_iterator(per_page)
+        Enumerator.new do |y|
+          page = 1
+          loop do
+            items = yield(page)
+            items.each do |item|
+              y << item
+            end
+            break if items.empty? || items.size < per_page
+            page += 1
+          end
+        end
+      end
+
+      def config
+        Gitlab.config.omniauth.providers.select{|provider| provider.name == "gitlab"}.first
+      end
+
+      def github_options
+        {
+          site: 'https://gitlab.com/',
+          authorize_url: 'oauth/authorize',
+          token_url: 'oauth/token'
+        }
+      end
+    end
+  end
+end

lib/gitlab/gitlab_import/importer.rb

+module Gitlab
+  module GitlabImport
+    class Importer
+      attr_reader :project, :client
+
+      def initialize(project)
+        @project = project
+        @client = Client.new(project.creator.gitlab_access_token)
+      end
+
+      def execute
+        project_identifier = URI.encode(project.import_source, '/')
+
+        #Issues && Comments
+        issues = client.issues(project_identifier)
+        
+        issues.each do |issue|
+          body = "*Created by: #{issue["author"]["name"]}*\n\n#{issue["description"]}"
+          
+          
+          comments = client.issue_comments(project_identifier, issue["id"])
+          if comments.any?
+            body += "\n\n\n**Imported comments:**\n"
+          end
+          comments.each do |comment|
+            body += "\n\n*By #{comment["author"]["name"]} on #{comment["created_at"]}*\n\n#{comment["body"]}"
+          end
+
+          project.issues.create!(
+            description: body, 
+            title: issue["title"],
+            state: issue["state"],
+            author_id: gl_user_id(project, issue["author"]["id"])
+          )
+        end
+        
+        true
+      end
+
+      private
+
+      def gl_user_id(project, gitlab_id)
+        user = User.joins(:identities).find_by("identities.extern_uid = ?", gitlab_id.to_s)
+        (user && user.id) || project.creator_id
+      end
+    end
+  end
+end

lib/gitlab/gitlab_import/project_creator.rb

+module Gitlab
+  module GitlabImport
+    class ProjectCreator
+      attr_reader :repo, :namespace, :current_user
+
+      def initialize(repo, namespace, current_user)
+        @repo = repo
+        @namespace = namespace
+        @current_user = current_user
+      end
+
+      def execute
+        @project = Project.new(
+          name: repo["name"],
+          path: repo["path"],
+          description: repo["description"],
+          namespace: namespace,
+          creator: current_user,
+          visibility_level: repo["visibility_level"],
+          import_type: "gitlab",
+          import_source: repo["path_with_namespace"],
+          import_url: repo["http_url_to_repo"]#.sub("://", "://oauth2@#{current_user.gitlab_access_token}")
+        )
+
+        if @project.save!
+          @project.reload
+
+          if @project.import_failed?
+            @project.import_retry
+          else
+            @project.import_start
+          end
+        end
+
+        @project
+      end
+    end
+  end
+end

spec/controllers/github_imports_controller_spec.rb → spec/controllers/importers/githubs_controller_spec.rb

 require 'spec_helper'
 
-describe GithubImportsController do
+describe Importers::GithubsController do
   let(:user) { create(:user, github_access_token: 'asd123') }
 
   before do
@@ -16,7 +16,7 @@ describe GithubImportsController do
       get :callback
       
       user.reload.github_access_token.should == token
-      controller.should redirect_to(status_github_import_url)
+      controller.should redirect_to(status_importers_github_url)
     end
   end
 

spec/controllers/importers/gitlabs_controller_spec.rb

+require 'spec_helper'
+
+describe Importers::GitlabsController do
+  let(:user) { create(:user, gitlab_access_token: 'asd123') }
+
+  before do
+    sign_in(user)
+  end
+
+  describe "GET callback" do
+    it "updates access token" do
+      token = "asdasd12345"
+      Gitlab::GitlabImport::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token)
+      Gitlab.config.omniauth.providers << OpenStruct.new(app_id: "asd123", app_secret: "asd123", name: "gitlab")
+
+      get :callback
+      
+      user.reload.gitlab_access_token.should == token
+      controller.should redirect_to(status_importers_gitlab_url)
+    end
+  end
+
+  describe "GET status" do
+    before do
+      @repo = OpenStruct.new(path: 'vim', path_with_namespace: 'asd/vim')
+    end
+
+    it "assigns variables" do
+      @project = create(:project, import_type: 'gitlab', creator_id: user.id)
+      controller.stub_chain(:client, :projects).and_return([@repo])
+      
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([@repo])
+    end
+
+    it "does not show already added project" do
+      @project = create(:project, import_type: 'gitlab', creator_id: user.id, import_source: 'asd/vim')
+      controller.stub_chain(:client, :projects).and_return([@repo])
+      
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([])
+    end
+  end
+
+  describe "POST create" do
+    before do
+      @repo = {
+        path: 'vim',
+        path_with_namespace: 'asd/vim',
+        owner: {name: "john"},
+        namespace: {path: "john"}
+      }.with_indifferent_access
+    end
+
+    it "takes already existing namespace" do
+      namespace = create(:namespace, name: "john", owner: user)
+      Gitlab::GitlabImport::ProjectCreator.should_receive(:new).with(@repo, namespace, user).
+        and_return(double(execute: true))
+      controller.stub_chain(:client, :project).and_return(@repo)
+
+      post :create, format: :js
+    end
+  end
+end