WIP: Add support for Bitbucket Server imports

app/controllers/application_controller.rb

@@ -30,7 +30,7 @@ class ApplicationController < ActionController::Base
   protect_from_forgery with: :exception, prepend: true
 
   helper_method :can?
-  helper_method :import_sources_enabled?, :github_import_enabled?, :gitea_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
+  helper_method :import_sources_enabled?, :github_import_enabled?, :gitea_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?, :bitbucket_server_import_enabled?
 
   rescue_from Encoding::CompatibilityError do |exception|
     log_exception(exception)
@@ -307,6 +307,10 @@ class ApplicationController < ActionController::Base
     !Gitlab::CurrentSettings.import_sources.empty?
   end
 
+  def bitbucket_server_import_enabled?
+    Gitlab::CurrentSettings.import_sources.include?('bitbucket_server')
+  end
+
   def github_import_enabled?
     Gitlab::CurrentSettings.import_sources.include?('github')
   end

app/controllers/import/bitbucket_server_controller.rb

+class Import::BitbucketServerController < Import::BaseController
+  before_action :verify_bitbucket_server_import_enabled
+  before_action :bitbucket_auth, except: [:new, :configure]
+
+  def new
+  end
+
+  def create
+    bitbucket_client = BitbucketServer::Client.new(credentials)
+
+    repo_id = params[:repo_id].to_s
+    # XXX must be a better way
+    project_slug, repo_slug = repo_id.split("___")
+    repo = bitbucket_client.repo(project_slug, repo_slug)
+    project_name = params[:new_name].presence || repo.name
+
+    repo_owner = repo.owner
+    repo_owner = current_user.username if repo_owner == bitbucket_client.user.username
+    namespace_path = params[:new_namespace].presence || repo_owner
+    target_namespace = find_or_create_namespace(namespace_path, current_user)
+
+    if current_user.can?(:create_projects, target_namespace)
+      project = Gitlab::BitbucketImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, credentials).execute
+
+      if project.persisted?
+        render json: ProjectSerializer.new.represent(project)
+      else
+        render json: { errors: project_save_error(project) }, status: :unprocessable_entity
+      end
+    else
+      render json: { errors: 'This namespace has already been taken! Please choose another one.' }, status: :unprocessable_entity
+    end
+  end
+
+  def configure
+    session[personal_access_token_key] = params[:personal_access_token]
+    session[bitbucket_server_username_key] = params[:bitbucket_username]
+    session[bitbucket_server_url_key] = params[:bitbucket_server_url]
+
+    redirect_to status_import_bitbucket_server_path
+  end
+
+  def status
+    bitbucket_client = BitbucketServer::Client.new(credentials)
+    repos = bitbucket_client.repos
+
+    @repos, @incompatible_repos = repos.partition { |repo| repo.valid? }
+
+    @already_added_projects = find_already_added_projects('bitbucket_server')
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @repos.to_a.reject! { |repo| already_added_projects_names.include?(repo.full_name) }
+  end
+
+  def jobs
+    render json: find_jobs('bitbucket_server')
+  end
+
+  private
+
+  def bitbucket_auth
+    unless session[bitbucket_server_url_key].present? &&
+           session[bitbucket_server_username_key].present? &&
+           session[personal_access_token_key].present?
+      redirect_to new_import_bitbucket_server_path
+    end
+  end
+
+  def verify_bitbucket_server_import_enabled
+    render_404 unless bitbucket_server_import_enabled?
+  end
+
+  def bitbucket_server_url_key
+    :bitbucket_server_url
+  end
+
+  def bitbucket_server_username_key
+    :bitbucket_server_username
+  end
+
+  def personal_access_token_key
+    :bitbucket_server_personal_access_token
+  end
+
+  def credentials
+    {
+      base_uri: session[bitbucket_server_url_key],
+      username: session[bitbucket_server_username_key],
+      personal_access_token: session[personal_access_token_key]
+    }
+  end
+end

app/views/projects/_import_project_pane.html.haml

@@ -17,9 +17,14 @@
       %div
         - if bitbucket_import_enabled?
           = link_to status_import_bitbucket_path, class: "btn import_bitbucket #{'how_to_import_link' unless bitbucket_import_configured?}" do
-            = icon('bitbucket', text: 'Bitbucket')
+            = icon('bitbucket', text: 'Bitbucket Cloud')
           - unless bitbucket_import_configured?
             = render 'bitbucket_import_modal'
+      %div
+        - if bitbucket_server_import_enabled?
+          = link_to status_import_bitbucket_server_path, class: "btn import_bitbucket" do
+            = icon('bitbucket', text: 'Bitbucket Server')
+            = render 'bitbucket_import_modal'
       %div
         - if gitlab_import_enabled?
           = link_to status_import_gitlab_path, class: "btn import_gitlab #{'how_to_import_link' unless gitlab_import_configured?}" do

config/routes/import.rb

@@ -24,6 +24,13 @@ namespace :import do
     get :jobs
   end
 
+  resource :bitbucket_server, only: [:create, :new], controller: :bitbucket_server do
+    post :configure
+    get :status
+    get :callback
+    get :jobs
+  end
+
   resource :google_code, only: [:create, :new], controller: :google_code do
     get :status
     post :callback

lib/bitbucket_server/client.rb

+module BitbucketServer
+  class Client
+    attr_reader :connection
+
+    def initialize(options = {})
+      @connection = Connection.new(options)
+    end
+
+    def issues(repo)
+      path = "/repositories/#{repo}/issues"
+      get_collection(path, :issue)
+    end
+
+    def issue_comments(repo, issue_id)
+      path = "/repositories/#{repo}/issues/#{issue_id}/comments"
+      get_collection(path, :comment)
+    end
+
+    def pull_requests(repo)
+      path = "/repositories/#{repo}/pullrequests?state=ALL"
+      get_collection(path, :pull_request)
+    end
+
+    def pull_request_comments(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments"
+      get_collection(path, :pull_request_comment)
+    end
+
+    def pull_request_diff(repo, pull_request)
+      path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff"
+      connection.get(path)
+    end
+
+    def repo(project, repo_name)
+      parsed_response = connection.get("/projects/#{project}/repos/#{repo_name}")
+      # XXXX TODO Handle failure
+      BitbucketServer::Representation::Repo.new(parsed_response)
+    end
+
+    def repos
+      path = "/repos"
+      get_collection(path, :repo)
+    end
+
+    def user
+      @user ||= begin
+        parsed_response = connection.get('/user')
+        BitbucketServer::Representation::User.new(parsed_response)
+      end
+    end
+
+    private
+
+    def get_collection(path, type)
+      paginator = BitbucketServer::Paginator.new(connection, path, type)
+      BitbucketServer::Collection.new(paginator)
+    end
+  end
+end

lib/bitbucket_server/connection.rb

+module BitbucketServer
+  class Connection
+    DEFAULT_API_VERSION = '1.0'.freeze
+
+    attr_reader :api_version, :base_uri, :username, :token
+
+    def initialize(options = {})
+      @api_version   = options.fetch(:api_version, DEFAULT_API_VERSION)
+      @base_uri      = options[:base_uri]
+      @username      = options[:username]
+      @token         = options[:personal_access_token]
+    end
+
+    def get(path, extra_query = {})
+      auth = { username: username, password: token }
+      response = Gitlab::HTTP.get(build_url(path),
+                                  basic_auth: auth,
+                                  params: extra_query)
+      ## Handle failure
+      response.parsed_response
+    end
+
+    private
+
+    def build_url(path)
+      return path if path.starts_with?(root_url)
+
+      "#{root_url}#{path}"
+    end
+
+    def root_url
+      "#{base_uri}/rest/api/#{api_version}"
+    end
+  end
+end

lib/bitbucket_server/page.rb

+module BitbucketServer
+  class Page
+    attr_reader :attrs, :items
+
+    def initialize(raw, type)
+      @attrs = parse_attrs(raw)
+      @items = parse_values(raw, representation_class(type))
+    end
+
+    def next?
+      !attrs.fetch(:isLastPage, true)
+    end
+
+    def next
+      attrs.fetch(:nextPageStart)
+    end
+
+    private
+
+    def parse_attrs(raw)
+      raw.slice(*%w(size nextPageStart isLastPage)).symbolize_keys
+    end
+
+    def parse_values(raw, bitbucket_rep_class)
+      return [] unless raw['values'] && raw['values'].is_a?(Array)
+
+      bitbucket_rep_class.decorate(raw['values'])
+    end
+
+    def representation_class(type)
+      BitbucketServer::Representation.const_get(type.to_s.camelize)
+    end
+  end
+end

lib/bitbucket_server/paginator.rb

+module BitbucketServer
+  class Paginator
+    PAGE_LENGTH = 25 # The minimum length is 10 and the maximum is 100.
+
+    def initialize(connection, url, type)
+      @connection = connection
+      @type = type
+      @url = url
+      @page = nil
+    end
+
+    def items
+      raise StopIteration unless has_next_page?
+
+      @page = fetch_next_page
+      @page.items
+    end
+
+    private
+
+    attr_reader :connection, :page, :url, :type
+
+    def has_next_page?
+      page.nil? || page.next?
+    end
+
+    def next_url
+      page.nil? ? url : page.next
+    end
+
+    def fetch_next_page
+      parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on)
+      Page.new(parsed_response, type)
+    end
+  end
+end

lib/bitbucket_server/representation/base.rb

+module BitbucketServer
+  module Representation
+    class Base
+      attr_reader :raw
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def self.decorate(entries)
+        entries.map { |entry| new(entry)}
+      end
+    end
+  end
+end

lib/bitbucket_server/representation/comment.rb

+module Bitbucket
+  module Representation
+    class Comment < Representation::Base
+      def author
+        user['username']
+      end
+
+      def note
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on'] || raw['created_on']
+      end
+
+      private
+
+      def user
+        raw.fetch('user', {})
+      end
+    end
+  end
+end

lib/bitbucket_server/representation/issue.rb

+module Bitbucket
+  module Representation
+    class Issue < Representation::Base
+      CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze
+
+      def iid
+        raw['id']
+      end
+
+      def kind
+        raw['kind']
+      end
+
+      def author
+        raw.dig('reporter', 'username')
+      end
+
+      def description
+        raw.fetch('content', {}).fetch('raw', nil)
+      end
+
+      def state
+        closed? ? 'closed' : 'opened'
+      end
+
+      def title
+        raw['title']
+      end
+
+      def milestone
+        raw['milestone']['name'] if raw['milestone'].present?
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['edited_on']
+      end
+
+      def to_s
+        iid
+      end
+
+      private
+
+      def closed?
+        CLOSED_STATUS.include?(raw['state'])
+      end
+    end
+  end
+end

lib/bitbucket_server/representation/pull_request.rb

+module BitbucketServer
+  module Representation
+    class PullRequest < Representation::Base
+      def author
+        raw.fetch('author', {}).fetch('username', nil)
+      end
+
+      def description
+        raw['description']
+      end
+
+      def iid
+        raw['id']
+      end
+
+      def state
+        if raw['state'] == 'MERGED'
+          'merged'
+        elsif raw['state'] == 'DECLINED'
+          'closed'
+        else
+          'opened'
+        end
+      end
+
+      def created_at
+        raw['created_on']
+      end
+
+      def updated_at
+        raw['updated_on']
+      end
+
+      def title
+        raw['title']
+      end
+
+      def source_branch_name
+        source_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def source_branch_sha
+        source_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      def target_branch_name
+        target_branch.fetch('branch', {}).fetch('name', nil)
+      end
+
+      def target_branch_sha
+        target_branch.fetch('commit', {}).fetch('hash', nil)
+      end
+
+      private
+
+      def source_branch
+        raw['source']
+      end
+
+      def target_branch
+        raw['destination']
+      end
+    end
+  end
+end

lib/bitbucket_server/representation/pull_request_comment.rb

+module Bitbucket
+  module Representation
+    class PullRequestComment < Comment
+      def iid
+        raw['id']
+      end
+
+      def file_path
+        inline.fetch('path')
+      end
+
+      def old_pos
+        inline.fetch('from')
+      end
+
+      def new_pos
+        inline.fetch('to')
+      end
+
+      def parent_id
+        raw.fetch('parent', {}).fetch('id', nil)
+      end
+
+      def inline?
+        raw.key?('inline')
+      end
+
+      def has_parent?
+        raw.key?('parent')
+      end
+
+      private
+
+      def inline
+        raw.fetch('inline', {})
+      end
+    end
+  end
+end

lib/bitbucket_server/representation/repo.rb

+module BitbucketServer
+  module Representation
+    class Repo < Representation::Base
+      attr_reader :owner, :slug
+
+      def initialize(raw)
+        super(raw)
+      end
+
+      def owner
+        project['name']
+      end
+
+      def slug
+        raw['slug']
+      end
+
+      def clone_url(token = nil)
+        url = raw['links']['clone'].find { |link| link['name'].starts_with?('http') }.fetch('href')
+
+        if token.present?
+          clone_url = URI.parse(url)
+          clone_url.user = "x-token-auth:#{token}"
+          clone_url.to_s
+        else
+          url
+        end
+      end
+
+      def description
+        project['description']
+      end
+
+      def full_name
+        "#{owner}/#{name}"
+      end
+
+      def issues_enabled?
+        true
+      end
+
+      def name
+        raw['name']
+      end
+
+      def valid?
+        raw['scmId'] == 'git'
+      end
+
+      def has_wiki?
+        false
+      end
+
+      def visibility_level
+        if project['public']
+          Gitlab::VisibilityLevel::PUBLIC
+        else
+          Gitlab::VisibilityLevel::PRIVATE
+        end
+      end
+
+      def project
+        raw['project']
+      end
+
+      def to_s
+        full_name
+      end
+    end
+  end
+end

lib/bitbucket_server/representation/user.rb

+module BitbucketServer
+  module Representation
+    class User < Representation::Base
+      def username
+        raw['username']
+      end
+    end
+  end
+end

lib/gitlab/import_sources.rb

@@ -10,7 +10,8 @@ module Gitlab
     # We exclude `bare_repository` here as it has no import class associated
     ImportTable = [
       ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::ParallelImporter),
-      ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
+      ImportSource.new('bitbucket',      'Bitbucket Cloud',     Gitlab::BitbucketImport::Importer),
+      ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer),
       ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
       ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
       ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),