Commit 7884a261 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'import-gh-pull-requests' into 'master'

Import GitHub Pull Requests into GitLab

Fixes #2833

See merge request !2168
parents 95b1adb3 e194b962
......@@ -38,6 +38,7 @@ v 8.3.1
- Fix LDAP identity and user retrieval when special characters are used
- Move Sidekiq-cron configuration to gitlab.yml
- Enable forcing Two-Factor authentication sitewide, with optional grace period
- Import GitHub Pull Requests into GitLab
v 8.3.0
- Bump rack-attack to 4.3.1 for security fix (Stan Hu)
......
......@@ -14,7 +14,7 @@ If you want to import from a GitHub Enterprise instance, you need to use GitLab
![Importer page](github_importer/importer.png)
* To import a project, you can simple click "Add". The importer will import your repository and issues. Once the importer is done, a new GitLab project will be created with your imported data.
* To import a project, you can simple click "Add". The importer will import your repository, issues, and pull requests. Once the importer is done, a new GitLab project will be created with your imported data.
### Note
When you import your projects from GitHub, it is not possible to keep your labels and milestones. We are working on improving this in the near future.
When you import your projects from GitHub, it is not possible to keep your labels, milestones, and cross-repository pull requests. We are working on improving this in the near future.
module Gitlab
module GithubImport
class BaseFormatter
attr_reader :formatter, :project, :raw_data
def initialize(project, raw_data)
@project = project
@raw_data = raw_data
@formatter = Gitlab::ImportFormatter.new
end
private
def gl_user_id(github_id)
User.joins(:identities).
find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s).
try(:id)
end
end
end
end
module Gitlab
module GithubImport
class CommentFormatter < BaseFormatter
def attributes
{
project: project,
note: note,
commit_id: raw_data.commit_id,
line_code: line_code,
author_id: author_id,
created_at: raw_data.created_at,
updated_at: raw_data.updated_at
}
end
private
def author
raw_data.user.login
end
def author_id
gl_user_id(raw_data.user.id) || project.creator_id
end
def body
raw_data.body || ""
end
def line_code
if on_diff?
Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0)
end
end
def on_diff?
raw_data.path && raw_data.position
end
def note
formatter.author_line(author) + body
end
end
end
end
......@@ -12,39 +12,59 @@ module Gitlab
end
def execute
#Issues && Comments
import_issues
import_pull_requests
true
end
private
def import_issues
client.list_issues(project.import_source, state: :all,
sort: :created,
direction: :asc).each do |issue|
if issue.pull_request.nil?
body = @formatter.author_line(issue.user.login)
body += issue.body || ""
direction: :asc).each do |raw_data|
gh_issue = IssueFormatter.new(project, raw_data)
if issue.comments > 0
body += @formatter.comments_header
if gh_issue.valid?
issue = Issue.create!(gh_issue.attributes)
client.issue_comments(project.import_source, issue.number).each do |c|
body += @formatter.comment(c.user.login, c.created_at, c.body)
if gh_issue.has_comments?
import_comments(gh_issue.number, issue)
end
end
end
end
project.issues.create!(
description: body,
title: issue.title,
state: issue.state == 'closed' ? 'closed' : 'opened',
author_id: gl_user_id(project, issue.user.id)
)
def import_pull_requests
client.pull_requests(project.import_source, state: :all,
sort: :created,
direction: :asc).each do |raw_data|
pull_request = PullRequestFormatter.new(project, raw_data)
if !pull_request.cross_project? && pull_request.valid?
merge_request = MergeRequest.create!(pull_request.attributes)
import_comments(pull_request.number, merge_request)
import_comments_on_diff(pull_request.number, merge_request)
end
end
end
private
def import_comments(issue_number, noteable)
comments = client.issue_comments(project.import_source, issue_number)
create_comments(comments, noteable)
end
def gl_user_id(project, github_id)
user = User.joins(:identities).
find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s)
(user && user.id) || project.creator_id
def import_comments_on_diff(pull_request_number, merge_request)
comments = client.pull_request_comments(project.import_source, pull_request_number)
create_comments(comments, merge_request)
end
def create_comments(comments, noteable)
comments.each do |raw_data|
comment = CommentFormatter.new(project, raw_data)
noteable.notes.create!(comment.attributes)
end
end
end
end
......
module Gitlab
module GithubImport
class IssueFormatter < BaseFormatter
def attributes
{
project: project,
title: raw_data.title,
description: description,
state: state,
author_id: author_id,
assignee_id: assignee_id,
created_at: raw_data.created_at,
updated_at: updated_at
}
end
def has_comments?
raw_data.comments > 0
end
def number
raw_data.number
end
def valid?
raw_data.pull_request.nil?
end
private
def assigned?
raw_data.assignee.present?
end
def assignee_id
if assigned?
gl_user_id(raw_data.assignee.id)
end
end
def author
raw_data.user.login
end
def author_id
gl_user_id(raw_data.user.id) || project.creator_id
end
def body
raw_data.body || ""
end
def description
@formatter.author_line(author) + body
end
def state
raw_data.state == 'closed' ? 'closed' : 'opened'
end
def updated_at
state == 'closed' ? raw_data.closed_at : raw_data.updated_at
end
end
end
end
module Gitlab
module GithubImport
class PullRequestFormatter < BaseFormatter
def attributes
{
title: raw_data.title,
description: description,
source_project: source_project,
source_branch: source_branch.name,
target_project: target_project,
target_branch: target_branch.name,
state: state,
author_id: author_id,
assignee_id: assignee_id,
created_at: raw_data.created_at,
updated_at: updated_at
}
end
def cross_project?
source_repo.fork == true
end
def number
raw_data.number
end
def valid?
source_branch.present? && target_branch.present?
end
private
def assigned?
raw_data.assignee.present?
end
def assignee_id
if assigned?
gl_user_id(raw_data.assignee.id)
end
end
def author
raw_data.user.login
end
def author_id
gl_user_id(raw_data.user.id) || project.creator_id
end
def body
raw_data.body || ""
end
def description
formatter.author_line(author) + body
end
def source_project
project
end
def source_repo
raw_data.head.repo
end
def source_branch
source_project.repository.find_branch(raw_data.head.ref)
end
def target_project
project
end
def target_branch
target_project.repository.find_branch(raw_data.base.ref)
end
def state
@state ||= case true
when raw_data.state == 'closed' && raw_data.merged_at.present?
'merged'
when raw_data.state == 'closed'
'closed'
else
'opened'
end
end
def updated_at
case state
when 'merged' then raw_data.merged_at
when 'closed' then raw_data.closed_at
else
raw_data.updated_at
end
end
end
end
end
require 'spec_helper'
describe Gitlab::GithubImport::CommentFormatter, lib: true do
let(:project) { create(:project) }
let(:octocat) { OpenStruct.new(id: 123456, login: 'octocat') }
let(:created_at) { DateTime.strptime('2013-04-10T20:09:31Z') }
let(:updated_at) { DateTime.strptime('2014-03-03T18:58:10Z') }
let(:base_data) do
{
body: "I'm having a problem with this.",
user: octocat,
created_at: created_at,
updated_at: updated_at
}
end
subject(:comment) { described_class.new(project, raw_data)}
describe '#attributes' do
context 'when do not reference a portion of the diff' do
let(:raw_data) { OpenStruct.new(base_data) }
it 'returns formatted attributes' do
expected = {
project: project,
note: "*Created by: octocat*\n\nI'm having a problem with this.",
commit_id: nil,
line_code: nil,
author_id: project.creator_id,
created_at: created_at,
updated_at: updated_at
}
expect(comment.attributes).to eq(expected)
end
end
context 'when on a portion of the diff' do
let(:diff_data) do
{
body: 'Great stuff',
commit_id: '6dcb09b5b57875f334f61aebed695e2e4193db5e',
diff_hunk: '@@ -16,33 +16,40 @@ public class Connection : IConnection...',
path: 'file1.txt',
position: 1
}
end
let(:raw_data) { OpenStruct.new(base_data.merge(diff_data)) }
it 'returns formatted attributes' do
expected = {
project: project,
note: "*Created by: octocat*\n\nGreat stuff",
commit_id: '6dcb09b5b57875f334f61aebed695e2e4193db5e',
line_code: 'ce1be0ff4065a6e9415095c95f25f47a633cef2b_0_1',
author_id: project.creator_id,
created_at: created_at,
updated_at: updated_at
}
expect(comment.attributes).to eq(expected)
end
end
context 'when author is a GitLab user' do
let(:raw_data) { OpenStruct.new(base_data.merge(user: octocat)) }
it 'returns project#creator_id as author_id when is not a GitLab user' do
expect(comment.attributes.fetch(:author_id)).to eq project.creator_id
end
it 'returns GitLab user id as author_id when is a GitLab user' do
gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github')
expect(comment.attributes.fetch(:author_id)).to eq gl_user.id
end
end
end
end
require 'spec_helper'
describe Gitlab::GithubImport::IssueFormatter, lib: true do
let!(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) }
let(:octocat) { OpenStruct.new(id: 123456, login: 'octocat') }
let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') }
let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') }
let(:base_data) do
{
number: 1347,
state: 'open',
title: 'Found a bug',
body: "I'm having a problem with this.",
assignee: nil,
user: octocat,
comments: 0,
pull_request: nil,
created_at: created_at,
updated_at: updated_at,
closed_at: nil
}
end
subject(:issue) { described_class.new(project, raw_data)}
describe '#attributes' do
context 'when issue is open' do
let(:raw_data) { OpenStruct.new(base_data.merge(state: 'open')) }
it 'returns formatted attributes' do
expected = {
project: project,
title: 'Found a bug',
description: "*Created by: octocat*\n\nI'm having a problem with this.",
state: 'opened',
author_id: project.creator_id,
assignee_id: nil,
created_at: created_at,
updated_at: updated_at
}
expect(issue.attributes).to eq(expected)
end
end
context 'when issue is closed' do
let(:closed_at) { DateTime.strptime('2011-01-28T19:01:12Z') }
let(:raw_data) { OpenStruct.new(base_data.merge(state: 'closed', closed_at: closed_at)) }
it 'returns formatted attributes' do
expected = {
project: project,
title: 'Found a bug',
description: "*Created by: octocat*\n\nI'm having a problem with this.",
state: 'closed',
author_id: project.creator_id,
assignee_id: nil,
created_at: created_at,
updated_at: closed_at
}
expect(issue.attributes).to eq(expected)
end
end
context 'when it is assigned to someone' do
let(:raw_data) { OpenStruct.new(base_data.merge(assignee: octocat)) }
it 'returns nil as assignee_id when is not a GitLab user' do
expect(issue.attributes.fetch(:assignee_id)).to be_nil
end
it 'returns GitLab user id as assignee_id when is a GitLab user' do
gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github')
expect(issue.attributes.fetch(:assignee_id)).to eq gl_user.id
end
end
context 'when author is a GitLab user' do
let(:raw_data) { OpenStruct.new(base_data.merge(user: octocat)) }
it 'returns project#creator_id as author_id when is not a GitLab user' do
expect(issue.attributes.fetch(:author_id)).to eq project.creator_id
end
it 'returns GitLab user id as author_id when is a GitLab user' do
gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github')
expect(issue.attributes.fetch(:author_id)).to eq gl_user.id
end
end
end
describe '#has_comments?' do
context 'when number of comments is greater than zero' do
let(:raw_data) { OpenStruct.new(base_data.merge(comments: 1)) }
it 'returns true' do
expect(issue.has_comments?).to eq true
end
end
context 'when number of comments is equal to zero' do
let(:raw_data) { OpenStruct.new(base_data.merge(comments: 0)) }
it 'returns false' do
expect(issue.has_comments?).to eq false
end
end
end
describe '#number' do
let(:raw_data) { OpenStruct.new(base_data.merge(number: 1347)) }
it 'returns pull request number' do
expect(issue.number).to eq 1347
end
end
describe '#valid?' do
context 'when mention a pull request' do
let(:raw_data) { OpenStruct.new(base_data.merge(pull_request: OpenStruct.new)) }
it 'returns false' do
expect(issue.valid?).to eq false
end
end
context 'when does not mention a pull request' do
let(:raw_data) { OpenStruct.new(base_data.merge(pull_request: nil)) }
it 'returns true' do
expect(issue.valid?).to eq true
end
end
end
end
require 'spec_helper'
describe Gitlab::GithubImport::PullRequestFormatter, lib: true do
let(:project) { create(:project) }
let(:source_branch) { OpenStruct.new(ref: 'feature') }
let(:target_branch) { OpenStruct.new(ref: 'master') }
let(:octocat) { OpenStruct.new(id: 123456, login: 'octocat') }
let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') }
let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') }
let(:base_data) do
{
number: 1347,
state: 'open',
title: 'New feature',
body: 'Please pull these awesome changes',
head: source_branch,
base: target_branch,
assignee: nil,
user: octocat,
created_at: created_at,
updated_at: updated_at,
closed_at: nil,
merged_at: nil
}
end
subject(:pull_request) { described_class.new(project, raw_data)}
describe '#attributes' do
context 'when pull request is open' do
let(:raw_data) { OpenStruct.new(base_data.merge(state: 'open')) }
it 'returns formatted attributes' do
expected = {
title: 'New feature',
description: "*Created by: octocat*\n\nPlease pull these awesome changes",
source_project: project,
source_branch: 'feature',
target_project: project,
target_branch: 'master',
state: 'opened',
author_id: project.creator_id,
assignee_id: nil,
created_at: created_at,
updated_at: updated_at
}
expect(pull_request.attributes).to eq(expected)
end
end
context 'when pull request is closed' do
let(:closed_at) { DateTime.strptime('2011-01-28T19:01:12Z') }
let(:raw_data) { OpenStruct.new(base_data.merge(state: 'closed', closed_at: closed_at)) }
it 'returns formatted attributes' do
expected = {
title: 'New feature',
description: "*Created by: octocat*\n\nPlease pull these awesome changes",
source_project: project,
source_branch: 'feature',
target_project: project,
target_branch: 'master',
state: 'closed',
author_id: project.creator_id,
assignee_id: nil,
created_at: created_at,
updated_at: closed_at
}
expect(pull_request.attributes).to eq(expected)
end
end
context 'when pull request is merged' do
let(:merged_at) { DateTime.strptime('2011-01-28T13:01:12Z') }
let(:raw_data) { OpenStruct.new(base_data.merge(state: 'closed', merged_at: merged_at)) }
it 'returns formatted attributes' do
expected = {
title: 'New feature',
description: "*Created by: octocat*\n\nPlease pull these awesome changes",
source_project: project,
source_branch: 'feature',
target_project: project,
target_branch: 'master',
state: 'merged',
author_id: project.creator_id,
assignee_id: nil,
created_at: created_at,
updated_at: merged_at
}
expect(pull_request.attributes).to eq(expected)
end
end
context 'when it is assigned to someone' do
let(:raw_data) { OpenStruct.new(base_data.merge(assignee: octocat)) }
it 'returns nil as assignee_id when is not a GitLab user' do
expect(pull_request.attributes.fetch(:assignee_id)).to be_nil
end
it 'returns GitLab user id as assignee_id when is a GitLab user' do
gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github')
expect(pull_request.attributes.fetch(:assignee_id)).to eq gl_user.id
end
end
context 'when author is a GitLab user' do
let(:raw_data) { OpenStruct.new(base_data.merge(user: octocat)) }
it 'returns project#creator_id as author_id when is not a GitLab user' do
expect(pull_request.attributes.fetch(:author_id)).to eq project.creator_id
end
it 'returns GitLab user id as author_id when is a GitLab user' do
gl_user = create(:omniauth_user, extern_uid: octocat.id, provider: 'github')
expect(pull_request.attributes.fetch(:author_id)).to eq gl_user.id
end
end
end
describe '#cross_project?' do
context 'when source repo is not a fork' do
let(:local_repo) { OpenStruct.new(fork: false) }
let(:source_branch) { OpenStruct.new(ref: 'feature', repo: local_repo) }
let(:raw_data) { OpenStruct.new(base_data.merge(head: source_branch)) }
it 'returns false' do
expect(pull_request.cross_project?).to eq false
end
end
context 'when source repo is a fork' do
let(:forked_repo) { OpenStruct.new(fork: true) }
let(:source_branch) { OpenStruct.new(ref: 'feature', repo: forked_repo) }
let(:raw_data) { OpenStruct.new(base_data.merge(head: source_branch)) }
it 'returns true' do
expect(pull_request.cross_project?).to eq true
end
end
end
describe '#number' do
let(:raw_data) { OpenStruct.new(base_data.merge(number: 1347)) }
it 'returns pull request number' do
expect(pull_request.number).to eq 1347
end
end
describe '#valid?' do
let(:invalid_branch) { OpenStruct.new(ref: 'invalid-branch') }
context 'when source and target branches exists' do
let(:raw_data) { OpenStruct.new(base_data.merge(head: source_branch, base: target_branch)) }
it 'returns true' do
expect(pull_request.valid?).to eq true
end
end
context 'when source branch doesn not exists' do
let(:raw_data) { OpenStruct.new(base_data.merge(head: invalid_branch, base: target_branch)) }
it 'returns false' do
expect(pull_request.valid?).to eq false
end
end
context 'when target branch doesn not exists' do
let(:raw_data) { OpenStruct.new(base_data.merge(head: source_branch, base: invalid_branch)) }
it 'returns false' do
expect(pull_request.valid?).to eq false
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment