Commit 2236e9d6 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'fix-bitbucket-import-issue-order' into 'master'

Import all Bitbucket issues and sort issues by creation date

Previously only the first 15 Bitbucket issues would be imported. Other issues in the API return call also calls import errors.

Tested by importing 400+ issues from this repo: https://bitbucket.org/ned/coveragepy

API reference here: https://bitbucket.org/site/master/issues/3571/api-issues-sorting-bb-3518

Closes https://github.com/gitlabhq/gitlabhq/issues/9519

See merge request !1226
parents 2ef16e76 2e9c922d
...@@ -2,6 +2,8 @@ Please view this file on the master branch, on stable branches it's out of date. ...@@ -2,6 +2,8 @@ Please view this file on the master branch, on stable branches it's out of date.
v 8.0.0 (unreleased) v 8.0.0 (unreleased)
- Prevent anchors from being hidden by header (Stan Hu) - Prevent anchors from being hidden by header (Stan Hu)
- Fix bug where only the first 15 Bitbucket issues would be imported (Stan Hu)
- Sort issues by creation date in Bitbucket importer (Stan Hu)
- Upgrade gitlab_git to 7.2.15 to fix `git blame` errors with ISO-encoded files (Stan Hu) - Upgrade gitlab_git to 7.2.15 to fix `git blame` errors with ISO-encoded files (Stan Hu)
- Prevent too many redirects upon login when home page URL is set to external_url (Stan Hu) - Prevent too many redirects upon login when home page URL is set to external_url (Stan Hu)
- Improve dropdown positioning on the project home page (Hannes Rosenögger) - Improve dropdown positioning on the project home page (Hannes Rosenögger)
......
...@@ -52,11 +52,26 @@ module Gitlab ...@@ -52,11 +52,26 @@ module Gitlab
end end
def issues(project_identifier) def issues(project_identifier)
JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues").body) all_issues = []
offset = 0
per_page = 50 # Maximum number allowed by Bitbucket
index = 0
begin
issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
# Find out how many total issues are present
total = issues["count"] if index == 0
all_issues.concat(issues["issues"])
offset += issues["issues"].count
index += 1
end while all_issues.count < total
all_issues
end end
def issue_comments(project_identifier, issue_id) def issue_comments(project_identifier, issue_id)
JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
comments.sort_by { |comment| comment["utc_created_on"] }
end end
def project(project_identifier) def project(project_identifier)
...@@ -100,6 +115,10 @@ module Gitlab ...@@ -100,6 +115,10 @@ module Gitlab
response response
end end
def issue_api_endpoint(project_identifier, per_page, offset)
"/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
end
def config def config
Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
end end
......
...@@ -20,8 +20,18 @@ module Gitlab ...@@ -20,8 +20,18 @@ module Gitlab
#Issues && Comments #Issues && Comments
issues = client.issues(project_identifier) issues = client.issues(project_identifier)
issues["issues"].each do |issue| issues.each do |issue|
body = @formatter.author_line(issue["reported_by"]["username"], issue["content"]) body = ''
reporter = nil
author = 'Anonymous'
if issue["reported_by"] && issue["reported_by"]["username"]
reporter = issue["reported_by"]["username"]
author = reporter
end
body = @formatter.author_line(author)
body += issue["content"]
comments = client.issue_comments(project_identifier, issue["local_id"]) comments = client.issue_comments(project_identifier, issue["local_id"])
...@@ -30,14 +40,20 @@ module Gitlab ...@@ -30,14 +40,20 @@ module Gitlab
end end
comments.each do |comment| comments.each do |comment|
body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"]) author = 'Anonymous'
if comment["author_info"] && comment["author_info"]["username"]
author = comment["author_info"]["username"]
end
body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
end end
project.issues.create!( project.issues.create!(
description: body, description: body,
title: issue["title"], title: issue["title"],
state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
author_id: gl_user_id(project, issue["reported_by"]["username"]) author_id: gl_user_id(project, reporter)
) )
end end
...@@ -47,8 +63,12 @@ module Gitlab ...@@ -47,8 +63,12 @@ module Gitlab
private private
def gl_user_id(project, bitbucket_id) def gl_user_id(project, bitbucket_id)
if bitbucket_id
user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
(user && user.id) || project.creator_id (user && user.id) || project.creator_id
else
project.creator_id
end
end end
end end
end end
......
...@@ -18,7 +18,8 @@ module Gitlab ...@@ -18,7 +18,8 @@ module Gitlab
direction: :asc).each do |issue| direction: :asc).each do |issue|
if issue.pull_request.nil? if issue.pull_request.nil?
body = @formatter.author_line(issue.user.login, issue.body) body = @formatter.author_line(issue.user.login)
body += issue.body
if issue.comments > 0 if issue.comments > 0
body += @formatter.comments_header body += @formatter.comments_header
......
...@@ -18,7 +18,8 @@ module Gitlab ...@@ -18,7 +18,8 @@ module Gitlab
issues = client.issues(project_identifier) issues = client.issues(project_identifier)
issues.each do |issue| issues.each do |issue|
body = @formatter.author_line(issue["author"]["name"], issue["description"]) body = @formatter.author_line(issue["author"]["name"])
body += issue["description"]
comments = client.issue_comments(project_identifier, issue["id"]) comments = client.issue_comments(project_identifier, issue["id"])
......
...@@ -8,8 +8,8 @@ module Gitlab ...@@ -8,8 +8,8 @@ module Gitlab
"\n\n\n**Imported comments:**\n" "\n\n\n**Imported comments:**\n"
end end
def author_line(author, body) def author_line(author)
"*Created by: #{author}*\n\n#{body}" "*Created by: #{author}*\n\n"
end end
end end
end end
...@@ -14,4 +14,38 @@ describe Gitlab::BitbucketImport::Client do ...@@ -14,4 +14,38 @@ describe Gitlab::BitbucketImport::Client do
expect(key).to be_kind_of(Symbol) expect(key).to be_kind_of(Symbol)
end end
end end
context 'issues' do
let(:per_page) { 50 }
let(:count) { 95 }
let(:sample_issues) do
issues = []
count.times do |i|
issues << { local_id: i }
end
issues
end
let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } }
let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } }
let(:project_id) { 'namespace/repo' }
it 'retrieves issues over a number of pages' do
stub_request(:get,
"https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0").
to_return(status: 200,
body: first_sample_data.to_json,
headers: {})
stub_request(:get,
"https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50").
to_return(status: 200,
body: second_sample_data.to_json,
headers: {})
issues = client.issues(project_id)
expect(issues.count).to eq(95)
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment