Merge branch 'fix-bitbucket-import-issue-order' into 'master'

Import all Bitbucket issues and sort issues by creation date

Previously only the first 15 Bitbucket issues would be imported. Other issues in the API return call also calls import errors.

Tested by importing 400+ issues from this repo: https://bitbucket.org/ned/coveragepy

API reference here: https://bitbucket.org/site/master/issues/3571/api-issues-sorting-bb-3518

Closes https://github.com/gitlabhq/gitlabhq/issues/9519

See merge request !1226

CHANGELOG

@@ -2,6 +2,8 @@ Please view this file on the master branch, on stable branches it's out of date.
 
 v 8.0.0 (unreleased)
   - Prevent anchors from being hidden by header (Stan Hu)
+  - Fix bug where only the first 15 Bitbucket issues would be imported (Stan Hu)
+  - Sort issues by creation date in Bitbucket importer (Stan Hu)
   - Upgrade gitlab_git to 7.2.15 to fix `git blame` errors with ISO-encoded files (Stan Hu)
   - Prevent too many redirects upon login when home page URL is set to external_url (Stan Hu)
   - Improve dropdown positioning on the project home page (Hannes Rosenögger)

lib/gitlab/bitbucket_import/client.rb

@@ -52,11 +52,26 @@ module Gitlab
       end
 
       def issues(project_identifier)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues").body)
+        all_issues = []
+        offset = 0
+        per_page = 50  # Maximum number allowed by Bitbucket
+        index = 0
+
+        begin
+          issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body)
+          # Find out how many total issues are present
+          total = issues["count"] if index == 0
+          all_issues.concat(issues["issues"])
+          offset += issues["issues"].count
+          index += 1
+        end while all_issues.count < total
+
+        all_issues
       end
 
       def issue_comments(project_identifier, issue_id)
-        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        comments.sort_by { |comment| comment["utc_created_on"] }
       end
 
       def project(project_identifier)
@@ -100,6 +115,10 @@ module Gitlab
         response
       end
 
+      def issue_api_endpoint(project_identifier, per_page, offset)
+        "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}"
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
       end

lib/gitlab/bitbucket_import/importer.rb

@@ -20,8 +20,18 @@ module Gitlab
         #Issues && Comments
         issues = client.issues(project_identifier)
 
-        issues["issues"].each do |issue|
-          body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
+        issues.each do |issue|
+          body = ''
+          reporter = nil
+          author = 'Anonymous'
+
+          if issue["reported_by"] && issue["reported_by"]["username"]
+            reporter = issue["reported_by"]["username"]
+            author = reporter
+          end
+
+          body = @formatter.author_line(author)
+          body += issue["content"]
 
           comments = client.issue_comments(project_identifier, issue["local_id"])
 
@@ -30,14 +40,20 @@ module Gitlab
           end
 
           comments.each do |comment|
-            body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"])
+            author = 'Anonymous'
+
+            if comment["author_info"] && comment["author_info"]["username"]
+              author = comment["author_info"]["username"]
+            end
+
+            body += @formatter.comment(author, comment["utc_created_on"], comment["content"])
           end
 
           project.issues.create!(
             description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
-            author_id: gl_user_id(project, issue["reported_by"]["username"])
+            author_id: gl_user_id(project, reporter)
           )
         end
 
@@ -47,9 +63,13 @@ module Gitlab
       private
 
       def gl_user_id(project, bitbucket_id)
-        user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
-        (user && user.id) || project.creator_id
-      end
+        if bitbucket_id
+          user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
+          (user && user.id) || project.creator_id
+        else
+          project.creator_id
+        end
+       end
     end
   end
 end

lib/gitlab/github_import/importer.rb

@@ -18,7 +18,8 @@ module Gitlab
                                                   direction: :asc).each do |issue|
           if issue.pull_request.nil?
 
-            body = @formatter.author_line(issue.user.login, issue.body)
+            body = @formatter.author_line(issue.user.login)
+            body += issue.body
 
             if issue.comments > 0
               body += @formatter.comments_header

lib/gitlab/gitlab_import/importer.rb

@@ -18,7 +18,8 @@ module Gitlab
         issues = client.issues(project_identifier)
 
         issues.each do |issue|
-          body = @formatter.author_line(issue["author"]["name"], issue["description"])
+          body = @formatter.author_line(issue["author"]["name"])
+          body += issue["description"]
 
           comments = client.issue_comments(project_identifier, issue["id"])
 

lib/gitlab/import_formatter.rb

@@ -8,8 +8,8 @@ module Gitlab
       "\n\n\n**Imported comments:**\n"
     end
 
-    def author_line(author, body)
-      "*Created by: #{author}*\n\n#{body}"
+    def author_line(author)
+      "*Created by: #{author}*\n\n"
     end
   end
 end

spec/lib/gitlab/bitbucket_import/client_spec.rb

@@ -14,4 +14,38 @@ describe Gitlab::BitbucketImport::Client do
       expect(key).to be_kind_of(Symbol)
     end
   end
+
+  context 'issues' do
+    let(:per_page) { 50 }
+    let(:count) { 95 }
+    let(:sample_issues) do
+      issues = []
+
+      count.times do |i|
+        issues << { local_id: i }
+      end
+
+      issues
+    end
+    let(:first_sample_data) { { count: count, issues: sample_issues[0..per_page - 1] } }
+    let(:second_sample_data) { { count: count, issues: sample_issues[per_page..count] } }
+    let(:project_id) { 'namespace/repo' }
+
+    it 'retrieves issues over a number of pages' do
+      stub_request(:get,
+                   "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=0").
+          to_return(status: 200,
+                    body: first_sample_data.to_json,
+                    headers: {})
+
+      stub_request(:get,
+                   "https://bitbucket.org/api/1.0/repositories/#{project_id}/issues?limit=50&sort=utc_created_on&start=50").
+          to_return(status: 200,
+                    body: second_sample_data.to_json,
+                    headers: {})
+
+      issues = client.issues(project_id)
+      expect(issues.count).to eq(95)
+    end
+  end
 end