Commit b2404508 authored by Douwe Maan's avatar Douwe Maan Committed by Tomasz Maczukin

Merge branch 'gh-rate-limit'

parent 9c5b68a8
Please view this file on the master branch, on stable branches it's out of date. Please view this file on the master branch, on stable branches it's out of date.
v 8.8.5
- Import GitHub repositories respecting the API rate limit
v 8.8.4 v 8.8.4
- Fix LDAP-based login for users with 2FA enabled. !4493 - Fix LDAP-based login for users with 2FA enabled. !4493
......
...@@ -9,6 +9,10 @@ module Gitlab ...@@ -9,6 +9,10 @@ module Gitlab
@formatter = Gitlab::ImportFormatter.new @formatter = Gitlab::ImportFormatter.new
end end
def create!
self.klass.create!(self.attributes)
end
private private
def gl_user_id(github_id) def gl_user_id(github_id)
......
...@@ -3,6 +3,9 @@ module Gitlab ...@@ -3,6 +3,9 @@ module Gitlab
class Importer class Importer
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :client, :project, :repo, :repo_url attr_reader :client, :project, :repo, :repo_url
def initialize(project) def initialize(project)
...@@ -25,14 +28,53 @@ module Gitlab ...@@ -25,14 +28,53 @@ module Gitlab
private private
def turn_auto_pagination_off!
client.auto_paginate = false
end
def turn_auto_pagination_on!
client.auto_paginate = true
end
def rate_limit
client.rate_limit!
end
def rate_limit_exceed?
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
end
def rate_limit_sleep_time
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
end
def paginate
turn_auto_pagination_off!
sleep rate_limit_sleep_time if rate_limit_exceed?
data = yield
last_response = client.last_response
while last_response.rels[:next]
sleep rate_limit_sleep_time if rate_limit_exceed?
last_response = last_response.rels[:next].get
data.concat(last_response.data) if last_response.data.is_a?(Array)
end
turn_auto_pagination_on!
data
end
def credentials def credentials
@credentials ||= project.import_data.credentials if project.import_data @credentials ||= project.import_data.credentials if project.import_data
end end
def import_labels def import_labels
client.labels(repo).each do |raw_data| labels = paginate { client.labels(repo, per_page: 100) }
Label.create!(LabelFormatter.new(project, raw_data).attributes) labels.each { |raw| LabelFormatter.new(project, raw).create! }
end
true true
rescue ActiveRecord::RecordInvalid => e rescue ActiveRecord::RecordInvalid => e
...@@ -40,9 +82,8 @@ module Gitlab ...@@ -40,9 +82,8 @@ module Gitlab
end end
def import_milestones def import_milestones
client.list_milestones(repo, state: :all).each do |raw_data| milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes) milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
end
true true
rescue ActiveRecord::RecordInvalid => e rescue ActiveRecord::RecordInvalid => e
...@@ -50,16 +91,15 @@ module Gitlab ...@@ -50,16 +91,15 @@ module Gitlab
end end
def import_issues def import_issues
client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data| data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
gh_issue = IssueFormatter.new(project, raw_data)
if gh_issue.valid? data.each do |raw|
issue = Issue.create!(gh_issue.attributes) gh_issue = IssueFormatter.new(project, raw)
apply_labels(gh_issue.number, issue)
if gh_issue.has_comments? if gh_issue.valid?
import_comments(gh_issue.number, issue) issue = gh_issue.create!
end apply_labels(issue)
import_comments(issue) if gh_issue.has_comments?
end end
end end
...@@ -69,9 +109,8 @@ module Gitlab ...@@ -69,9 +109,8 @@ module Gitlab
end end
def import_pull_requests def import_pull_requests
pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc) pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
.map { |raw| PullRequestFormatter.new(project, raw) } pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
.select(&:valid?)
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] } target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
...@@ -80,13 +119,10 @@ module Gitlab ...@@ -80,13 +119,10 @@ module Gitlab
create_refs(branches_removed) create_refs(branches_removed)
pull_requests.each do |pull_request| pull_requests.each do |pull_request|
merge_request = MergeRequest.new(pull_request.attributes) merge_request = pull_request.create!
apply_labels(merge_request)
if merge_request.save import_comments(merge_request)
apply_labels(pull_request.number, merge_request) import_comments_on_diff(merge_request)
import_comments(pull_request.number, merge_request)
import_comments_on_diff(pull_request.number, merge_request)
end
end end
delete_refs(branches_removed) delete_refs(branches_removed)
...@@ -98,6 +134,7 @@ module Gitlab ...@@ -98,6 +134,7 @@ module Gitlab
def create_refs(branches) def create_refs(branches)
branches.each do |name, sha| branches.each do |name, sha|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.create_ref(repo, "refs/heads/#{name}", sha) client.create_ref(repo, "refs/heads/#{name}", sha)
end end
...@@ -106,13 +143,16 @@ module Gitlab ...@@ -106,13 +143,16 @@ module Gitlab
def delete_refs(branches) def delete_refs(branches)
branches.each do |name, _| branches.each do |name, _|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.delete_ref(repo, "heads/#{name}") client.delete_ref(repo, "heads/#{name}")
project.repository.rm_branch(project.creator, name) project.repository.rm_branch(project.creator, name)
end end
end end
def apply_labels(number, issuable) def apply_labels(issuable)
issue = client.issue(repo, number) sleep rate_limit_sleep_time if rate_limit_exceed?
issue = client.issue(repo, issuable.iid)
if issue.labels.count > 0 if issue.labels.count > 0
label_ids = issue.labels.map do |raw| label_ids = issue.labels.map do |raw|
...@@ -123,20 +163,20 @@ module Gitlab ...@@ -123,20 +163,20 @@ module Gitlab
end end
end end
def import_comments(issue_number, noteable) def import_comments(issuable)
comments = client.issue_comments(repo, issue_number) comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
create_comments(comments, noteable) create_comments(issuable, comments)
end end
def import_comments_on_diff(pull_request_number, merge_request) def import_comments_on_diff(merge_request)
comments = client.pull_request_comments(repo, pull_request_number) comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
create_comments(comments, merge_request) create_comments(merge_request, comments)
end end
def create_comments(comments, noteable) def create_comments(issuable, comments)
comments.each do |raw_data| comments.each do |raw|
comment = CommentFormatter.new(project, raw_data) comment = CommentFormatter.new(project, raw)
noteable.notes.create!(comment.attributes) issuable.notes.create!(comment.attributes)
end end
end end
......
...@@ -20,6 +20,10 @@ module Gitlab ...@@ -20,6 +20,10 @@ module Gitlab
raw_data.comments > 0 raw_data.comments > 0
end end
def klass
Issue
end
def number def number
raw_data.number raw_data.number
end end
......
...@@ -9,6 +9,10 @@ module Gitlab ...@@ -9,6 +9,10 @@ module Gitlab
} }
end end
def klass
Label
end
private private
def color def color
......
...@@ -14,6 +14,10 @@ module Gitlab ...@@ -14,6 +14,10 @@ module Gitlab
} }
end end
def klass
Milestone
end
private private
def number def number
......
...@@ -24,6 +24,10 @@ module Gitlab ...@@ -24,6 +24,10 @@ module Gitlab
} }
end end
def klass
MergeRequest
end
def number def number
raw_data.number raw_data.number
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment