Commit b2404508 authored by Douwe Maan's avatar Douwe Maan Committed by Tomasz Maczukin

Merge branch 'gh-rate-limit'

parent 9c5b68a8
Please view this file on the master branch, on stable branches it's out of date.
v 8.8.5
- Import GitHub repositories respecting the API rate limit
v 8.8.4
- Fix LDAP-based login for users with 2FA enabled. !4493
......
......@@ -9,6 +9,10 @@ module Gitlab
@formatter = Gitlab::ImportFormatter.new
end
def create!
self.klass.create!(self.attributes)
end
private
def gl_user_id(github_id)
......
......@@ -3,6 +3,9 @@ module Gitlab
class Importer
include Gitlab::ShellAdapter
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :client, :project, :repo, :repo_url
def initialize(project)
......@@ -25,14 +28,53 @@ module Gitlab
private
def turn_auto_pagination_off!
client.auto_paginate = false
end
def turn_auto_pagination_on!
client.auto_paginate = true
end
def rate_limit
client.rate_limit!
end
def rate_limit_exceed?
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
end
def rate_limit_sleep_time
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
end
def paginate
turn_auto_pagination_off!
sleep rate_limit_sleep_time if rate_limit_exceed?
data = yield
last_response = client.last_response
while last_response.rels[:next]
sleep rate_limit_sleep_time if rate_limit_exceed?
last_response = last_response.rels[:next].get
data.concat(last_response.data) if last_response.data.is_a?(Array)
end
turn_auto_pagination_on!
data
end
def credentials
@credentials ||= project.import_data.credentials if project.import_data
end
def import_labels
client.labels(repo).each do |raw_data|
Label.create!(LabelFormatter.new(project, raw_data).attributes)
end
labels = paginate { client.labels(repo, per_page: 100) }
labels.each { |raw| LabelFormatter.new(project, raw).create! }
true
rescue ActiveRecord::RecordInvalid => e
......@@ -40,9 +82,8 @@ module Gitlab
end
def import_milestones
client.list_milestones(repo, state: :all).each do |raw_data|
Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes)
end
milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
true
rescue ActiveRecord::RecordInvalid => e
......@@ -50,16 +91,15 @@ module Gitlab
end
def import_issues
client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data|
gh_issue = IssueFormatter.new(project, raw_data)
data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
if gh_issue.valid?
issue = Issue.create!(gh_issue.attributes)
apply_labels(gh_issue.number, issue)
data.each do |raw|
gh_issue = IssueFormatter.new(project, raw)
if gh_issue.has_comments?
import_comments(gh_issue.number, issue)
end
if gh_issue.valid?
issue = gh_issue.create!
apply_labels(issue)
import_comments(issue) if gh_issue.has_comments?
end
end
......@@ -69,9 +109,8 @@ module Gitlab
end
def import_pull_requests
pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc)
.map { |raw| PullRequestFormatter.new(project, raw) }
.select(&:valid?)
pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
......@@ -80,13 +119,10 @@ module Gitlab
create_refs(branches_removed)
pull_requests.each do |pull_request|
merge_request = MergeRequest.new(pull_request.attributes)
if merge_request.save
apply_labels(pull_request.number, merge_request)
import_comments(pull_request.number, merge_request)
import_comments_on_diff(pull_request.number, merge_request)
end
merge_request = pull_request.create!
apply_labels(merge_request)
import_comments(merge_request)
import_comments_on_diff(merge_request)
end
delete_refs(branches_removed)
......@@ -98,6 +134,7 @@ module Gitlab
def create_refs(branches)
branches.each do |name, sha|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.create_ref(repo, "refs/heads/#{name}", sha)
end
......@@ -106,13 +143,16 @@ module Gitlab
def delete_refs(branches)
branches.each do |name, _|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.delete_ref(repo, "heads/#{name}")
project.repository.rm_branch(project.creator, name)
end
end
def apply_labels(number, issuable)
issue = client.issue(repo, number)
def apply_labels(issuable)
sleep rate_limit_sleep_time if rate_limit_exceed?
issue = client.issue(repo, issuable.iid)
if issue.labels.count > 0
label_ids = issue.labels.map do |raw|
......@@ -123,20 +163,20 @@ module Gitlab
end
end
def import_comments(issue_number, noteable)
comments = client.issue_comments(repo, issue_number)
create_comments(comments, noteable)
def import_comments(issuable)
comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
create_comments(issuable, comments)
end
def import_comments_on_diff(pull_request_number, merge_request)
comments = client.pull_request_comments(repo, pull_request_number)
create_comments(comments, merge_request)
def import_comments_on_diff(merge_request)
comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
create_comments(merge_request, comments)
end
def create_comments(comments, noteable)
comments.each do |raw_data|
comment = CommentFormatter.new(project, raw_data)
noteable.notes.create!(comment.attributes)
def create_comments(issuable, comments)
comments.each do |raw|
comment = CommentFormatter.new(project, raw)
issuable.notes.create!(comment.attributes)
end
end
......
......@@ -20,6 +20,10 @@ module Gitlab
raw_data.comments > 0
end
def klass
Issue
end
def number
raw_data.number
end
......
......@@ -9,6 +9,10 @@ module Gitlab
}
end
def klass
Label
end
private
def color
......
......@@ -14,6 +14,10 @@ module Gitlab
}
end
def klass
Milestone
end
private
def number
......
......@@ -24,6 +24,10 @@ module Gitlab
}
end
def klass
MergeRequest
end
def number
raw_data.number
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment