Commit a9a9f19b authored by Douwe Maan's avatar Douwe Maan

Merge branch 'gh-rate-limit' into 'master'

Wrap all rate limiting logic inside GitHub API client

## What does this MR do?

Move the actual rate limiting logic to GitHub API to clean the code inside the GitHub importer, and avoid code duplication.

## Are there points in the code the reviewer needs to double check?

No there aren't.

## Why was this MR needed?

Avoid code duplication to handle API rate limit in every call to the GitHub API.

## What are the relevant issue numbers?

There are none.

## Screenshots (if relevant)

Not relevant.

See merge request !4552
parents e0f3e44b cef3fd28
module Gitlab module Gitlab
module GithubImport module GithubImport
class Client class Client
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :client, :api attr_reader :client, :api
def initialize(access_token) def initialize(access_token)
...@@ -11,7 +14,7 @@ module Gitlab ...@@ -11,7 +14,7 @@ module Gitlab
) )
if access_token if access_token
::Octokit.auto_paginate = true ::Octokit.auto_paginate = false
@api = ::Octokit::Client.new( @api = ::Octokit::Client.new(
access_token: access_token, access_token: access_token,
...@@ -36,7 +39,7 @@ module Gitlab ...@@ -36,7 +39,7 @@ module Gitlab
def method_missing(method, *args, &block) def method_missing(method, *args, &block)
if api.respond_to?(method) if api.respond_to?(method)
api.send(method, *args, &block) request { api.send(method, *args, &block) }
else else
super(method, *args, &block) super(method, *args, &block)
end end
...@@ -55,6 +58,34 @@ module Gitlab ...@@ -55,6 +58,34 @@ module Gitlab
def github_options def github_options
config["args"]["client_options"].deep_symbolize_keys config["args"]["client_options"].deep_symbolize_keys
end end
def rate_limit
api.rate_limit!
end
def rate_limit_exceed?
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
end
def rate_limit_sleep_time
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
end
def request
sleep rate_limit_sleep_time if rate_limit_exceed?
data = yield
last_response = api.last_response
while last_response.rels[:next]
sleep rate_limit_sleep_time if rate_limit_exceed?
last_response = last_response.rels[:next].get
data.concat(last_response.data) if last_response.data.is_a?(Array)
end
data
end
end end
end end
end end
...@@ -3,9 +3,6 @@ module Gitlab ...@@ -3,9 +3,6 @@ module Gitlab
class Importer class Importer
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
GITHUB_SAFE_REMAINING_REQUESTS = 100
GITHUB_SAFE_SLEEP_TIME = 500
attr_reader :client, :project, :repo, :repo_url attr_reader :client, :project, :repo, :repo_url
def initialize(project) def initialize(project)
...@@ -28,52 +25,12 @@ module Gitlab ...@@ -28,52 +25,12 @@ module Gitlab
private private
def turn_auto_pagination_off!
client.auto_paginate = false
end
def turn_auto_pagination_on!
client.auto_paginate = true
end
def rate_limit
client.rate_limit!
end
def rate_limit_exceed?
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
end
def rate_limit_sleep_time
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
end
def paginate
turn_auto_pagination_off!
sleep rate_limit_sleep_time if rate_limit_exceed?
data = yield
last_response = client.last_response
while last_response.rels[:next]
sleep rate_limit_sleep_time if rate_limit_exceed?
last_response = last_response.rels[:next].get
data.concat(last_response.data) if last_response.data.is_a?(Array)
end
turn_auto_pagination_on!
data
end
def credentials def credentials
@credentials ||= project.import_data.credentials if project.import_data @credentials ||= project.import_data.credentials if project.import_data
end end
def import_labels def import_labels
labels = paginate { client.labels(repo, per_page: 100) } labels = client.labels(repo, per_page: 100)
labels.each { |raw| LabelFormatter.new(project, raw).create! } labels.each { |raw| LabelFormatter.new(project, raw).create! }
true true
...@@ -82,7 +39,7 @@ module Gitlab ...@@ -82,7 +39,7 @@ module Gitlab
end end
def import_milestones def import_milestones
milestones = paginate { client.milestones(repo, state: :all, per_page: 100) } milestones = client.milestones(repo, state: :all, per_page: 100)
milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
true true
...@@ -91,9 +48,9 @@ module Gitlab ...@@ -91,9 +48,9 @@ module Gitlab
end end
def import_issues def import_issues
data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) } issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
data.each do |raw| issues.each do |raw|
gh_issue = IssueFormatter.new(project, raw) gh_issue = IssueFormatter.new(project, raw)
if gh_issue.valid? if gh_issue.valid?
...@@ -112,7 +69,7 @@ module Gitlab ...@@ -112,7 +69,7 @@ module Gitlab
hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?)
disable_webhooks(hooks) disable_webhooks(hooks)
pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) } pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] } source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
...@@ -146,14 +103,12 @@ module Gitlab ...@@ -146,14 +103,12 @@ module Gitlab
def update_webhooks(hooks, options) def update_webhooks(hooks, options)
hooks.each do |hook| hooks.each do |hook|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.edit_hook(repo, hook.id, hook.name, hook.config, options) client.edit_hook(repo, hook.id, hook.name, hook.config, options)
end end
end end
def restore_branches(branches) def restore_branches(branches)
branches.each do |name, sha| branches.each do |name, sha|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.create_ref(repo, "refs/heads/#{name}", sha) client.create_ref(repo, "refs/heads/#{name}", sha)
end end
...@@ -162,15 +117,12 @@ module Gitlab ...@@ -162,15 +117,12 @@ module Gitlab
def clean_up_restored_branches(branches) def clean_up_restored_branches(branches)
branches.each do |name, _| branches.each do |name, _|
sleep rate_limit_sleep_time if rate_limit_exceed?
client.delete_ref(repo, "heads/#{name}") client.delete_ref(repo, "heads/#{name}")
project.repository.rm_branch(project.creator, name) project.repository.rm_branch(project.creator, name)
end end
end end
def apply_labels(issuable) def apply_labels(issuable)
sleep rate_limit_sleep_time if rate_limit_exceed?
issue = client.issue(repo, issuable.iid) issue = client.issue(repo, issuable.iid)
if issue.labels.count > 0 if issue.labels.count > 0
...@@ -183,12 +135,12 @@ module Gitlab ...@@ -183,12 +135,12 @@ module Gitlab
end end
def import_comments(issuable) def import_comments(issuable)
comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) } comments = client.issue_comments(repo, issuable.iid, per_page: 100)
create_comments(issuable, comments) create_comments(issuable, comments)
end end
def import_comments_on_diff(merge_request) def import_comments_on_diff(merge_request)
comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) } comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100)
create_comments(merge_request, comments) create_comments(merge_request, comments)
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment