Skip to content

G
gitlab-ce
Commit 517f4257 authored by Yorick Peterse's avatar Yorick Peterse
Browse files
Options

Resolved GH importer conflicts

parent d6d04136
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 378 deletions
lib/gitlab/github_import/client.rb
View file @ 517f4257
......@@ -15,167 +15,6 @@ module Gitlab
# end
class Client
attr_reader :octokit
<<<<<<< HEAD
# A single page of data and the corresponding page number.
Page = Struct.new(:objects, :number)
# The minimum number of requests we want to keep available.
#
# We don't use a value of 0 as multiple threads may be using the same
# token in parallel. This could result in all of them hitting the GitHub
# rate limit at once. The threshold is put in place to not hit the limit
# in most cases.
RATE_LIMIT_THRESHOLD = 50
# token - The GitHub API token to use.
#
# per_page - The number of objects that should be displayed per page.
#
# parallel - When set to true hitting the rate limit will result in a
# dedicated error being raised. When set to `false` we will
# instead just `sleep()` until the rate limit is reset. Setting
# this value to `true` for parallel importing is crucial as
# otherwise hitting the rate limit will result in a thread
# being blocked in a `sleep()` call for up to an hour.
def initialize(token, per_page: 100, parallel: true)
@octokit = Octokit::Client.new(access_token: token, per_page: per_page)
@parallel = parallel
end
def parallel?
@parallel
end
# Returns the details of a GitHub user.
#
# username - The username of the user.
def user(username)
with_rate_limit { octokit.user(username) }
end
# Returns the details of a GitHub repository.
#
# name - The path (in the form `owner/repository`) of the repository.
def repository(name)
with_rate_limit { octokit.repo(name) }
end
def labels(*args)
each_object(:labels, *args)
end
def milestones(*args)
each_object(:milestones, *args)
end
def releases(*args)
each_object(:releases, *args)
end
# Fetches data from the GitHub API and yields a Page object for every page
# of data, without loading all of them into memory.
#
# method - The Octokit method to use for getting the data.
# args - Arguments to pass to the Octokit method.
#
# rubocop: disable GitlabSecurity/PublicSend
def each_page(method, *args, &block)
return to_enum(__method__, method, *args) unless block_given?
page =
if args.last.is_a?(Hash) && args.last[:page]
args.last[:page]
else
1
end
collection = with_rate_limit { octokit.public_send(method, *args) }
next_url = octokit.last_response.rels[:next]
yield Page.new(collection, page)
while next_url
response = with_rate_limit { next_url.get }
next_url = response.rels[:next]
yield Page.new(response.data, page += 1)
end
end
# Iterates over all of the objects for the given method (e.g. `:labels`).
#
# method - The method to send to Octokit for querying data.
# args - Any arguments to pass to the Octokit method.
def each_object(method, *args, &block)
return to_enum(__method__, method, *args) unless block_given?
each_page(method, *args) do |page|
page.objects.each do |object|
yield object
end
end
end
# Yields the supplied block, responding to any rate limit errors.
#
# The exact strategy used for handling rate limiting errors depends on
# whether we are running in parallel mode or not. For more information see
# `#rate_or_wait_for_rate_limit`.
def with_rate_limit
request_count_counter.increment
raise_or_wait_for_rate_limit unless requests_remaining?
begin
yield
rescue Octokit::TooManyRequests
raise_or_wait_for_rate_limit
# This retry will only happen when running in sequential mode as we'll
# raise an error in parallel mode.
retry
end
end
# Returns `true` if we're still allowed to perform API calls.
def requests_remaining?
remaining_requests > RATE_LIMIT_THRESHOLD
end
def remaining_requests
octokit.rate_limit.remaining
end
def raise_or_wait_for_rate_limit
rate_limit_counter.increment
if parallel?
raise RateLimitError
else
sleep(rate_limit_resets_in)
end
end
def rate_limit_resets_in
# We add a few seconds to the rate limit so we don't _immediately_
# resume when the rate limit resets as this may result in us performing
# a request before GitHub has a chance to reset the limit.
octokit.rate_limit.resets_in + 5
end
def respond_to_missing?(method, include_private = false)
octokit.respond_to?(method, include_private)
end
def rate_limit_counter
@rate_limit_counter ||= Gitlab::Metrics.counter(
:github_importer_rate_limit_hits,
'The number of times we hit the GitHub rate limit when importing projects'
)
end
=======
# A single page of data and the corresponding page number.
Page = Struct.new(:objects, :number)
......@@ -367,7 +206,6 @@ module Gitlab
)
end
>>>>>>> ce/master
def request_count_counter
@request_counter ||= Gitlab::Metrics.counter(
:github_importer_request_count,
......
spec/lib/gitlab/github_import/client_spec.rb
View file @ 517f4257
......@@ -48,7 +48,6 @@ describe Gitlab::GithubImport::Client do
client.labels('foo/bar')
end
end
<<<<<<< HEAD
describe '#milestones' do
it 'returns the milestones' do
......@@ -66,25 +65,6 @@ describe Gitlab::GithubImport::Client do
it 'returns the releases' do
client = described_class.new('foo')
=======
describe '#milestones' do
it 'returns the milestones' do
client = described_class.new('foo')
expect(client)
.to receive(:each_object)
.with(:milestones, 'foo/bar')
client.milestones('foo/bar')
end
end
describe '#releases' do
it 'returns the releases' do
client = described_class.new('foo')
>>>>>>> ce/master
expect(client)
.to receive(:each_object)
.with(:releases, 'foo/bar')
......@@ -109,12 +89,6 @@ describe Gitlab::GithubImport::Client do
response = double(:response, data: [object2], rels: { next: nil })
next_page = double(:next_page, get: response)
<<<<<<< HEAD
allow(client.octokit)
.to receive(:last_response)
.and_return(double(:last_response, rels: { next: next_page }))
=======
allow(client.octokit)
.to receive(:last_response)
......@@ -294,47 +268,9 @@ describe Gitlab::GithubImport::Client do
expect(client.octokit).to receive(:rate_limit).and_return(rate_limit)
expect(client.rate_limit_resets_in).to eq(6)
>>>>>>> ce/master
end
end
<<<<<<< HEAD
context 'without a block' do
it 'returns an Enumerator' do
expect(client.each_page(:foo)).to be_an_instance_of(Enumerator)
end
it 'the returned Enumerator returns Page objects' do
enum = client.each_page(:foo)
page1 = enum.next
page2 = enum.next
expect(page1).to be_an_instance_of(described_class::Page)
expect(page2).to be_an_instance_of(described_class::Page)
expect(page1.objects).to eq([object1])
expect(page1.number).to eq(1)
expect(page2.objects).to eq([object2])
expect(page2.number).to eq(2)
end
end
context 'with a block' do
it 'yields every retrieved page to the supplied block' do
pages = []
client.each_page(:foo) { |page| pages << page }
expect(pages[0]).to be_an_instance_of(described_class::Page)
expect(pages[1]).to be_an_instance_of(described_class::Page)
expect(pages[0].objects).to eq([object1])
expect(pages[0].number).to eq(1)
expect(pages[1].objects).to eq([object2])
expect(pages[1].number).to eq(2)
=======
describe '#api_endpoint' do
let(:client) { described_class.new('foo') }
......@@ -357,7 +293,6 @@ describe Gitlab::GithubImport::Client do
.and_return(endpoint)
expect(client.api_endpoint).to eq(endpoint)
>>>>>>> ce/master
end
end
end
......@@ -383,46 +318,6 @@ describe Gitlab::GithubImport::Client do
.to receive(:github_omniauth_provider)
.and_return({ 'args' => { 'client_options' => { 'site' => endpoint } } })
<<<<<<< HEAD
it 'starts at the given page' do
pages = []
client.each_page(:foo, page: 2) { |page| pages << page }
expect(pages[0].number).to eq(2)
expect(pages[1].number).to eq(3)
end
end
end
describe '#with_rate_limit' do
let(:client) { described_class.new('foo') }
it 'yields the supplied block when enough requests remain' do
expect(client).to receive(:requests_remaining?).and_return(true)
expect { |b| client.with_rate_limit(&b) }.to yield_control
end
it 'waits before yielding if not enough requests remain' do
expect(client).to receive(:requests_remaining?).and_return(false)
expect(client).to receive(:raise_or_wait_for_rate_limit)
expect { |b| client.with_rate_limit(&b) }.to yield_control
end
it 'waits and retries the operation if all requests were consumed in the supplied block' do
retries = 0
expect(client).to receive(:requests_remaining?).and_return(true)
expect(client).to receive(:raise_or_wait_for_rate_limit)
client.with_rate_limit do
if retries.zero?
retries += 1
raise(Octokit::TooManyRequests)
end
=======
expect(client.custom_api_endpoint).to eq(endpoint)
end
end
......@@ -475,119 +370,10 @@ describe Gitlab::GithubImport::Client do
expect(hash['name']).to eq('github')
expect(hash['url']).to eq('https://github.com/')
>>>>>>> ce/master
end
expect(retries).to eq(1)
end
it 'increments the request count counter' do
expect(client.request_count_counter)
.to receive(:increment)
.and_call_original
expect(client).to receive(:requests_remaining?).and_return(true)
client.with_rate_limit { }
end
end
describe '#requests_remaining?' do
let(:client) { described_class.new('foo') }
it 'returns true if enough requests remain' do
expect(client).to receive(:remaining_requests).and_return(9000)
expect(client.requests_remaining?).to eq(true)
end
it 'returns false if not enough requests remain' do
expect(client).to receive(:remaining_requests).and_return(1)
expect(client.requests_remaining?).to eq(false)
end
end
describe '#raise_or_wait_for_rate_limit' do
it 'raises RateLimitError when running in parallel mode' do
client = described_class.new('foo', parallel: true)
expect { client.raise_or_wait_for_rate_limit }
.to raise_error(Gitlab::GithubImport::RateLimitError)
end
it 'sleeps when running in sequential mode' do
client = described_class.new('foo', parallel: false)
expect(client).to receive(:rate_limit_resets_in).and_return(1)
expect(client).to receive(:sleep).with(1)
client.raise_or_wait_for_rate_limit
end
it 'increments the rate limit counter' do
client = described_class.new('foo', parallel: false)
expect(client)
.to receive(:rate_limit_resets_in)
.and_return(1)
expect(client)
.to receive(:sleep)
.with(1)
expect(client.rate_limit_counter)
.to receive(:increment)
.and_call_original
client.raise_or_wait_for_rate_limit
end
end
<<<<<<< HEAD
describe '#remaining_requests' do
it 'returns the number of remaining requests' do
client = described_class.new('foo')
rate_limit = double(remaining: 1)
expect(client.octokit).to receive(:rate_limit).and_return(rate_limit)
expect(client.remaining_requests).to eq(1)
end
end
describe '#rate_limit_resets_in' do
it 'returns the number of seconds after which the rate limit is reset' do
client = described_class.new('foo')
rate_limit = double(resets_in: 1)
expect(client.octokit).to receive(:rate_limit).and_return(rate_limit)
expect(client.rate_limit_resets_in).to eq(6)
end
end
describe '#method_missing' do
it 'delegates missing methods to the request method' do
client = described_class.new('foo')
expect(client).to receive(:milestones).with(state: 'all')
client.milestones(state: 'all')
end
end
describe '#respond_to_missing?' do
it 'returns true for methods supported by Octokit' do
client = described_class.new('foo')
expect(client.respond_to?(:milestones)).to eq(true)
end
it 'returns false for methods not supported by Octokit' do
client = described_class.new('foo')
expect(client.respond_to?(:kittens)).to eq(false)
=======
describe '#rate_limiting_enabled?' do
let(:client) { described_class.new('foo') }
......@@ -601,7 +387,6 @@ describe Gitlab::GithubImport::Client do
.and_return('https://github.kittens.com/')
expect(client.rate_limiting_enabled?).to eq(false)
>>>>>>> ce/master
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7