Commit bb011d3f authored by Dan Davison's avatar Dan Davison

Merge branch 'acunskis-handle-errors-paginate' into 'master'

E2E: Add retry capability for paginated request and correctly handle errors

See merge request gitlab-org/gitlab!67305
parents 6283f5b8 c101c26c
...@@ -84,10 +84,13 @@ module QA ...@@ -84,10 +84,13 @@ module QA
# Get issue comments # Get issue comments
# #
# @return [Array] # @return [Array]
def comments(auto_paginate: false) def comments(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_comments_path)) unless auto_paginate return parse_body(api_get_from(api_comments_path)) unless auto_paginate
auto_paginated_response(Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url) auto_paginated_response(
Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url,
attempts: attempts
)
end end
end end
end end
......
...@@ -160,10 +160,13 @@ module QA ...@@ -160,10 +160,13 @@ module QA
# Get MR comments # Get MR comments
# #
# @return [Array] # @return [Array]
def comments(auto_paginate: false) def comments(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_comments_path)) unless auto_paginate return parse_body(api_get_from(api_comments_path)) unless auto_paginate
auto_paginated_response(Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url) auto_paginated_response(
Runtime::API::Request.new(api_client, api_comments_path, per_page: '100').url,
attempts: attempts
)
end end
private private
......
...@@ -264,21 +264,24 @@ module QA ...@@ -264,21 +264,24 @@ module QA
result = parse_body(response) result = parse_body(response)
Runtime::Logger.error("Import failed: #{result[:import_error]}") if result[:import_status] == "failed" if result[:import_status] == "failed"
Runtime::Logger.error("Import failed: #{result[:import_error]}")
Runtime::Logger.error("Failed relations: #{result[:failed_relations]}")
end
result[:import_status] result[:import_status]
end end
def commits(auto_paginate: false) def commits(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_commits_path)) unless auto_paginate return parse_body(api_get_from(api_commits_path)) unless auto_paginate
auto_paginated_response(request_url(api_commits_path, per_page: '100')) auto_paginated_response(request_url(api_commits_path, per_page: '100'), attempts: attempts)
end end
def merge_requests(auto_paginate: false) def merge_requests(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_merge_requests_path)) unless auto_paginate return parse_body(api_get_from(api_merge_requests_path)) unless auto_paginate
auto_paginated_response(request_url(api_merge_requests_path, per_page: '100')) auto_paginated_response(request_url(api_merge_requests_path, per_page: '100'), attempts: attempts)
end end
def merge_request_with_title(title) def merge_request_with_title(title)
...@@ -302,10 +305,10 @@ module QA ...@@ -302,10 +305,10 @@ module QA
parse_body(response) parse_body(response)
end end
def repository_branches(auto_paginate: false) def repository_branches(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_repository_branches_path)) unless auto_paginate return parse_body(api_get_from(api_repository_branches_path)) unless auto_paginate
auto_paginated_response(request_url(api_repository_branches_path, per_page: '100')) auto_paginated_response(request_url(api_repository_branches_path, per_page: '100'), attempts: attempts)
end end
def repository_tags def repository_tags
...@@ -328,22 +331,22 @@ module QA ...@@ -328,22 +331,22 @@ module QA
parse_body(response) parse_body(response)
end end
def issues(auto_paginate: false) def issues(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_issues_path)) unless auto_paginate return parse_body(api_get_from(api_issues_path)) unless auto_paginate
auto_paginated_response(request_url(api_issues_path, per_page: '100')) auto_paginated_response(request_url(api_issues_path, per_page: '100'), attempts: attempts)
end end
def labels(auto_paginate: false) def labels(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_labels_path)) unless auto_paginate return parse_body(api_get_from(api_labels_path)) unless auto_paginate
auto_paginated_response(request_url(api_labels_path, per_page: '100')) auto_paginated_response(request_url(api_labels_path, per_page: '100'), attempts: attempts)
end end
def milestones(auto_paginate: false) def milestones(auto_paginate: false, attempts: 0)
return parse_body(api_get_from(api_milestones_path)) unless auto_paginate return parse_body(api_get_from(api_milestones_path)) unless auto_paginate
auto_paginated_response(request_url(api_milestones_path, per_page: '100')) auto_paginated_response(request_url(api_milestones_path, per_page: '100'), attempts: attempts)
end end
def wikis def wikis
......
...@@ -6,6 +6,10 @@ module QA ...@@ -6,6 +6,10 @@ module QA
class Request class Request
API_VERSION = 'v4' API_VERSION = 'v4'
def self.masked_url(url)
url.sub(/private_token=.*/, "private_token=[****]")
end
def initialize(api_client, path, **query_string) def initialize(api_client, path, **query_string)
query_string[:private_token] ||= api_client.personal_access_token unless query_string[:oauth_access_token] query_string[:private_token] ||= api_client.personal_access_token unless query_string[:oauth_access_token]
request_path = request_path(path, **query_string) request_path = request_path(path, **query_string)
...@@ -13,7 +17,7 @@ module QA ...@@ -13,7 +17,7 @@ module QA
end end
def mask_url def mask_url
@session_address.address.sub(/private_token=.*/, "private_token=[****]") QA::Runtime::API::Request.masked_url(url)
end end
def url def url
......
...@@ -135,10 +135,12 @@ module QA ...@@ -135,10 +135,12 @@ module QA
imported_project # import the project imported_project # import the project
fetch_github_objects # fetch all objects right after import has started fetch_github_objects # fetch all objects right after import has started
expect { imported_project.reload!.import_status }.to eventually_eq('finished').within( import_status = lambda do
duration: 3600, imported_project.reload!.import_status.tap do |status|
interval: 30 raise "Import of '#{imported_project.name}' failed!" if status == 'failed'
) end
end
expect(import_status).to eventually_eq('finished').within(duration: 3600, interval: 30)
@import_time = Time.now - start @import_time = Time.now - start
aggregate_failures do aggregate_failures do
...@@ -264,7 +266,7 @@ module QA ...@@ -264,7 +266,7 @@ module QA
def gl_commits def gl_commits
@gl_commits ||= begin @gl_commits ||= begin
logger.debug("= Fetching commits =") logger.debug("= Fetching commits =")
imported_project.commits(auto_paginate: true).map { |c| c[:id] } imported_project.commits(auto_paginate: true, attempts: 2).map { |c| c[:id] }
end end
end end
...@@ -294,7 +296,7 @@ module QA ...@@ -294,7 +296,7 @@ module QA
def mrs def mrs
@mrs ||= begin @mrs ||= begin
logger.debug("= Fetching merge requests =") logger.debug("= Fetching merge requests =")
imported_mrs = imported_project.merge_requests(auto_paginate: true) imported_mrs = imported_project.merge_requests(auto_paginate: true, attempts: 2)
logger.debug("= Transforming merge request objects for comparison =") logger.debug("= Transforming merge request objects for comparison =")
imported_mrs.each_with_object({}) do |mr, hash| imported_mrs.each_with_object({}) do |mr, hash|
resource = Resource::MergeRequest.init do |resource| resource = Resource::MergeRequest.init do |resource|
...@@ -305,7 +307,7 @@ module QA ...@@ -305,7 +307,7 @@ module QA
hash[mr[:title]] = { hash[mr[:title]] = {
body: mr[:description], body: mr[:description],
comments: resource.comments(auto_paginate: true) comments: resource.comments(auto_paginate: true, attempts: 2)
# remove system notes # remove system notes
.reject { |c| c[:system] || c[:body].match?(/^(\*\*Review:\*\*)|(\*Merged by:).*/) } .reject { |c| c[:system] || c[:body].match?(/^(\*\*Review:\*\*)|(\*Merged by:).*/) }
.map { |c| sanitize(c[:body]) } .map { |c| sanitize(c[:body]) }
...@@ -320,7 +322,7 @@ module QA ...@@ -320,7 +322,7 @@ module QA
def gl_issues def gl_issues
@gl_issues ||= begin @gl_issues ||= begin
logger.debug("= Fetching issues =") logger.debug("= Fetching issues =")
imported_issues = imported_project.issues(auto_paginate: true) imported_issues = imported_project.issues(auto_paginate: true, attempts: 2)
logger.debug("= Transforming issue objects for comparison =") logger.debug("= Transforming issue objects for comparison =")
imported_issues.each_with_object({}) do |issue, hash| imported_issues.each_with_object({}) do |issue, hash|
resource = Resource::Issue.init do |issue_resource| resource = Resource::Issue.init do |issue_resource|
...@@ -331,7 +333,7 @@ module QA ...@@ -331,7 +333,7 @@ module QA
hash[issue[:title]] = { hash[issue[:title]] = {
body: issue[:description], body: issue[:description],
comments: resource.comments(auto_paginate: true).map { |c| sanitize(c[:body]) } comments: resource.comments(auto_paginate: true, attempts: 2).map { |c| sanitize(c[:body]) }
} }
end end
end end
......
...@@ -79,16 +79,27 @@ module QA ...@@ -79,16 +79,27 @@ module QA
error.response error.response
end end
def auto_paginated_response(url) def auto_paginated_response(url, attempts: 0)
pages = [] pages = []
with_paginated_response_body(url) { |response| pages << response } with_paginated_response_body(url, attempts: attempts) { |response| pages << response }
pages.flatten pages.flatten
end end
def with_paginated_response_body(url) def with_paginated_response_body(url, attempts: 0)
not_ok_error = lambda do |resp|
raise "Failed to GET #{QA::Runtime::API::Request.masked_url(url)} - (#{resp.code}): `#{resp}`."
end
loop do loop do
response = get(url) response = if attempts > 0
Retrier.retry_on_exception(max_attempts: attempts, log: false) do
get(url).tap { |resp| not_ok_error.call(resp) if resp.code != HTTP_STATUS_OK }
end
else
get(url).tap { |resp| not_ok_error.call(resp) if resp.code != HTTP_STATUS_OK }
end
page, pages = response.headers.values_at(:x_page, :x_total_pages) page, pages = response.headers.values_at(:x_page, :x_total_pages)
api_endpoint = url.match(%r{v4/(\S+)\?})[1] api_endpoint = url.match(%r{v4/(\S+)\?})[1]
...@@ -104,7 +115,10 @@ module QA ...@@ -104,7 +115,10 @@ module QA
end end
def pagination_links(response) def pagination_links(response)
response.headers[:link].split(',').map do |link| link = response.headers[:link]
return unless link
link.split(',').map do |link|
match = link.match(/<(?<url>.*)>; rel="(?<rel>\w+)"/) match = link.match(/<(?<url>.*)>; rel="(?<rel>\w+)"/)
break nil unless match break nil unless match
......
...@@ -11,7 +11,15 @@ module QA ...@@ -11,7 +11,15 @@ module QA
RetriesExceededError = Class.new(RepeaterConditionExceededError) RetriesExceededError = Class.new(RepeaterConditionExceededError)
WaitExceededError = Class.new(RepeaterConditionExceededError) WaitExceededError = Class.new(RepeaterConditionExceededError)
def repeat_until(max_attempts: nil, max_duration: nil, reload_page: nil, sleep_interval: 0, raise_on_failure: true, retry_on_exception: false, log: true) def repeat_until(
max_attempts: nil,
max_duration: nil,
reload_page: nil,
sleep_interval: 0,
raise_on_failure: true,
retry_on_exception: false,
log: true
)
attempts = 0 attempts = 0
start = Time.now start = Time.now
...@@ -29,17 +37,19 @@ module QA ...@@ -29,17 +37,19 @@ module QA
raise unless retry_on_exception raise unless retry_on_exception
attempts += 1 attempts += 1
if remaining_attempts?(attempts, max_attempts) && remaining_time?(start, max_duration) raise unless remaining_attempts?(attempts, max_attempts) && remaining_time?(start, max_duration)
sleep_and_reload_if_needed(sleep_interval, reload_page)
retry sleep_and_reload_if_needed(sleep_interval, reload_page)
else retry
raise
end
end end
if raise_on_failure if raise_on_failure
raise RetriesExceededError, "Retry condition not met after #{max_attempts} #{'attempt'.pluralize(max_attempts)}" unless remaining_attempts?(attempts, max_attempts) unless remaining_attempts?(attempts, max_attempts)
raise(
RetriesExceededError,
"Retry condition not met after #{max_attempts} #{'attempt'.pluralize(max_attempts)}"
)
end
raise WaitExceededError, "Wait condition not met after #{max_duration} #{'second'.pluralize(max_duration)}" raise WaitExceededError, "Wait condition not met after #{max_duration} #{'second'.pluralize(max_duration)}"
end end
......
...@@ -7,21 +7,21 @@ module QA ...@@ -7,21 +7,21 @@ module QA
module_function module_function
def retry_on_exception(max_attempts: 3, reload_page: nil, sleep_interval: 0.5) def retry_on_exception(max_attempts: 3, reload_page: nil, sleep_interval: 0.5, log: true)
QA::Runtime::Logger.debug( if log
<<~MSG.tr("\n", ' ') msg = ["with retry_on_exception: max_attempts: #{max_attempts}"]
with retry_on_exception: max_attempts: #{max_attempts}; msg << "reload_page: #{reload_page}" if reload_page
reload_page: #{reload_page}; msg << "sleep_interval: #{sleep_interval}"
sleep_interval: #{sleep_interval} QA::Runtime::Logger.debug(msg.join('; '))
MSG end
)
result = nil result = nil
repeat_until( repeat_until(
max_attempts: max_attempts, max_attempts: max_attempts,
reload_page: reload_page, reload_page: reload_page,
sleep_interval: sleep_interval, sleep_interval: sleep_interval,
retry_on_exception: true retry_on_exception: true,
log: log
) do ) do
result = yield result = yield
...@@ -29,7 +29,7 @@ module QA ...@@ -29,7 +29,7 @@ module QA
# We set it to `true` so that it doesn't repeat if there's no exception # We set it to `true` so that it doesn't repeat if there's no exception
true true
end end
QA::Runtime::Logger.debug("ended retry_on_exception") QA::Runtime::Logger.debug("ended retry_on_exception") if log
result result
end end
......
...@@ -70,8 +70,9 @@ RSpec.describe QA::Support::Retrier do ...@@ -70,8 +70,9 @@ RSpec.describe QA::Support::Retrier do
describe '.retry_on_exception' do describe '.retry_on_exception' do
context 'when the condition is true' do context 'when the condition is true' do
it 'logs max_attempts, reload_page, and sleep_interval parameters' do it 'logs max_attempts, reload_page, and sleep_interval parameters' do
expect { subject.retry_on_exception(max_attempts: 1, reload_page: nil, sleep_interval: 0) { true } } message = /with retry_on_exception: max_attempts: 1; reload_page: true; sleep_interval: 0/
.to output(/with retry_on_exception: max_attempts: 1; reload_page: ; sleep_interval: 0/).to_stdout_from_any_process expect { subject.retry_on_exception(max_attempts: 1, reload_page: true, sleep_interval: 0) { true } }
.to output(message).to_stdout_from_any_process
end end
it 'logs the end' do it 'logs the end' do
...@@ -82,8 +83,9 @@ RSpec.describe QA::Support::Retrier do ...@@ -82,8 +83,9 @@ RSpec.describe QA::Support::Retrier do
context 'when the condition is false' do context 'when the condition is false' do
it 'logs the start' do it 'logs the start' do
expect { subject.retry_on_exception(max_attempts: 1, reload_page: nil, sleep_interval: 0) { false } } message = /with retry_on_exception: max_attempts: 1; reload_page: true; sleep_interval: 0/
.to output(/with retry_on_exception: max_attempts: 1; reload_page: ; sleep_interval: 0/).to_stdout_from_any_process expect { subject.retry_on_exception(max_attempts: 1, reload_page: true, sleep_interval: 0) { false } }
.to output(message).to_stdout_from_any_process
end end
it 'logs the end' do it 'logs the end' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment