Commit 815208ba authored by Stan Hu's avatar Stan Hu

Merge branch 'sh-patch-google-api-client' into 'master'

Extend Google Cloud Storage max transfer timeout to 60 minutes

See merge request gitlab-org/gitlab!78526
parents 9ca38fa6 af96b09e
...@@ -10,3 +10,14 @@ require 'signet/errors' ...@@ -10,3 +10,14 @@ require 'signet/errors'
# enabling retries is strongly encouraged but disabled by default. Large uploads # enabling retries is strongly encouraged but disabled by default. Large uploads
# that may hit timeouts will mainly benefit from this. # that may hit timeouts will mainly benefit from this.
Google::Apis::RequestOptions.default.retries = 3 if Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_GOOGLE_API_RETRIES', true)) Google::Apis::RequestOptions.default.retries = 3 if Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_GOOGLE_API_RETRIES', true))
# By default, httpclient will set a send timeout of 120 seconds (https://github.com/nahi/httpclient/blob/82929c4baae14c2319c3f9aba49488c6f6def875/lib/httpclient/session.rb#L147),
# which causes any request to be interrupted every 2 minutes (https://github.com/nahi/httpclient/blob/82929c4baae14c2319c3f9aba49488c6f6def875/lib/httpclient/session.rb#L515).
#
# The Google API client uses resumable uploads so that if a transfer
# request is interrupted, it can retry where it left off. The client
# will retry at most N + 1 times, which means transfers can only last as
# long as this (N + 1) * send timeout. We raise this timeout to an hour
# since otherwise transfers can only last 8 minutes (4 * 2 min) before
# being interrupted.
Google::Apis::ClientOptions.default.send_timeout_sec = 3600
# frozen_string_literal: true
require 'google/apis/core/http_command'
raise 'This patch is only tested with google-api-client-ruby v0.50.0' unless Google::Apis::VERSION == "0.50.0"
# The google-api-ruby-client does not have a way to increase or disable
# the maximum allowed time for a request to be retried. By default, it
# is using the Retriable gem's 15-minute timeout, which appears to be
# too low for uploads over 10 GB. This patches the gem with the upstream
# changes:
# https://github.com/googleapis/google-api-ruby-client/pull/8106
module Google
module Apis
module Core
# Command for HTTP request/response.
class HttpCommand
MAX_ELAPSED_TIME = 3600
# Execute the command, retrying as necessary
#
# @param [HTTPClient] client
# HTTP client
# @yield [result, err] Result or error if block supplied
# @return [Object]
# @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
# @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
# @raise [Google::Apis::AuthorizationError] Authorization is required
def execute(client)
prepare!
opencensus_begin_span
begin
Retriable.retriable tries: options.retries + 1,
max_elapsed_time: MAX_ELAPSED_TIME,
base_interval: 1,
multiplier: 2,
on: RETRIABLE_ERRORS do |try|
# This 2nd level retriable only catches auth errors, and supports 1 retry, which allows
# auth to be re-attempted without having to retry all sorts of other failures like
# NotFound, etc
auth_tries = (try == 1 && authorization_refreshable? ? 2 : 1)
Retriable.retriable tries: auth_tries,
on: [Google::Apis::AuthorizationError, Signet::AuthorizationError, Signet::RemoteServerError, Signet::UnexpectedStatusError],
on_retry: proc { |*| refresh_authorization } do
execute_once(client).tap do |result|
if block_given?
yield result, nil
end
end
end
end
rescue => e # rubocop:disable Style/RescueStandardError
if block_given?
yield nil, e
else
raise e
end
end
ensure
opencensus_end_span
@http_res = nil
release!
end
end
end
end
end
# frozen_string_literal: true
# Extracted from https://github.com/googleapis/google-api-ruby-client/blob/main/google-apis-core/spec/google/apis/core/http_command_spec.rb
require 'spec_helper'
require 'google/apis/core/base_service'
RSpec.describe Google::Apis::Core::HttpCommand do # rubocop:disable RSpec/FilePath
context('with a successful response') do
let(:client) { Google::Apis::Core::BaseService.new('', '').client }
let(:command) { Google::Apis::Core::HttpCommand.new(:get, 'https://www.googleapis.com/zoo/animals') }
before do
stub_request(:get, 'https://www.googleapis.com/zoo/animals').to_return(body: %(Hello world))
end
it 'returns the response body if block not present' do
result = command.execute(client)
expect(result).to eql 'Hello world'
end
it 'calls block if present' do
expect { |b| command.execute(client, &b) }.to yield_with_args('Hello world', nil)
end
it 'retries with max elapsed_time and retries' do
expect(Retriable).to receive(:retriable).with(
tries: Google::Apis::RequestOptions.default.retries + 1,
max_elapsed_time: 3600,
base_interval: 1,
multiplier: 2,
on: described_class::RETRIABLE_ERRORS).and_call_original
allow(Retriable).to receive(:retriable).and_call_original
command.execute(client)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment