Commit 4fe77a6a authored by Stan Hu's avatar Stan Hu

Extend Google Cloud Storage max transfer timeout to 60 minutes

By default, the Retriable gem sets a max_elapsed_time to 15
minutes. This can result in large transfers to Google Cloud Storage
being aborted prematurely rather than being retried. Raise the max
timeout from 15 minutes to 60 minutes with a monkey patch extracted
from https://github.com/googleapis/google-api-ruby-client/pull/8106.

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/349425

Changelog: fixed
parent 0245456e
# frozen_string_literal: true
require 'google/apis/core/http_command'
raise 'This patch is only tested with google-api-client-ruby v0.50.0' unless Google::Apis::VERSION == "0.50.0"
# The google-api-ruby-client does not have a way to increase or disable
# the maximum allowed time for a request to be retried. By default, it
# is using the Retriable gem's 15-minute timeout, which appears to be
# too low for uploads over 10 GB. This patches the gem with the upstream
# changes:
# https://github.com/googleapis/google-api-ruby-client/pull/8106
module Google
module Apis
module Core
# Command for HTTP request/response.
class HttpCommand
MAX_ELAPSED_TIME = 3600
# Execute the command, retrying as necessary
#
# @param [HTTPClient] client
# HTTP client
# @yield [result, err] Result or error if block supplied
# @return [Object]
# @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
# @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
# @raise [Google::Apis::AuthorizationError] Authorization is required
def execute(client)
prepare!
opencensus_begin_span
begin
Retriable.retriable tries: options.retries + 1,
max_elapsed_time: MAX_ELAPSED_TIME,
base_interval: 1,
multiplier: 2,
on: RETRIABLE_ERRORS do |try|
# This 2nd level retriable only catches auth errors, and supports 1 retry, which allows
# auth to be re-attempted without having to retry all sorts of other failures like
# NotFound, etc
auth_tries = (try == 1 && authorization_refreshable? ? 2 : 1)
Retriable.retriable tries: auth_tries,
on: [Google::Apis::AuthorizationError, Signet::AuthorizationError, Signet::RemoteServerError, Signet::UnexpectedStatusError],
on_retry: proc { |*| refresh_authorization } do
execute_once(client).tap do |result|
if block_given?
yield result, nil
end
end
end
end
rescue => e # rubocop:disable Style/RescueStandardError
if block_given?
yield nil, e
else
raise e
end
end
ensure
opencensus_end_span
@http_res = nil
release!
end
end
end
end
end
# frozen_string_literal: true
# Extracted from https://github.com/googleapis/google-api-ruby-client/blob/main/google-apis-core/spec/google/apis/core/http_command_spec.rb
require 'spec_helper'
require 'google/apis/core/base_service'
RSpec.describe Google::Apis::Core::HttpCommand do # rubocop:disable RSpec/FilePath
context('with a successful response') do
let(:client) { Google::Apis::Core::BaseService.new('', '').client }
let(:command) { Google::Apis::Core::HttpCommand.new(:get, 'https://www.googleapis.com/zoo/animals') }
before do
stub_request(:get, 'https://www.googleapis.com/zoo/animals').to_return(body: %(Hello world))
end
it 'returns the response body if block not present' do
result = command.execute(client)
expect(result).to eql 'Hello world'
end
it 'calls block if present' do
expect { |b| command.execute(client, &b) }.to yield_with_args('Hello world', nil)
end
it 'retries with max elapsed_time and retries' do
expect(Retriable).to receive(:retriable).with(
tries: Google::Apis::RequestOptions.default.retries + 1,
max_elapsed_time: 3600,
base_interval: 1,
multiplier: 2,
on: described_class::RETRIABLE_ERRORS).and_call_original
allow(Retriable).to receive(:retriable).and_call_original
command.execute(client)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment