Commit 4b771924 authored by Alex Kalderimis's avatar Alex Kalderimis

Merge branch '352245-increase-retry-lease-for-LogExecutionService' into 'master'

Conditionally raise when lease cannot be obtained in LogExecutionService

See merge request gitlab-org/gitlab!81770
parents 0d2264a0 f0347bc3
...@@ -42,10 +42,20 @@ module WebHooks ...@@ -42,10 +42,20 @@ module WebHooks
hook.failed! hook.failed!
end end
end end
rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError
raise if raise_lock_error?
end end
def lock_name def lock_name
"web_hooks:update_hook_failure_state:#{hook.id}" "web_hooks:update_hook_failure_state:#{hook.id}"
end end
# Allow an error to be raised after failing to obtain a lease only if the hook
# is not already in the correct failure state.
def raise_lock_error?
hook.reset # Reload so properties are guaranteed to be current.
hook.executable? != (response_category == :ok)
end
end end
end end
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
RSpec.describe WebHooks::LogExecutionService do RSpec.describe WebHooks::LogExecutionService do
include ExclusiveLeaseHelpers include ExclusiveLeaseHelpers
using RSpec::Parameterized::TableSyntax
describe '#execute' do describe '#execute' do
around do |example| around do |example|
...@@ -34,16 +35,47 @@ RSpec.describe WebHooks::LogExecutionService do ...@@ -34,16 +35,47 @@ RSpec.describe WebHooks::LogExecutionService do
expect(WebHookLog.recent.first).to have_attributes(data) expect(WebHookLog.recent.first).to have_attributes(data)
end end
it 'updates failure state using a lease that ensures fresh state is written' do context 'obtaining an exclusive lease' do
service = described_class.new(hook: project_hook, log_data: data, response_category: :error) let(:lease_key) { "web_hooks:update_hook_failure_state:#{project_hook.id}" }
WebHook.find(project_hook.id).update!(backoff_count: 1)
lease_key = "web_hooks:update_hook_failure_state:#{project_hook.id}" it 'updates failure state using a lease that ensures fresh state is written' do
lease = stub_exclusive_lease(lease_key, timeout: described_class::LOCK_TTL) service = described_class.new(hook: project_hook, log_data: data, response_category: :error)
WebHook.find(project_hook.id).update!(backoff_count: 1)
expect(lease).to receive(:try_obtain) lease = stub_exclusive_lease(lease_key, timeout: described_class::LOCK_TTL)
expect(lease).to receive(:cancel)
expect { service.execute }.to change { WebHook.find(project_hook.id).backoff_count }.to(2) expect(lease).to receive(:try_obtain)
expect(lease).to receive(:cancel)
expect { service.execute }.to change { WebHook.find(project_hook.id).backoff_count }.to(2)
end
context 'when a lease cannot be obtained' do
where(:response_category, :executable, :needs_updating) do
:ok | true | false
:ok | false | true
:failed | true | true
:failed | false | false
:error | true | true
:error | false | false
end
with_them do
subject(:service) { described_class.new(hook: project_hook, log_data: data, response_category: response_category) }
before do
stub_exclusive_lease_taken(lease_key, timeout: described_class::LOCK_TTL)
allow(project_hook).to receive(:executable?).and_return(executable)
end
it 'raises an error if the hook needs to be updated' do
if needs_updating
expect { service.execute }.to raise_error(Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError)
else
expect { service.execute }.not_to raise_error
end
end
end
end
end end
context 'when response_category is :ok' do context 'when response_category is :ok' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment