Ensure progressive backoff of syncs-due-to-verification-failures

ee/app/models/concerns/geo/verifiable_registry.rb

@@ -39,6 +39,7 @@ module Geo
       extend ::Gitlab::Utils::Override
 
       sha_attribute :verification_checksum_mismatched
+
       scope :available_verifiables, -> { all }
 
       override :clear_verification_failure_fields!

ee/lib/gitlab/geo/verification_state.rb

@@ -22,6 +22,7 @@ module Gitlab
         verification_succeeded: 2,
         verification_failed: 3
       }.freeze
+
       VERIFICATION_TIMEOUT = 8.hours
 
       included do
@@ -54,9 +55,14 @@ module Gitlab
             instance.verification_started_at = Time.current
           end
 
-          before_transition any => :verification_pending do |instance, _|
-            instance.verification_retry_count = 0
-            instance.verification_retry_at = nil
+          before_transition [:verification_pending, :verification_started, :verification_succeeded] => :verification_pending do |instance, _|
+            instance.clear_verification_failure_fields!
+          end
+
+          before_transition verification_failed: :verification_pending do |instance, _|
+            # If transitioning from verification_failed, then don't clear
+            # verification_retry_count and verification_retry_at to ensure
+            # progressive backoff of syncs-due-to-verification-failures
             instance.verification_failure = nil
           end
 

ee/spec/lib/gitlab/geo/verification_state_spec.rb

@@ -23,6 +23,33 @@ RSpec.describe Gitlab::Geo::VerificationState do
 
   subject { DummyModel.new }
 
+  context 'state machine' do
+    context 'when failed' do
+      before do
+        subject.verification_started
+        subject.verification_failed_with_message!('foo')
+      end
+
+      context 'and transitioning to pending' do
+        it 'marks verification as pending' do
+          subject.verification_pending!
+
+          expect(subject.reload.verification_pending?).to be_truthy
+        end
+
+        it 'does not clear retry attributes' do
+          subject.verification_pending!
+
+          expect(subject.reload).to have_attributes(
+            verification_state: DummyModel.verification_state_value(:verification_pending),
+            verification_retry_count: 1,
+            verification_retry_at: be_present
+          )
+        end
+      end
+    end
+  end
+
   describe '.verification_pending_batch' do
     # Insert 2 records for a total of 3 with subject
     let!(:other_pending_records) do