Geo: adapt verification timed out query to use state table

EE: true
Changelog: fixed

ee/lib/gitlab/geo/verification_state.rb

@@ -35,7 +35,7 @@ module Gitlab
         scope :verification_failed, -> { available_verifiables.with_verification_state(:verification_failed) }
         scope :checksummed, -> { where.not(verification_checksum: nil) }
         scope :not_checksummed, -> { where(verification_checksum: nil) }
-        scope :verification_timed_out, -> { verification_started.where("verification_started_at < ?", VERIFICATION_TIMEOUT.ago) }
+        scope :verification_timed_out, -> { available_verifiables.where(verification_arel_table[:verification_state].eq(1)).where(verification_arel_table[:verification_started_at].lt(VERIFICATION_TIMEOUT.ago)) }
         scope :verification_retry_due, -> { where(verification_arel_table[:verification_retry_at].eq(nil).or(verification_arel_table[:verification_retry_at].lt(Time.current))) }
         scope :needs_verification, -> { available_verifiables.merge(with_verification_state(:verification_pending).or(with_verification_state(:verification_failed).verification_retry_due)) }
         scope :needs_reverification, -> { verification_succeeded.where("verified_at < ?", ::Gitlab::Geo.current_node.minimum_reverification_interval.days.ago) }
@@ -231,6 +231,14 @@ module Gitlab
           verification_state_table_class.arel_table
         end
 
+        # rubocop:disable CodeReuse/ActiveRecord
+        def verification_timed_out_batch_query
+          return verification_timed_out unless separate_verification_state_table?
+
+          verification_state_table_class.where(self.verification_state_model_key => verification_timed_out)
+        end
+        # rubocop:enable CodeReuse/ActiveRecord
+
         # Fail verification for records which started verification a long time ago
         def fail_verification_timeouts
           attrs = {
@@ -242,7 +250,7 @@ module Gitlab
             verified_at: Time.current
           }
 
-          verification_timed_out.each_batch do |relation|
+          verification_timed_out_batch_query.each_batch do |relation|
             relation.update_all(attrs)
           end
         end

ee/spec/lib/gitlab/geo/verification_state_spec.rb

@@ -8,417 +8,447 @@ RSpec.describe Gitlab::Geo::VerificationState do
   let_it_be(:primary_node) { create(:geo_node, :primary) }
   let_it_be(:secondary_node) { create(:geo_node) }
 
-  before(:all) do
-    create_dummy_model_table
-  end
-
-  after(:all) do
-    drop_dummy_model_table
-  end
-
-  before do
-    stub_dummy_replicator_class
-    stub_dummy_model_class
-  end
-
-  subject { DummyModel.new }
-
-  context 'state machine' do
-    context 'when failed' do
-      before do
-        subject.verification_started
-        subject.verification_failed_with_message!('foo')
-      end
-
-      context 'and transitioning to pending' do
-        it 'marks verification as pending' do
-          subject.verification_pending!
-
-          expect(subject.reload.verification_pending?).to be_truthy
-        end
-
-        it 'does not clear retry attributes' do
-          subject.verification_pending!
-
-          expect(subject.reload).to have_attributes(
-            verification_state: DummyModel.verification_state_value(:verification_pending),
-            verification_retry_count: 1,
-            verification_retry_at: be_present
-          )
-        end
-      end
+  context 'when verification state is stored in the model table' do
+    before(:all) do
+      create_dummy_model_table
     end
-  end
 
-  describe '.verification_pending_batch' do
-    # Insert 2 records for a total of 3 with subject
-    let!(:other_pending_records) do
-      DummyModel.insert_all([
-        { verification_state: pending_value, verified_at: 7.days.ago },
-        { verification_state: pending_value, verified_at: 6.days.ago }
-      ], returning: [:id])
+    after(:all) do
+      drop_dummy_model_table
     end
 
-    let(:pending_value) { DummyModel.verification_state_value(:verification_pending) }
-    let(:other_pending_ids) { other_pending_records.map { |result| result['id'] } }
-
     before do
-      subject.save!
-    end
-
-    it 'returns IDs of rows pending verification' do
-      expect(subject.class.verification_pending_batch(batch_size: 3)).to include(subject.id)
-    end
-
-    it 'marks verification as started' do
-      subject.class.verification_pending_batch(batch_size: 3)
-
-      expect(subject.reload.verification_started?).to be_truthy
-      expect(subject.verification_started_at).to be_present
+      stub_dummy_replicator_class
+      stub_dummy_model_class
     end
 
-    it 'limits with batch_size and orders records by verified_at with NULLs first' do
-      expected = [subject.id, other_pending_ids.first]
-
-      # `match_array` instead of `eq` because the UPDATE query does not
-      # guarantee that results are returned in the same order as the subquery
-      # used to SELECT the correct batch.
-      expect(subject.class.verification_pending_batch(batch_size: 2)).to match_array(expected)
-    end
+    subject { DummyModel.new }
 
-    context 'other verification states' do
-      it 'does not include them' do
-        subject.verification_started!
-
-        expect(subject.class.verification_pending_batch(batch_size: 3)).not_to include(subject.id)
+    context 'state machine' do
+      context 'when failed' do
+        before do
+          subject.verification_started
+          subject.verification_failed_with_message!('foo')
+        end
 
-        subject.verification_succeeded_with_checksum!('foo', Time.current)
+        context 'and transitioning to pending' do
+          it 'marks verification as pending' do
+            subject.verification_pending!
 
-        expect(subject.class.verification_pending_batch(batch_size: 3)).not_to include(subject.id)
+            expect(subject.reload.verification_pending?).to be_truthy
+          end
 
-        subject.verification_started
-        subject.verification_failed_with_message!('foo')
+          it 'does not clear retry attributes' do
+            subject.verification_pending!
 
-        expect(subject.class.verification_pending_batch(batch_size: 3)).not_to include(subject.id)
+            expect(subject.reload).to have_attributes(
+              verification_state: DummyModel.verification_state_value(:verification_pending),
+              verification_retry_count: 1,
+              verification_retry_at: be_present
+            )
+          end
+        end
       end
     end
-  end
 
-  describe '.verification_failed_batch' do
-    # Insert 2 records for a total of 3 with subject
-    let!(:other_failed_records) do
-      DummyModel.insert_all([
-        { verification_state: failed_value, verification_retry_at: 7.days.ago },
-        { verification_state: failed_value, verification_retry_at: 6.days.ago }
-      ], returning: [:id])
-    end
-
-    let(:failed_value) { DummyModel.verification_state_value(:verification_failed) }
-    let(:other_failed_ids) { other_failed_records.map { |result| result['id'] } }
+    describe '.verification_pending_batch' do
+      # Insert 2 records for a total of 3 with subject
+      let!(:other_pending_records) do
+        DummyModel.insert_all([
+          { verification_state: pending_value, verified_at: 7.days.ago },
+          { verification_state: pending_value, verified_at: 6.days.ago }
+        ], returning: [:id])
+      end
 
-    before do
-      subject.verification_started
-      subject.verification_failed_with_message!('foo')
-    end
+      let(:pending_value) { DummyModel.verification_state_value(:verification_pending) }
+      let(:other_pending_ids) { other_pending_records.map { |result| result['id'] } }
 
-    context 'with a failed record with retry due' do
       before do
-        subject.update!(verification_retry_at: 1.minute.ago)
+        subject.save!
       end
 
       it 'returns IDs of rows pending verification' do
-        expect(subject.class.verification_failed_batch(batch_size: 3)).to include(subject.id)
+        expect(subject.class.verification_pending_batch(batch_size: 3)).to include(subject.id)
       end
 
       it 'marks verification as started' do
-        subject.class.verification_failed_batch(batch_size: 3)
+        subject.class.verification_pending_batch(batch_size: 3)
 
         expect(subject.reload.verification_started?).to be_truthy
         expect(subject.verification_started_at).to be_present
       end
 
-      it 'limits with batch_size and orders records by verification_retry_at with NULLs first' do
-        expected = other_failed_ids
+      it 'limits with batch_size and orders records by verified_at with NULLs first' do
+        expected = [subject.id, other_pending_ids.first]
 
         # `match_array` instead of `eq` because the UPDATE query does not
         # guarantee that results are returned in the same order as the subquery
         # used to SELECT the correct batch.
-        expect(subject.class.verification_failed_batch(batch_size: 2)).to match_array(expected)
+        expect(subject.class.verification_pending_batch(batch_size: 2)).to match_array(expected)
       end
 
       context 'other verification states' do
         it 'does not include them' do
           subject.verification_started!
 
-          expect(subject.class.verification_failed_batch(batch_size: 5)).not_to include(subject.id)
+          expect(subject.class.verification_pending_batch(batch_size: 3)).not_to include(subject.id)
 
           subject.verification_succeeded_with_checksum!('foo', Time.current)
 
-          expect(subject.class.verification_failed_batch(batch_size: 5)).not_to include(subject.id)
+          expect(subject.class.verification_pending_batch(batch_size: 3)).not_to include(subject.id)
 
-          subject.verification_pending!
+          subject.verification_started
+          subject.verification_failed_with_message!('foo')
 
-          expect(subject.class.verification_failed_batch(batch_size: 5)).not_to include(subject.id)
+          expect(subject.class.verification_pending_batch(batch_size: 3)).not_to include(subject.id)
         end
       end
     end
 
-    context 'when verification_retry_at is in the future' do
-      it 'does not return the row' do
-        subject.update!(verification_retry_at: 1.minute.from_now)
+    describe '.verification_failed_batch' do
+      # Insert 2 records for a total of 3 with subject
+      let!(:other_failed_records) do
+        DummyModel.insert_all([
+          { verification_state: failed_value, verification_retry_at: 7.days.ago },
+          { verification_state: failed_value, verification_retry_at: 6.days.ago }
+        ], returning: [:id])
+      end
+
+      let(:failed_value) { DummyModel.verification_state_value(:verification_failed) }
+      let(:other_failed_ids) { other_failed_records.map { |result| result['id'] } }
 
-        expect(subject.class.verification_failed_batch(batch_size: 3)).not_to include(subject.id)
+      before do
+        subject.verification_started
+        subject.verification_failed_with_message!('foo')
       end
-    end
-  end
 
-  describe '.needs_verification' do
-    it 'includes verification_pending' do
-      subject.save!
+      context 'with a failed record with retry due' do
+        before do
+          subject.update!(verification_retry_at: 1.minute.ago)
+        end
 
-      expect(subject.class.needs_verification).to include(subject)
-    end
+        it 'returns IDs of rows pending verification' do
+          expect(subject.class.verification_failed_batch(batch_size: 3)).to include(subject.id)
+        end
 
-    it 'includes verification_failed and verification_retry_due' do
-      subject.verification_started
-      subject.verification_failed_with_message!('foo')
-      subject.update!(verification_retry_at: 1.minute.ago)
+        it 'marks verification as started' do
+          subject.class.verification_failed_batch(batch_size: 3)
 
-      expect(subject.class.needs_verification).to include(subject)
-    end
+          expect(subject.reload.verification_started?).to be_truthy
+          expect(subject.verification_started_at).to be_present
+        end
 
-    it 'excludes verification_failed with future verification_retry_at' do
-      subject.verification_started
-      subject.verification_failed_with_message!('foo')
-      subject.update!(verification_retry_at: 1.minute.from_now)
+        it 'limits with batch_size and orders records by verification_retry_at with NULLs first' do
+          expected = other_failed_ids
 
-      expect(subject.class.needs_verification).not_to include(subject)
-    end
-  end
+          # `match_array` instead of `eq` because the UPDATE query does not
+          # guarantee that results are returned in the same order as the subquery
+          # used to SELECT the correct batch.
+          expect(subject.class.verification_failed_batch(batch_size: 2)).to match_array(expected)
+        end
 
-  describe '.needs_reverification' do
-    before do
-      stub_current_geo_node(primary_node)
-    end
+        context 'other verification states' do
+          it 'does not include them' do
+            subject.verification_started!
 
-    let(:pending_value) { DummyModel.verification_state_value(:verification_pending) }
-    let(:failed_value) { DummyModel.verification_state_value(:verification_failed) }
-    let(:succeeded_value) { DummyModel.verification_state_value(:verification_succeeded) }
+            expect(subject.class.verification_failed_batch(batch_size: 5)).not_to include(subject.id)
 
-    it 'includes verification_succeeded with expired checksum' do
-      DummyModel.insert_all([
-        { verification_state: succeeded_value, verified_at: 15.days.ago }
-      ])
+            subject.verification_succeeded_with_checksum!('foo', Time.current)
 
-      expect(subject.class.needs_reverification.count).to eq 1
-    end
+            expect(subject.class.verification_failed_batch(batch_size: 5)).not_to include(subject.id)
 
-    it 'excludes non-success verification states and fresh checksums' do
-      DummyModel.insert_all([
-        { verification_state: pending_value, verified_at: 7.days.ago },
-        { verification_state: failed_value, verified_at: 6.days.ago },
-        { verification_state: succeeded_value, verified_at: 3.days.ago }
-      ])
+            subject.verification_pending!
 
-      expect(subject.class.needs_reverification.count).to eq 0
-    end
-  end
+            expect(subject.class.verification_failed_batch(batch_size: 5)).not_to include(subject.id)
+          end
+        end
+      end
+
+      context 'when verification_retry_at is in the future' do
+        it 'does not return the row' do
+          subject.update!(verification_retry_at: 1.minute.from_now)
 
-  describe '.reverify_batch' do
-    let!(:other_verified_records) do
-      DummyModel.insert_all([
-        { verification_state: succeeded_value, verified_at: 3.days.ago },
-        { verification_state: succeeded_value, verified_at: 4.days.ago }
-      ])
+          expect(subject.class.verification_failed_batch(batch_size: 3)).not_to include(subject.id)
+        end
+      end
     end
 
-    let(:succeeded_value) { DummyModel.verification_state_value(:verification_succeeded) }
+    describe '.needs_verification' do
+      it 'includes verification_pending' do
+        subject.save!
 
-    before do
-      stub_current_geo_node(primary_node)
+        expect(subject.class.needs_verification).to include(subject)
+      end
 
-      subject.verification_started
+      it 'includes verification_failed and verification_retry_due' do
+        subject.verification_started
+        subject.verification_failed_with_message!('foo')
+        subject.update!(verification_retry_at: 1.minute.ago)
 
-      subject.verification_succeeded_with_checksum!('foo', Time.current)
+        expect(subject.class.needs_verification).to include(subject)
+      end
 
-      subject.update!(verified_at: 15.days.ago)
-    end
+      it 'excludes verification_failed with future verification_retry_at' do
+        subject.verification_started
+        subject.verification_failed_with_message!('foo')
+        subject.update!(verification_retry_at: 1.minute.from_now)
 
-    it 'sets pending status to records with outdated verification' do
-      expect do
-        expect(subject.class.reverify_batch(batch_size: 100)).to eq 1
-      end.to change { subject.reload.verification_pending? }.to be_truthy
+        expect(subject.class.needs_verification).not_to include(subject)
+      end
     end
 
-    it 'limits the update with batch_size' do
-      DummyModel.update_all(verified_at: 15.days.ago)
+    describe '.needs_reverification' do
+      before do
+        stub_current_geo_node(primary_node)
+      end
 
-      expect(subject.class.reverify_batch(batch_size: 2)).to eq 2
-      expect(DummyModel.verification_pending.count).to eq 2
-    end
-  end
+      let(:pending_value) { DummyModel.verification_state_value(:verification_pending) }
+      let(:failed_value) { DummyModel.verification_state_value(:verification_failed) }
+      let(:succeeded_value) { DummyModel.verification_state_value(:verification_succeeded) }
 
-  describe '.fail_verification_timeouts' do
-    before do
-      subject.verification_started!
-    end
+      it 'includes verification_succeeded with expired checksum' do
+        DummyModel.insert_all([
+          { verification_state: succeeded_value, verified_at: 15.days.ago }
+        ])
 
-    context 'when verification has not timed out for a record' do
-      it 'does not update verification state' do
-        subject.update!(verification_started_at: (described_class::VERIFICATION_TIMEOUT - 1.minute).ago)
+        expect(subject.class.needs_reverification.count).to eq 1
+      end
 
-        DummyModel.fail_verification_timeouts
+      it 'excludes non-success verification states and fresh checksums' do
+        DummyModel.insert_all([
+          { verification_state: pending_value, verified_at: 7.days.ago },
+          { verification_state: failed_value, verified_at: 6.days.ago },
+          { verification_state: succeeded_value, verified_at: 3.days.ago }
+        ])
 
-        expect(subject.reload.verification_started?).to be_truthy
+        expect(subject.class.needs_reverification.count).to eq 0
       end
     end
 
-    context 'when verification has timed out for a record' do
-      it 'sets verification state to failed' do
-        subject.update!(verification_started_at: (described_class::VERIFICATION_TIMEOUT + 1.minute).ago)
+    describe '.reverify_batch' do
+      let!(:other_verified_records) do
+        DummyModel.insert_all([
+          { verification_state: succeeded_value, verified_at: 3.days.ago },
+          { verification_state: succeeded_value, verified_at: 4.days.ago }
+        ])
+      end
 
-        DummyModel.fail_verification_timeouts
+      let(:succeeded_value) { DummyModel.verification_state_value(:verification_succeeded) }
 
-        expect(subject.reload.verification_failed?).to be_truthy
+      before do
+        stub_current_geo_node(primary_node)
+
+        subject.verification_started
+
+        subject.verification_succeeded_with_checksum!('foo', Time.current)
+
+        subject.update!(verified_at: 15.days.ago)
       end
-    end
-  end
 
-  describe '#track_checksum_attempt!', :aggregate_failures do
-    context 'when verification was not yet started' do
-      it 'starts verification' do
+      it 'sets pending status to records with outdated verification' do
         expect do
-          subject.track_checksum_attempt! do
-            'a_checksum_value'
-          end
-        end.to change { subject.verification_started_at }.from(nil)
+          expect(subject.class.reverify_batch(batch_size: 100)).to eq 1
+        end.to change { subject.reload.verification_pending? }.to be_truthy
       end
 
-      it 'sets verification_succeeded' do
-        expect do
-          subject.track_checksum_attempt! do
-            'a_checksum_value'
-          end
-        end.to change { subject.verification_succeeded? }.from(false).to(true)
+      it 'limits the update with batch_size' do
+        DummyModel.update_all(verified_at: 15.days.ago)
+
+        expect(subject.class.reverify_batch(batch_size: 2)).to eq 2
+        expect(DummyModel.verification_pending.count).to eq 2
       end
     end
 
-    context 'when verification was started' do
-      it 'does not update verification_started_at' do
+    describe '.fail_verification_timeouts' do
+      before do
         subject.verification_started!
-        expected = subject.verification_started_at
+      end
+
+      context 'when verification has not timed out for a record' do
+        it 'does not update verification state' do
+          subject.update!(verification_started_at: (described_class::VERIFICATION_TIMEOUT - 1.minute).ago)
 
-        subject.track_checksum_attempt! do
-          'a_checksum_value'
+          DummyModel.fail_verification_timeouts
+
+          expect(subject.reload.verification_started?).to be_truthy
         end
+      end
+
+      context 'when verification has timed out for a record' do
+        it 'sets verification state to failed' do
+          subject.update!(verification_started_at: (described_class::VERIFICATION_TIMEOUT + 1.minute).ago)
 
-        expect(subject.verification_started_at).to be_within(1.second).of(expected)
+          DummyModel.fail_verification_timeouts
+
+          expect(subject.reload.verification_failed?).to be_truthy
+        end
       end
     end
 
-    it 'yields to the checksum calculation' do
-      expect do |probe|
-        subject.track_checksum_attempt!(&probe)
-      end.to yield_with_no_args
-    end
+    describe '#track_checksum_attempt!', :aggregate_failures do
+      context 'when verification was not yet started' do
+        it 'starts verification' do
+          expect do
+            subject.track_checksum_attempt! do
+              'a_checksum_value'
+            end
+          end.to change { subject.verification_started_at }.from(nil)
+        end
 
-    context 'when an error occurs while yielding' do
-      context 'when the record was failed' do
-        it 'sets verification_failed and increments verification_retry_count' do
-          subject.verification_failed_with_message!('foo')
+        it 'sets verification_succeeded' do
+          expect do
+            subject.track_checksum_attempt! do
+              'a_checksum_value'
+            end
+          end.to change { subject.verification_succeeded? }.from(false).to(true)
+        end
+      end
+
+      context 'when verification was started' do
+        it 'does not update verification_started_at' do
+          subject.verification_started!
+          expected = subject.verification_started_at
 
           subject.track_checksum_attempt! do
-            raise 'an error'
+            'a_checksum_value'
           end
 
-          expect(subject.reload.verification_failed?).to be_truthy
-          expect(subject.verification_retry_count).to eq(2)
+          expect(subject.verification_started_at).to be_within(1.second).of(expected)
         end
       end
-    end
 
-    context 'when the yielded block returns nil' do
-      context 'when the record was pending' do
-        it 'sets verification_failed and sets verification_retry_count to 1' do
-          subject.track_checksum_attempt! { nil }
+      it 'yields to the checksum calculation' do
+        expect do |probe|
+          subject.track_checksum_attempt!(&probe)
+        end.to yield_with_no_args
+      end
 
-          expect(subject.reload.verification_failed?).to be_truthy
-          expect(subject.verification_retry_count).to eq(1)
+      context 'when an error occurs while yielding' do
+        context 'when the record was failed' do
+          it 'sets verification_failed and increments verification_retry_count' do
+            subject.verification_failed_with_message!('foo')
+
+            subject.track_checksum_attempt! do
+              raise 'an error'
+            end
+
+            expect(subject.reload.verification_failed?).to be_truthy
+            expect(subject.verification_retry_count).to eq(2)
+          end
         end
       end
 
-      context 'when the record was failed' do
-        it 'sets verification_failed and increments verification_retry_count' do
-          subject.verification_failed_with_message!('foo')
+      context 'when the yielded block returns nil' do
+        context 'when the record was pending' do
+          it 'sets verification_failed and sets verification_retry_count to 1' do
+            subject.track_checksum_attempt! { nil }
 
-          subject.track_checksum_attempt! { nil }
+            expect(subject.reload.verification_failed?).to be_truthy
+            expect(subject.verification_retry_count).to eq(1)
+          end
+        end
 
-          expect(subject.reload.verification_failed?).to be_truthy
-          expect(subject.verification_retry_count).to eq(2)
+        context 'when the record was failed' do
+          it 'sets verification_failed and increments verification_retry_count' do
+            subject.verification_failed_with_message!('foo')
+
+            subject.track_checksum_attempt! { nil }
+
+            expect(subject.reload.verification_failed?).to be_truthy
+            expect(subject.verification_retry_count).to eq(2)
+          end
         end
       end
     end
-  end
 
-  describe '#verification_succeeded_with_checksum!' do
-    before do
-      subject.verification_started!
-    end
+    describe '#verification_succeeded_with_checksum!' do
+      before do
+        subject.verification_started!
+      end
 
-    context 'when the resource was updated during checksum calculation' do
-      let(:calculation_started_at) { subject.verification_started_at - 1.second }
+      context 'when the resource was updated during checksum calculation' do
+        let(:calculation_started_at) { subject.verification_started_at - 1.second }
 
-      it 'sets state to pending' do
-        subject.verification_succeeded_with_checksum!('abc123', calculation_started_at)
+        it 'sets state to pending' do
+          subject.verification_succeeded_with_checksum!('abc123', calculation_started_at)
 
-        expect(subject.reload.verification_pending?).to be_truthy
+          expect(subject.reload.verification_pending?).to be_truthy
+        end
       end
-    end
 
-    context 'when the resource was not updated during checksum calculation' do
-      let(:calculation_started_at) { subject.verification_started_at + 1.second }
+      context 'when the resource was not updated during checksum calculation' do
+        let(:calculation_started_at) { subject.verification_started_at + 1.second }
 
-      it 'saves the checksum' do
-        subject.verification_succeeded_with_checksum!('abc123', calculation_started_at)
+        it 'saves the checksum' do
+          subject.verification_succeeded_with_checksum!('abc123', calculation_started_at)
 
-        expect(subject.reload.verification_succeeded?).to be_truthy
-        expect(subject.reload.verification_checksum).to eq('abc123')
-        expect(subject.verified_at).not_to be_nil
+          expect(subject.reload.verification_succeeded?).to be_truthy
+          expect(subject.reload.verification_checksum).to eq('abc123')
+          expect(subject.verified_at).not_to be_nil
+        end
       end
-    end
 
-    context 'primary node' do
-      it 'calls replicator.handle_after_checksum_succeeded' do
-        stub_current_geo_node(primary_node)
+      context 'primary node' do
+        it 'calls replicator.handle_after_checksum_succeeded' do
+          stub_current_geo_node(primary_node)
 
-        expect(subject.replicator).to receive(:handle_after_checksum_succeeded)
+          expect(subject.replicator).to receive(:handle_after_checksum_succeeded)
 
-        subject.verification_succeeded_with_checksum!('abc123', Time.current)
+          subject.verification_succeeded_with_checksum!('abc123', Time.current)
+        end
+      end
+
+      context 'secondary node' do
+        it 'does not call replicator.handle_after_checksum_succeeded' do
+          stub_current_geo_node(secondary_node)
+
+          expect(subject.replicator).not_to receive(:handle_after_checksum_succeeded)
+
+          subject.verification_succeeded_with_checksum!('abc123', Time.current)
+        end
       end
     end
 
-    context 'secondary node' do
-      it 'does not call replicator.handle_after_checksum_succeeded' do
-        stub_current_geo_node(secondary_node)
+    describe '#verification_failed_with_message!' do
+      it 'saves the error message and increments retry counter' do
+        error = double('error', message: 'An error message')
 
-        expect(subject.replicator).not_to receive(:handle_after_checksum_succeeded)
+        subject.verification_started!
+        subject.verification_failed_with_message!('Failure to calculate checksum', error)
 
-        subject.verification_succeeded_with_checksum!('abc123', Time.current)
+        expect(subject.reload.verification_failed?).to be_truthy
+        expect(subject.reload.verification_failure).to eq 'Failure to calculate checksum: An error message'
+        expect(subject.verification_retry_count).to be 1
+        expect(subject.verification_checksum).to be_nil
       end
     end
   end
 
-  describe '#verification_failed_with_message!' do
-    it 'saves the error message and increments retry counter' do
-      error = double('error', message: 'An error message')
+  context 'when verification state is stored in a separate table' do
+    before(:all) do
+      create_dummy_model_with_separate_state_table
+    end
 
-      subject.verification_started!
-      subject.verification_failed_with_message!('Failure to calculate checksum', error)
+    after(:all) do
+      drop_dummy_model_with_separate_state_table
+    end
+
+    before do
+      stub_dummy_replicator_class(model_class: 'DummyModelWithSeparateState')
+      stub_dummy_model_with_separate_state_class
+    end
+
+    subject { TestDummyModelWithSeparateState.new }
 
-      expect(subject.reload.verification_failed?).to be_truthy
-      expect(subject.reload.verification_failure).to eq 'Failure to calculate checksum: An error message'
-      expect(subject.verification_retry_count).to be 1
-      expect(subject.verification_checksum).to be_nil
+    describe '.fail_verification_timeouts' do
+      it 'sets verification state to failed' do
+        state = subject.verification_state_object
+        state.update!(verification_started_at: (described_class::VERIFICATION_TIMEOUT + 1.minute).ago, verification_state: 1)
+
+        TestDummyModelWithSeparateState.fail_verification_timeouts
+
+        expect(subject.reload.verification_failed?).to be_truthy
+      end
     end
   end
 end

ee/spec/support/helpers/ee/geo_helpers.rb

@@ -48,7 +48,7 @@ module EE
       allow(::Gitlab::Geo).to receive(:geo_database_configured?).and_call_original
     end
 
-    def stub_dummy_replicator_class
+    def stub_dummy_replicator_class(model_class: 'DummyModel')
       stub_const('Geo::DummyReplicator', Class.new(::Gitlab::Geo::Replicator))
 
       ::Geo::DummyReplicator.class_eval do
@@ -56,7 +56,7 @@ module EE
         event :another_test
 
         def self.model
-          ::DummyModel
+          model_class.constantize
         end
 
         def handle_after_create_commit
@@ -120,5 +120,112 @@ module EE
         drop_table :dummy_models, force: true
       end
     end
+
+    # Example:
+    #
+    # before(:all) do
+    #   create_dummy_model_with_separate_state_table
+    # end
+
+    # after(:all) do
+    #   drop_dummy_model_with_separate_state_table
+    # end
+
+    # before do
+    #   stub_dummy_model_with_separate_state_class
+    # end
+
+    def create_dummy_model_with_separate_state_table
+      ActiveRecord::Schema.define do
+        create_table :_test_dummy_model_with_separate_states, force: true
+      end
+
+      ActiveRecord::Schema.define do
+        create_table :_test_dummy_model_states, id: false, force: true do |t|
+          t.bigint :_test_dummy_model_with_separate_state_id
+          t.binary :verification_checksum
+          t.integer :verification_state
+          t.datetime_with_timezone :verification_started_at
+          t.datetime_with_timezone :verified_at
+          t.datetime_with_timezone :verification_retry_at
+          t.integer :verification_retry_count
+          t.text :verification_failure
+        end
+      end
+    end
+
+    def drop_dummy_model_with_separate_state_table
+      ActiveRecord::Schema.define do
+        drop_table :_test_dummy_model_with_separate_states, force: true
+      end
+
+      ActiveRecord::Schema.define do
+        drop_table :_test_dummy_model_states, force: true
+      end
+    end
+
+    def stub_dummy_model_with_separate_state_class
+      stub_const('TestDummyModelWithSeparateState', Class.new(ApplicationRecord))
+
+      TestDummyModelWithSeparateState.class_eval do
+        self.table_name = '_test_dummy_model_with_separate_states'
+
+        include ::Gitlab::Geo::ReplicableModel
+        include ::Gitlab::Geo::VerificationState
+
+        with_replicator Geo::DummyReplicator
+
+        has_one :_test_dummy_model_state,
+          autosave: false,
+          inverse_of: :_test_dummy_model_with_separate_state,
+          foreign_key: :_test_dummy_model_with_separate_state_id
+
+        after_save :save_verification_details
+
+        delegate :verification_retry_at, :verification_retry_at=,
+                 :verified_at, :verified_at=,
+                 :verification_checksum, :verification_checksum=,
+                 :verification_failure, :verification_failure=,
+                 :verification_retry_count, :verification_retry_count=,
+                 :verification_state=, :verification_state,
+                 :verification_started_at=, :verification_started_at,
+          to: :_test_dummy_model_state, allow_nil: true
+
+        scope :available_verifiables, -> { joins(:_test_dummy_model_state) }
+
+        def verification_state_object
+          _test_dummy_model_state
+        end
+
+        def self.replicables_for_current_secondary(primary_key_in)
+          self.primary_key_in(primary_key_in)
+        end
+
+        def self.verification_state_table_class
+          TestDummyModelState
+        end
+
+        private
+
+        def _test_dummy_model_state
+          super || build__test_dummy_model_state
+        end
+      end
+
+      TestDummyModelWithSeparateState.reset_column_information
+
+      stub_const('TestDummyModelState', Class.new(ApplicationRecord))
+
+      TestDummyModelState.class_eval do
+        include EachBatch
+
+        self.table_name = '_test_dummy_model_states'
+        self.primary_key = '_test_dummy_model_with_separate_state_id'
+
+        belongs_to :_test_dummy_model_with_separate_state, inverse_of: :_test_dummy_model_state
+      end
+
+      TestDummyModelState.reset_column_information
+    end
   end
 end