Add a health check to warn when reusing an existing tracking database

Geo cannot reuse an existing tracking database.

ee/lib/gitlab/geo/health_check.rb

@@ -8,6 +8,7 @@ module Gitlab
       def perform_checks
         return '' unless Gitlab::Geo.secondary?
         return 'Geo database configuration file is missing.' unless Gitlab::Geo.geo_database_configured?
+        return 'An existing tracking database cannot be reused.' if reusing_existing_tracking_database?
         return 'Geo node has a database that is writable which is an indication it is not configured for replication with the primary node.' unless Gitlab::Database.db_read_only?
         return 'Geo node does not appear to be replicating the database from the primary node.' if replication_enabled? && !replication_working?
         return "Geo database version (#{database_version}) does not match latest migration (#{migration_version}).\nYou may have to run `gitlab-rake geo:db:migrate` as root on the secondary." unless database_migration_version_match?
@@ -42,6 +43,13 @@ module Gitlab
         some_replication_active?
       end
 
+      def reusing_existing_tracking_database?
+        return false unless ::Geo::EventLogState.exists?
+        return false if Gitlab::Geo.current_node.created_at.nil?
+
+        Gitlab::Geo.current_node.created_at.utc > ::Geo::EventLogState.last.created_at.utc
+      end
+
       private
 
       def db_replication_lag_seconds_query

ee/lib/system_check/geo/geo_database_configured_check.rb

@@ -6,9 +6,12 @@ module SystemCheck
       set_name 'GitLab Geo secondary database is correctly configured'
       set_skip_reason 'not a secondary node'
 
+      DATABASE_DOCS = 'doc/gitlab-geo/database.md'.freeze
+      TROUBLESHOOTING_DOCS = 'doc/gitlab-geo/troubleshooting.md'.freeze
       WRONG_CONFIGURATION_MESSAGE = 'Check if you enabled the `geo_secondary_role` or `geo_postgresql` in the gitlab.rb config file.'.freeze
       UNHEALTHY_CONNECTION_MESSAGE = 'Check the tracking database configuration as the connection could not be established'.freeze
       NO_TABLES_MESSAGE = 'Run the tracking database migrations: gitlab-rake geo:db:migrate'.freeze
+      REUSING_EXISTING_DATABASE_MESSAGE = 'If you are reusing an existing tracking database, make sure you have reset it.'.freeze
 
       def skip?
         !Gitlab::Geo.secondary?
@@ -17,30 +20,32 @@ module SystemCheck
       def multi_check
         unless Gitlab::Geo.geo_database_configured?
           $stdout.puts 'no'.color(:red)
-
           try_fixing_it(WRONG_CONFIGURATION_MESSAGE)
-
-          for_more_information('doc/gitlab-geo/database.md')
+          for_more_information(DATABASE_DOCS)
 
           return false
         end
 
         unless connection_healthy?
           $stdout.puts 'no'.color(:red)
-
           try_fixing_it(UNHEALTHY_CONNECTION_MESSAGE)
-
-          for_more_information('doc/gitlab-geo/database.md')
+          for_more_information(DATABASE_DOCS)
 
           return false
         end
 
         unless tables_present?
           $stdout.puts 'no'.color(:red)
-
           try_fixing_it(NO_TABLES_MESSAGE)
+          for_more_information(DATABASE_DOCS)
 
-          for_more_information('doc/gitlab-geo/database.md')
+          return false
+        end
+
+        unless fresh_database?
+          $stdout.puts 'no'.color(:red)
+          try_fixing_it(REUSING_EXISTING_DATABASE_MESSAGE)
+          for_more_information(TROUBLESHOOTING_DOCS)
 
           return false
         end
@@ -58,6 +63,14 @@ module SystemCheck
       def tables_present?
         Gitlab::Geo::DatabaseTasks.with_geo_db { !ActiveRecord::Base.connection.migration_context.needs_migration? }
       end
+
+      def geo_health_check
+        @geo_health_check ||= Gitlab::Geo::HealthCheck.new
+      end
+
+      def fresh_database?
+        !geo_health_check.reusing_existing_tracking_database?
+      end
     end
   end
 end

ee/spec/lib/gitlab/geo/health_check_spec.rb

@@ -46,6 +46,14 @@ describe Gitlab::Geo::HealthCheck, :geo do
           end
         end
 
+        context 'when reusing an existing tracking database' do
+          it 'returns an error when event_log_state is older than current node created_at' do
+            create(:geo_event_log_state, created_at: 3.months.ago)
+
+            expect(subject.perform_checks).to include('An existing tracking database cannot be reused.')
+          end
+        end
+
         context 'when the database is writable' do
           let(:db_read_only) { false }
 

ee/spec/lib/system_check/geo/geo_database_configured_check_spec.rb

@@ -35,10 +35,22 @@ describe SystemCheck::Geo::GeoDatabaseConfiguredCheck do
       expect(subject.multi_check).to be_falsey
     end
 
+    it "checks if existing database is being reused" do
+      stub_configuration_check(true)
+      stub_connection_state(true)
+      stub_tables_existence(true)
+      stub_fresh_database(false)
+
+      expect(subject).to receive(:try_fixing_it).with(described_class::REUSING_EXISTING_DATABASE_MESSAGE)
+
+      expect(subject.multi_check).to be_falsey
+    end
+
     it "returns true when all checks passed" do
       stub_configuration_check(true)
       stub_connection_state(true)
       stub_tables_existence(true)
+      stub_fresh_database(true)
 
       expect(subject).not_to receive(:try_fixing_it)
 
@@ -59,4 +71,10 @@ describe SystemCheck::Geo::GeoDatabaseConfiguredCheck do
   def stub_tables_existence(state)
     expect_any_instance_of(ActiveRecord::MigrationContext).to receive(:needs_migration?).and_return(!state)
   end
+
+  def stub_fresh_database(state)
+    expect_next_instance_of(Gitlab::Geo::HealthCheck) do |geo_health_check|
+      expect(geo_health_check).to receive(:reusing_existing_tracking_database?).and_return(!state)
+    end
+  end
 end