Merge branch '3360-geo-skip-unhealthy-shards' into 'master'

Only run Geo backfill for projects on healthy shards

Closes #3360

See merge request gitlab-org/gitlab-ee!2974

app/workers/geo/repository_sync_worker.rb

@@ -22,7 +22,7 @@ module Geo
     end
 
     def find_project_ids_not_synced
-      current_node.unsynced_projects
+      healthy_shards_restriction(current_node.unsynced_projects)
         .reorder(last_repository_updated_at: :desc)
         .limit(db_retrieve_batch_size)
         .pluck(:id)
@@ -35,5 +35,25 @@ module Geo
                   .limit(db_retrieve_batch_size)
                   .pluck(:project_id)
     end
+
+    def healthy_shards_restriction(relation)
+      configured = Gitlab.config.repositories.storages.keys
+      referenced = Project.distinct(:repository_storage).pluck(:repository_storage)
+      healthy = healthy_shards
+
+      known = configured | referenced
+      return relation if (known - healthy).empty?
+
+      relation.where(repository_storage: healthy)
+    end
+
+    def healthy_shards
+      Gitlab::HealthChecks::FsShardsCheck
+        .readiness
+        .select(&:success)
+        .map { |check| check.labels[:shard] }
+        .compact
+        .uniq
+    end
   end
 end

spec/workers/geo/repository_sync_worker_spec.rb

@@ -116,5 +116,32 @@ describe Geo::RepositorySyncWorker, :postgresql do
         end
       end
     end
+
+    context 'unhealthy shards' do
+      it 'skips backfill for repositories on unhealthy shards' do
+        unhealthy = create(:project, group: synced_group, repository_storage: 'broken')
+
+        # Make the shard unhealthy
+        FileUtils.rm_rf(unhealthy.repository_storage_path)
+
+        expect(Geo::ProjectSyncWorker).to receive(:perform_async).with(project_in_synced_group.id, anything)
+        expect(Geo::ProjectSyncWorker).not_to receive(:perform_async).with(unhealthy.id, anything)
+
+        Sidekiq::Testing.inline! { subject.perform }
+      end
+
+      it 'skips backfill for projects on missing shards' do
+        missing = create(:project, group: synced_group)
+        missing.update_column(:repository_storage, 'unknown')
+
+        # hide the 'broken' storage for this spec
+        stub_storage_settings({})
+
+        expect(Geo::ProjectSyncWorker).to receive(:perform_async).with(project_in_synced_group.id, anything)
+        expect(Geo::ProjectSyncWorker).not_to receive(:perform_async).with(missing.id, anything)
+
+        Sidekiq::Testing.inline! { subject.perform }
+      end
+    end
   end
 end