Merge branch 'bw-geo-add-postgresql-10-to-ci' into 'master'

add PostgreSQL 10 to Geo CI build

See merge request gitlab-org/gitlab-ee!5220

.gitlab-ci.yml

@@ -90,6 +90,11 @@ stages:
     - postgres:9.6
     - redis:alpine
 
+.use-pg-10-2: &use-pg-10-2
+  services:
+    - postgres:10.2
+    - redis:alpine
+
 .use-pg-with-elasticsearch: &use-pg-with-elasticsearch
   services:
     - postgres:9.2
@@ -219,7 +224,7 @@ stages:
   <<: *rspec-metadata-ee
   <<: *use-mysql-with-elasticsearch
 
-.rspec-geo-pg-9-6: &rspec-metadata-pg-geo
+.rspec-geo-pg-9-6: &rspec-metadata-pg-geo-9-6
   <<: *rspec-metadata
   <<: *use-pg-9-6
   stage: test
@@ -230,6 +235,17 @@ stages:
     - scripts/gitaly-test-spawn
     - bundle exec rspec --color --format documentation --tag geo ee/spec/
 
+.rspec-geo-pg-10-2: &rspec-metadata-pg-geo-10-2
+  <<: *rspec-metadata
+  <<: *use-pg-10-2
+  stage: test
+  script:
+    - export NO_KNAPSACK=1
+    - export CACHE_CLASSES=true
+    - source scripts/prepare_postgres_fdw.sh
+    - scripts/gitaly-test-spawn
+    - bundle exec rspec --color --format documentation --tag geo ee/spec/
+
 .rspec-ee-pg-rails5: &rspec-ee-pg-rails5
   <<: *rspec-ee-pg
   <<: *rails5
@@ -238,8 +254,12 @@ stages:
   <<: *rspec-ee-mysql
   <<: *rails5
 
-.rspec-geo-pg-9-6-rails5: &rspec-metadata-pg-geo-rails5
-  <<: *rspec-metadata-pg-geo
+.rspec-geo-pg-9-6-rails5: &rspec-metadata-pg-geo-9-6-rails5
+  <<: *rspec-metadata-pg-geo-9-6
+  <<: *rails5
+
+.rspec-geo-pg-10-2-rails5: &rspec-metadata-pg-geo-10-2-rails5
+  <<: *rspec-metadata-pg-geo-10-2
   <<: *rails5
 
 .spinach-metadata: &spinach-metadata
@@ -525,13 +545,15 @@ rspec-pg-ee 0 2: *rspec-ee-pg
 rspec-pg-ee 1 2: *rspec-ee-pg
 rspec-mysql-ee 0 2: *rspec-ee-mysql
 rspec-mysql-ee 1 2: *rspec-ee-mysql
-rspec-pg geo: *rspec-metadata-pg-geo
+rspec-pg geo 0 2: *rspec-metadata-pg-geo-9-6
+rspec-pg geo 1 2: *rspec-metadata-pg-geo-10-2
 
 rspec-pg-ee-rails5 0 2: *rspec-ee-pg-rails5
 rspec-pg-ee-rails5 1 2: *rspec-ee-pg-rails5
 rspec-mysql-ee-rails5 0 2: *rspec-ee-mysql-rails5
 rspec-mysql-ee-rails5 1 2: *rspec-ee-mysql-rails5
-rspec-pg-geo-rails5: *rspec-metadata-pg-geo-rails5
+rspec-pg-geo-rails5 0 2: *rspec-metadata-pg-geo-9-6-rails5
+rspec-pg-geo-rails5 1 2: *rspec-metadata-pg-geo-10-2-rails5
 ## EE jobs
 
 rspec-pg 0 28: *rspec-metadata-pg

ee/app/finders/geo/job_artifact_registry_finder.rb

@@ -90,7 +90,7 @@ module Geo
       if use_legacy_queries?
         legacy_find_failed_job_artifacts
       else
-        fdw_find_job_artifacts.merge(find_synced_job_artifacts_registries)
+        fdw_find_job_artifacts.merge(find_failed_job_artifacts_registries)
       end
     end
 

ee/spec/finders/geo/lfs_object_registry_finder_spec.rb

@@ -22,52 +22,36 @@ describe Geo::LfsObjectRegistryFinder, :geo do
     stub_lfs_object_storage
   end
 
-  describe '#count_synced_lfs_objects' do
-    it 'delegates to #legacy_find_synced_lfs_objects' do
-      expect(subject).to receive(:legacy_find_synced_lfs_objects).and_call_original
-
-      subject.count_synced_lfs_objects
-    end
-
-    it 'counts LFS objects that has been synced' do
-      create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
-
-      expect(subject.count_synced_lfs_objects).to eq 2
-    end
-
-    it 'ignores remote LFS objects' do
-      create(:geo_file_registry, :lfs, file_id: lfs_object_remote_1.id)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
-
-      expect(subject.count_synced_lfs_objects).to eq 2
+  context 'aggregate pushdown not supported' do
+    before do
+      allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false)
     end
 
-    context 'with selective sync' do
-      before do
-        allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
-
-        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
-        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
-        create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
-
-        secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-      end
-
+    describe '#count_synced_lfs_objects' do
       it 'delegates to #legacy_find_synced_lfs_objects' do
         expect(subject).to receive(:legacy_find_synced_lfs_objects).and_call_original
 
         subject.count_synced_lfs_objects
       end
+    end
+
+    describe '#count_failed_lfs_objects' do
+      it 'delegates to #legacy_find_failed_lfs_objects' do
+        expect(subject).to receive(:legacy_find_failed_lfs_objects).and_call_original
 
+        subject.count_failed_lfs_objects
+      end
+    end
+  end
+
+  shared_examples 'counts all the things' do
+    describe '#count_synced_lfs_objects' do
       it 'counts LFS objects that has been synced' do
         create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
         create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
         create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
 
-        expect(subject.count_synced_lfs_objects).to eq 1
+        expect(subject.count_synced_lfs_objects).to eq 2
       end
 
       it 'ignores remote LFS objects' do
@@ -75,57 +59,51 @@ describe Geo::LfsObjectRegistryFinder, :geo do
         create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
         create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
 
-        expect(subject.count_synced_lfs_objects).to eq 1
+        expect(subject.count_synced_lfs_objects).to eq 2
       end
-    end
-  end
 
-  describe '#count_failed_lfs_objects' do
-    it 'delegates to #legacy_find_failed_lfs_objects' do
-      expect(subject).to receive(:legacy_find_failed_lfs_objects).and_call_original
+      context 'with selective sync' do
+        before do
+          allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
 
-      subject.count_failed_lfs_objects
-    end
+          create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
+          create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
+          create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
 
-    it 'counts LFS objects that sync has failed' do
-      create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
+          secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
+        end
 
-      expect(subject.count_failed_lfs_objects).to eq 2
-    end
+        it 'delegates to #legacy_find_synced_lfs_objects' do
+          expect(subject).to receive(:legacy_find_synced_lfs_objects).and_call_original
 
-    it 'ignores remote LFS objects' do
-      create(:geo_file_registry, :lfs, file_id: lfs_object_remote_1.id, success: false)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_2.id, success: false)
-      create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
+          subject.count_synced_lfs_objects
+        end
 
-      expect(subject.count_failed_lfs_objects).to eq 2
-    end
+        it 'counts LFS objects that has been synced' do
+          create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
 
-    context 'with selective sync' do
-      before do
-        allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
+          expect(subject.count_synced_lfs_objects).to eq 1
+        end
 
-        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
-        create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
-        create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
+        it 'ignores remote LFS objects' do
+          create(:geo_file_registry, :lfs, file_id: lfs_object_remote_1.id)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_3.id)
 
-        secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-      end
-
-      it 'delegates to #legacy_find_failed_lfs_objects' do
-        expect(subject).to receive(:legacy_find_failed_lfs_objects).and_call_original
-
-        subject.count_failed_lfs_objects
+          expect(subject.count_synced_lfs_objects).to eq 1
+        end
       end
+    end
 
+    describe '#count_failed_lfs_objects' do
       it 'counts LFS objects that sync has failed' do
         create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
         create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
         create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
 
-        expect(subject.count_failed_lfs_objects).to eq 1
+        expect(subject.count_failed_lfs_objects).to eq 2
       end
 
       it 'ignores remote LFS objects' do
@@ -133,7 +111,41 @@ describe Geo::LfsObjectRegistryFinder, :geo do
         create(:geo_file_registry, :lfs, file_id: lfs_object_2.id, success: false)
         create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
 
-        expect(subject.count_failed_lfs_objects).to eq 1
+        expect(subject.count_failed_lfs_objects).to eq 2
+      end
+
+      context 'with selective sync' do
+        before do
+          allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
+
+          create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
+          create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
+          create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
+
+          secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
+        end
+
+        it 'delegates to #legacy_find_failed_lfs_objects' do
+          expect(subject).to receive(:legacy_find_failed_lfs_objects).and_call_original
+
+          subject.count_failed_lfs_objects
+        end
+
+        it 'counts LFS objects that sync has failed' do
+          create(:geo_file_registry, :lfs, file_id: lfs_object_1.id, success: false)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_2.id)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
+
+          expect(subject.count_failed_lfs_objects).to eq 1
+        end
+
+        it 'ignores remote LFS objects' do
+          create(:geo_file_registry, :lfs, file_id: lfs_object_remote_1.id, success: false)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_2.id, success: false)
+          create(:geo_file_registry, :lfs, file_id: lfs_object_3.id, success: false)
+
+          expect(subject.count_failed_lfs_objects).to eq 1
+        end
       end
     end
   end
@@ -214,6 +226,8 @@ describe Geo::LfsObjectRegistryFinder, :geo do
       skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo::Fdw.enabled?
     end
 
+    include_examples 'counts all the things'
+
     include_examples 'finds all the things' do
       let(:method_prefix) { 'fdw' }
     end
@@ -224,6 +238,8 @@ describe Geo::LfsObjectRegistryFinder, :geo do
       allow(Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false)
     end
 
+    include_examples 'counts all the things'
+
     include_examples 'finds all the things' do
       let(:method_prefix) { 'legacy' }
     end

ee/spec/models/geo_node_status_spec.rb

@@ -156,7 +156,9 @@ describe GeoNodeStatus, :geo do
     end
   end
 
-  describe '#lfs_objects_failed' do
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe '#lfs_objects_failed', :delete do
     it 'counts failed LFS objects' do
       # These four should be ignored
       create(:geo_file_registry, success: false)
@@ -170,7 +172,9 @@ describe GeoNodeStatus, :geo do
     end
   end
 
-  describe '#lfs_objects_synced_in_percentage' do
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe '#lfs_objects_synced_in_percentage', :delete do
     let(:lfs_object_project) { create(:lfs_objects_project, project: project_1) }
 
     before do
@@ -198,7 +202,9 @@ describe GeoNodeStatus, :geo do
     end
   end
 
-  describe '#job_artifacts_synced_count' do
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe '#job_artifacts_synced_count', :delete do
     it 'counts synced job artifacts' do
       # These should be ignored
       create(:geo_file_registry, success: false)
@@ -212,7 +218,9 @@ describe GeoNodeStatus, :geo do
     end
   end
 
-  describe '#job_artifacts_failed_count' do
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe '#job_artifacts_failed_count', :delete do
     it 'counts failed job artifacts' do
       # These should be ignored
       create(:geo_file_registry, success: false)
@@ -226,7 +234,9 @@ describe GeoNodeStatus, :geo do
   end
 
   describe '#job_artifacts_synced_in_percentage' do
-    context 'when artifacts are available' do
+    # Disable transactions via :delete method because a foreign table
+    # can't see changes inside a transaction of a different connection.
+    context 'when artifacts are available', :delete do
       before do
         [project_1, project_2, project_3, project_4].each_with_index do |project, index|
           build = create(:ci_build, project: project)