Skip to content

G
gitlab-ce
Commit 5f6b110d authored by Gabriel Mazetto's avatar Gabriel Mazetto
Browse files
Options

Simplify job_artifact_registry_finder specs 

Standardize example fixtures to be able to reason easily on each spec.
parent ac9043cf
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 272 deletions
ee/app/finders/geo/job_artifact_registry_finder.rb
View file @ 5f6b110d
...@@ -52,7 +52,7 @@ module Geo ...@@ -52,7 +52,7 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_artifact_ids: []) def find_migrated_local(batch_size:, except_artifact_ids: [])
job_artifacts current_node.job_artifacts
.inner_join_job_artifact_registry .inner_join_job_artifact_registry
.with_files_stored_remotely .with_files_stored_remotely
.id_not_in(except_artifact_ids) .id_not_in(except_artifact_ids)
......
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
View file @ 5f6b110d
...@@ -20,285 +20,180 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do ...@@ -20,285 +20,180 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do
stub_artifacts_object_storage stub_artifacts_object_storage
end end
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_5) { create(:ci_job_artifact, :expired, project: synced_project) }
let!(:job_artifact_6) { create(:ci_job_artifact, :expired, project: project_broken_storage) }
let!(:job_artifact_remote_1) { create(:ci_job_artifact, :remote_store, project: synced_project) }
let!(:job_artifact_remote_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) }
let!(:job_artifact_remote_3) { create(:ci_job_artifact, :expired, :remote_store, project: project_broken_storage) }
context 'counts all the things' do context 'counts all the things' do
describe '#count_syncable' do describe '#count_syncable' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) } it 'counts non-expired job artifacts' do
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, :remote_store, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
it 'counts job artifacts' do
expect(subject.count_syncable).to eq 6 expect(subject.count_syncable).to eq 6
end end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_syncable).to eq 5
end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'counts job artifacts' do it 'counts non-expired job artifacts' do
expect(subject.count_syncable).to eq 2 expect(subject.count_syncable).to eq 2
end end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_syncable).to eq 1
end
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'counts job artifacts' do it 'counts non-expired job artifacts' do
expect(subject.count_syncable).to eq 2 expect(subject.count_syncable).to eq 2
end end
end
it 'ignores expired job artifacts' do context 'with object storage sync disabled' do
job_artifact_5.update_column(:expire_at, Date.yesterday) let(:secondary) { create(:geo_node, :local_storage) }
expect(subject.count_syncable).to eq 1 it 'counts non-expired job artifacts' do
expect(subject.count_syncable).to eq 4
end end
end end
end end
describe '#count_synced' do describe '#count_synced' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) } before do
let!(:job_artifact_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) } create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, success: false)
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) } create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
end
context 'without selective sync' do context 'without selective sync' do
before do it 'counts job artifacts that have been synced ignoring expired job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) expect(subject.count_synced).to eq 3
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
end
it 'counts job artifacts that have been synced' do
expect(subject.count_synced).to eq 2
end
it 'ignores expired job artifacts' do
job_artifact_2.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced).to eq 1
end end
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
end
it 'counts job artifacts that has been synced' do
expect(subject.count_synced).to eq 2
end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
it 'counts job artifacts that has been synced ignoring expired job artifacts' do
expect(subject.count_synced).to eq 1 expect(subject.count_synced).to eq 1
end end
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id) it 'counts job artifacts that has been synced ignoring expired job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id) expect(subject.count_synced).to eq 1
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
it 'counts job artifacts that has been synced' do
expect(subject.count_synced).to eq 2
end end
end
it 'ignores expired job artifacts' do context 'with object storage sync disabled' do
job_artifact_5.update_column(:expire_at, Date.yesterday) let(:secondary) { create(:geo_node, :local_storage) }
expect(subject.count_synced).to eq 1 it 'counts job artifacts that has been synced ignoring expired job artifacts' do
expect(subject.count_synced).to eq 2
end end
end end
end end
describe '#count_failed' do describe '#count_failed' do
let!(:job_artifact_1) { create(:ci_job_artifact, :remote_store, project: synced_project) } before do
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) } create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) } create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id, success: false)
end
context 'without selective sync' do context 'without selective sync' do
before do it 'counts job artifacts that sync has failed ignoring expired ones' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
end
it 'counts job artifacts that sync has failed' do
expect(subject.count_failed).to eq 3 expect(subject.count_failed).to eq 3
end end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_failed).to eq 2
end
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) it 'counts job artifacts that sync has failed ignoring expired ones' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
end
it 'counts job artifacts that sync has failed' do
expect(subject.count_failed).to eq 2 expect(subject.count_failed).to eq 2
end end
end
it 'ignores expired job artifacts' do context 'with selective sync by shard' do
job_artifact_1.update_column(:expire_at, Date.yesterday) let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts job artifacts that sync has failed ignoring expired ones' do
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
end end
context 'with selective sync by shard' do context 'with object storage sync disabled' do
before do let(:secondary) { create(:geo_node, :local_storage) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, success: false)
end
it 'counts job artifacts that sync has failed' do it 'counts job artifacts that sync has failed ignoring expired ones' do
expect(subject.count_failed).to eq 2 expect(subject.count_failed).to eq 2
end end
it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
expect(subject.count_failed).to eq 1
end
end end
end end
describe '#count_synced_missing_on_primary' do describe '#count_synced_missing_on_primary' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) } before do
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false, missing_on_primary: false)
let!(:job_artifact_3) { create(:ci_job_artifact, :remote_store, project: synced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) } create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id)
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) } create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id, missing_on_primary: false)
end
context 'without selective sync' do context 'without selective sync' do
before do it 'counts job artifacts that have been synced and are missing on the primary, ignoring expired ones' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
it 'counts job artifacts that have been synced and are missing on the primary' do
expect(subject.count_synced_missing_on_primary).to eq 2 expect(subject.count_synced_missing_on_primary).to eq 2
end end
it 'ignores expired job artifacts' do
job_artifact_3.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced_missing_on_primary).to eq 1
end
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
it 'counts job artifacts that have been synced and are missing on the primary' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
it 'counts job artifacts that have been synced and are missing on the primary, ignoring expired ones' do
expect(subject.count_synced_missing_on_primary).to eq 1 expect(subject.count_synced_missing_on_primary).to eq 1
end end
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, missing_on_primary: true)
end
it 'counts job artifacts that have been synced and are missing on the primary' do it 'counts job artifacts that have been synced and are missing on the primary, ignoring expired ones' do
expect(subject.count_synced_missing_on_primary).to eq 2 expect(subject.count_synced_missing_on_primary).to eq 1
end end
end
it 'ignores expired job artifacts' do context 'with object storage sync disabled' do
job_artifact_5.update_column(:expire_at, Date.yesterday) let(:secondary) { create(:geo_node, :local_storage) }
it 'counts job artifacts that have been synced and are missing on the primary, ignoring expired ones' do
expect(subject.count_synced_missing_on_primary).to eq 1 expect(subject.count_synced_missing_on_primary).to eq 1
end end
end end
end end
describe '#count_registry' do describe '#count_registry' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
before do before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
end end
it 'counts file registries for job artifacts' do it 'counts file registries for job artifacts' do
...@@ -306,9 +201,7 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do ...@@ -306,9 +201,7 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'does not apply the selective sync restriction' do it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 4 expect(subject.count_registry).to eq 4
...@@ -316,9 +209,15 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do ...@@ -316,9 +209,15 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 4
end end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage) }
it 'does not apply the selective sync restriction' do it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 4 expect(subject.count_registry).to eq 4
...@@ -329,121 +228,75 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do ...@@ -329,121 +228,75 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do
context 'finds all the things' do context 'finds all the things' do
describe '#find_unsynced' do describe '#find_unsynced' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) } before do
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: true)
let!(:job_artifact_4) { create(:ci_job_artifact, :remote_store, project: unsynced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, success: true)
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) } end
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do context 'without selective sync' do
before do it 'returns job artifacts without an entry on the tracking database, ignoring expired ones' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
end
it 'returns job artifacts without an entry on the tracking database' do
job_artifacts = subject.find_unsynced(batch_size: 10, except_artifact_ids: [job_artifact_2.id]) job_artifacts = subject.find_unsynced(batch_size: 10, except_artifact_ids: [job_artifact_2.id])
expect(job_artifacts).to match_ids(job_artifact_4, job_artifact_5, job_artifact_6) expect(job_artifacts).to match_ids(job_artifact_remote_1, job_artifact_remote_2, job_artifact_4)
end
it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_4, job_artifact_6)
end end
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'returns job artifacts without an entry on the tracking database' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_3)
end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
it 'returns job artifacts without an entry on the tracking database, ignoring expired ones' do
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_3) expect(job_artifacts).to match_ids(job_artifact_remote_1)
end end
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'returns job artifacts without an entry on the tracking database' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: true)
it 'returns job artifacts without an entry on the tracking database, ignoring expired ones' do
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_6) expect(job_artifacts).to match_ids(job_artifact_4)
end end
end
it 'ignores expired job artifacts' do context 'with object storage sync disabled' do
job_artifact_5.update_column(:expire_at, Date.yesterday) let(:secondary) { create(:geo_node, :local_storage) }
it 'returns job artifacts without an entry on the tracking database, ignoring expired ones' do
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_6) expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_4)
end end
end end
end end
describe '#find_migrated_local' do describe '#find_migrated_local' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) } before do
let!(:job_artifact_remote_1) { create(:ci_job_artifact, :remote_store, project: synced_project) } create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
let!(:job_artifact_remote_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) }
let!(:job_artifact_remote_3) { create(:ci_job_artifact, :remote_store, project: project_broken_storage, expire_at: Date.yesterday) }
it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id])
expect(job_artifacts).to match_ids(job_artifact_remote_2)
end end
it 'excludes synced job artifacts that are stored locally' do it 'returns job artifacts excluding ones from the exception list' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id])
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_1, job_artifact_remote_2) expect(job_artifacts).to match_ids(job_artifact_remote_2, job_artifact_remote_3)
end end
it 'includes synced job artifacts that are expired' do it 'includes synced job artifacts that are expired, exclude stored locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_3) expect(job_artifacts).to match_ids(job_artifact_remote_1, job_artifact_remote_2, job_artifact_remote_3)
end end
context 'with selective sync by namespace' do context 'with selective sync by namespace' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'returns job artifacts remotely and successfully synced locally' do it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_1) expect(job_artifacts).to match_ids(job_artifact_remote_1)
...@@ -451,19 +304,24 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do ...@@ -451,19 +304,24 @@ describe Geo::JobArtifactRegistryFinder, :geo_fdw do
end end
context 'with selective sync by shard' do context 'with selective sync by shard' do
before do let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'returns job artifacts remotely and successfully synced locally' do it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_3) expect(job_artifacts).to match_ids(job_artifact_remote_3)
end end
end end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage) }
it 'returns job artifacts excluding ones from the exception list' do
job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id])
expect(job_artifacts).to match_ids(job_artifact_remote_2, job_artifact_remote_3)
end
end
end end
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7