Commit c901936a authored by Yorick Peterse's avatar Yorick Peterse

Merge branch '23096-expire-artifacts-per-job' into 'master'

ExpireBuildArtifactsWorker query builds table without ordering enqueuing one job…

See merge request !6732
parents 2cd8b5d2 9afb2dac
...@@ -7,6 +7,7 @@ v 8.13.0 (unreleased) ...@@ -7,6 +7,7 @@ v 8.13.0 (unreleased)
- Use gitlab-shell v3.6.2 (GIT TRACE logging) - Use gitlab-shell v3.6.2 (GIT TRACE logging)
- Add `/projects/visible` API endpoint (Ben Boeckel) - Add `/projects/visible` API endpoint (Ben Boeckel)
- Fix centering of custom header logos (Ashley Dumaine) - Fix centering of custom header logos (Ashley Dumaine)
- ExpireBuildArtifactsWorker query builds table without ordering enqueuing one job per build to cleanup
- AbstractReferenceFilter caches project_refs on RequestStore when active - AbstractReferenceFilter caches project_refs on RequestStore when active
- Replaced the check sign to arrow in the show build view. !6501 - Replaced the check sign to arrow in the show build view. !6501
- Add a /wip slash command to toggle the Work In Progress status of a merge request. !6259 (tbalthazar) - Add a /wip slash command to toggle the Work In Progress status of a merge request. !6259 (tbalthazar)
......
...@@ -2,12 +2,11 @@ class ExpireBuildArtifactsWorker ...@@ -2,12 +2,11 @@ class ExpireBuildArtifactsWorker
include Sidekiq::Worker include Sidekiq::Worker
def perform def perform
Rails.logger.info 'Cleaning old build artifacts' Rails.logger.info 'Scheduling removal of build artifacts'
builds = Ci::Build.with_expired_artifacts build_ids = Ci::Build.with_expired_artifacts.pluck(:id)
builds.find_each(batch_size: 50).each do |build| build_ids = build_ids.map { |build_id| [build_id] }
Rails.logger.debug "Removing artifacts build #{build.id}..."
build.erase_artifacts! Sidekiq::Client.push_bulk('class' => ExpireBuildInstanceArtifactsWorker, 'args' => build_ids )
end
end end
end end
class ExpireBuildInstanceArtifactsWorker
include Sidekiq::Worker
def perform(build_id)
build = Ci::Build.with_expired_artifacts.reorder(nil).find_by(id: build_id)
return unless build
Rails.logger.info "Removing artifacts build #{build.id}..."
build.erase_artifacts!
end
end
...@@ -5,65 +5,42 @@ describe ExpireBuildArtifactsWorker do ...@@ -5,65 +5,42 @@ describe ExpireBuildArtifactsWorker do
let(:worker) { described_class.new } let(:worker) { described_class.new }
before { Sidekiq::Worker.clear_all }
describe '#perform' do describe '#perform' do
before { build } before { build }
subject! { worker.perform } subject! do
Sidekiq::Testing.fake! { worker.perform }
end
context 'with expired artifacts' do context 'with expired artifacts' do
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) } let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) }
it 'does expire' do it 'enqueues that build' do
expect(build.reload.artifacts_expired?).to be_truthy expect(jobs_enqueued.size).to eq(1)
end expect(jobs_enqueued[0]["args"]).to eq([build.id])
it 'does remove files' do
expect(build.reload.artifacts_file.exists?).to be_falsey
end
it 'does nullify artifacts_file column' do
expect(build.reload.artifacts_file_identifier).to be_nil
end end
end end
context 'with not yet expired artifacts' do context 'with not yet expired artifacts' do
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) } let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) }
it 'does not expire' do it 'does not enqueue that build' do
expect(build.reload.artifacts_expired?).to be_falsey expect(jobs_enqueued.size).to eq(0)
end
it 'does not remove files' do
expect(build.reload.artifacts_file.exists?).to be_truthy
end
it 'does not nullify artifacts_file column' do
expect(build.reload.artifacts_file_identifier).not_to be_nil
end end
end end
context 'without expire date' do context 'without expire date' do
let(:build) { create(:ci_build, :artifacts) } let(:build) { create(:ci_build, :artifacts) }
it 'does not expire' do it 'does not enqueue that build' do
expect(build.reload.artifacts_expired?).to be_falsey expect(jobs_enqueued.size).to eq(0)
end
it 'does not remove files' do
expect(build.reload.artifacts_file.exists?).to be_truthy
end
it 'does not nullify artifacts_file column' do
expect(build.reload.artifacts_file_identifier).not_to be_nil
end end
end end
context 'for expired artifacts' do def jobs_enqueued
let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) } Sidekiq::Queues.jobs_by_worker['ExpireBuildInstanceArtifactsWorker']
it 'is still expired' do
expect(build.reload.artifacts_expired?).to be_truthy
end
end end
end end
end end
require 'spec_helper'
describe ExpireBuildInstanceArtifactsWorker do
include RepoHelpers
let(:worker) { described_class.new }
describe '#perform' do
before { build }
subject! { worker.perform(build.id) }
context 'with expired artifacts' do
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) }
it 'does expire' do
expect(build.reload.artifacts_expired?).to be_truthy
end
it 'does remove files' do
expect(build.reload.artifacts_file.exists?).to be_falsey
end
it 'does nullify artifacts_file column' do
expect(build.reload.artifacts_file_identifier).to be_nil
end
end
context 'with not yet expired artifacts' do
let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) }
it 'does not expire' do
expect(build.reload.artifacts_expired?).to be_falsey
end
it 'does not remove files' do
expect(build.reload.artifacts_file.exists?).to be_truthy
end
it 'does not nullify artifacts_file column' do
expect(build.reload.artifacts_file_identifier).not_to be_nil
end
end
context 'without expire date' do
let(:build) { create(:ci_build, :artifacts) }
it 'does not expire' do
expect(build.reload.artifacts_expired?).to be_falsey
end
it 'does not remove files' do
expect(build.reload.artifacts_file.exists?).to be_truthy
end
it 'does not nullify artifacts_file column' do
expect(build.reload.artifacts_file_identifier).not_to be_nil
end
end
context 'for expired artifacts' do
let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) }
it 'is still expired' do
expect(build.reload.artifacts_expired?).to be_truthy
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment