Commit 140eda3a authored by Heinrich Lee Yu's avatar Heinrich Lee Yu

Merge branch 'mwaw/211433-metrics-dashboard-annotations-reaper-job-worker' into 'master'

211433 metrics dashboard annotations reaper job worker

Closes #211433

See merge request gitlab-org/gitlab!32838
parents c8c9c638 e53d500e
......@@ -163,6 +163,14 @@
:weight: 1
:idempotent:
:tags: []
- :name: cronjob:metrics_dashboard_schedule_annotations_prune
:feature_category: :metrics
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:namespaces_prune_aggregation_schedules
:feature_category: :source_code_management
:has_external_dependencies:
......@@ -1355,6 +1363,14 @@
:weight: 1
:idempotent: true
:tags: []
- :name: metrics_dashboard_prune_old_annotations
:feature_category: :metrics
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: migrate_external_diffs
:feature_category: :source_code_management
:has_external_dependencies:
......
# frozen_string_literal: true
module Metrics
module Dashboard
class PruneOldAnnotationsWorker
include ApplicationWorker
DELETE_LIMIT = 10_000
DEFAULT_CUT_OFF_PERIOD = 2.weeks
feature_category :metrics
idempotent! # in the scope of 24 hours
def perform
stale_annotations = ::Metrics::Dashboard::Annotation.ending_before(DEFAULT_CUT_OFF_PERIOD.ago.beginning_of_day)
stale_annotations.delete_with_limit(DELETE_LIMIT)
self.class.perform_async if stale_annotations.exists?
end
end
end
end
# frozen_string_literal: true
module Metrics
module Dashboard
class ScheduleAnnotationsPruneWorker
include ApplicationWorker
# rubocop:disable Scalability/CronWorkerContext
# This worker does not perform work scoped to a context
include CronjobQueue
# rubocop:enable Scalability/CronWorkerContext
feature_category :metrics
idempotent! # PruneOldAnnotationsWorker worker is idempotent in the scope of 24 hours
def perform
# Process is split into two jobs to avoid long running jobs, which are more prone to be disrupted
# mid work, which may cause some data not be delete, especially because cronjobs has retry option disabled
PruneOldAnnotationsWorker.perform_async
end
end
end
end
---
title: Remove metrics dashboard annotations attached to time periods older than two weeks.
merge_request: 32838
author:
type: added
......@@ -481,6 +481,9 @@ Settings.cron_jobs['issue_due_scheduler_worker']['job_class'] = 'IssueDueSchedul
Settings.cron_jobs['prune_web_hook_logs_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['prune_web_hook_logs_worker']['cron'] ||= '0 */1 * * *'
Settings.cron_jobs['prune_web_hook_logs_worker']['job_class'] = 'PruneWebHookLogsWorker'
Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker']['cron'] ||= '0 1 * * *'
Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker']['job_class'] = 'Metrics::Dashboard::ScheduleAnnotationsPruneWorker'
Settings.cron_jobs['schedule_migrate_external_diffs_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['schedule_migrate_external_diffs_worker']['cron'] ||= '15 * * * *'
Settings.cron_jobs['schedule_migrate_external_diffs_worker']['job_class'] = 'ScheduleMigrateExternalDiffsWorker'
......
......@@ -150,6 +150,8 @@
- 5
- - merge_request_mergeability_check
- 1
- - metrics_dashboard_prune_old_annotations
- 1
- - migrate_external_diffs
- 1
- - namespaceless_project_destroy
......
......@@ -827,6 +827,14 @@ You can create annotations by making requests to the
![Annotations UI](img/metrics_dashboard_annotations_ui_v13.0.png)
#### Retention policy
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/211433) in GitLab 13.01.
To avoid excessive storage space consumption by stale annotations, records attached
to time periods older than two weeks are removed daily. This recurring background
job runs at 1:00 a.m. local server time.
### Expand panel
> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0.
......
# frozen_string_literal: true
require 'spec_helper'
describe Metrics::Dashboard::PruneOldAnnotationsWorker do
let_it_be(:now) { DateTime.parse('2020-06-02T00:12:00Z') }
let_it_be(:two_weeks_old_annotation) { create(:metrics_dashboard_annotation, starting_at: now.advance(weeks: -2)) }
let_it_be(:one_day_old_annotation) { create(:metrics_dashboard_annotation, starting_at: now.advance(days: -1)) }
let_it_be(:month_old_annotation) { create(:metrics_dashboard_annotation, starting_at: now.advance(months: -1)) }
describe '#perform' do
it 'removes all annotations older than cut off', :aggregate_failures do
Timecop.freeze(now) do
described_class.new.perform
expect(Metrics::Dashboard::Annotation.all).to match_array([one_day_old_annotation, two_weeks_old_annotation])
# is idempotent in the scope of 24h
expect { described_class.new.perform }.not_to change { Metrics::Dashboard::Annotation.all.to_a }
Timecop.travel(24.hours.from_now) do
described_class.new.perform
expect(Metrics::Dashboard::Annotation.all).to match_array([one_day_old_annotation])
end
end
end
context 'batch to be deleted is bigger than upper limit' do
it 'schedules second job to clear remaining records' do
Timecop.freeze(now) do
create(:metrics_dashboard_annotation, starting_at: 1.month.ago)
stub_const("#{described_class}::DELETE_LIMIT", 1)
expect(described_class).to receive(:perform_async)
described_class.new.perform
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Metrics::Dashboard::ScheduleAnnotationsPruneWorker do
describe '#perform' do
it 'schedules annotations prune job with default cut off date' do
expect(Metrics::Dashboard::PruneOldAnnotationsWorker).to receive(:perform_async)
described_class.new.perform
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment