batch_worker.rb 3.04 KB
Newer Older
1 2
# frozen_string_literal: true

3
module RepositoryCheck
4
  class BatchWorker # rubocop:disable Scalability/IdempotentWorker
5
    include ApplicationWorker
6
    include RepositoryCheckQueue
7
    include ExclusiveLeaseGuard
8

9
    RUN_TIME = 3600
10
    BATCH_SIZE = 10_000
11
    LEASE_TIMEOUT = 1.hour
12

13 14
    attr_reader :shard_name

15 16
    loggable_arguments 0

17 18 19
    def perform(shard_name)
      @shard_name = shard_name

Toon Claes's avatar
Toon Claes committed
20
      return unless Gitlab::CurrentSettings.repository_checks_enabled
21
      return unless Gitlab::ShardHealthCache.healthy_shard?(shard_name)
22

23 24 25 26 27 28 29 30 31 32 33 34 35 36
      try_obtain_lease do
        perform_repository_checks
      end
    end

    def lease_timeout
      LEASE_TIMEOUT
    end

    def lease_key
      "repository_check_batch_worker:#{shard_name}"
    end

    def perform_repository_checks
Sean Arnold's avatar
Sean Arnold committed
37
      start = Time.current
38

39 40 41 42 43
      # This loop will break after a little more than one hour ('a little
      # more' because `git fsck` may take a few minutes), or if it runs out of
      # projects to check. By default sidekiq-cron will start a new
      # RepositoryCheckWorker each hour so that as long as there are repositories to
      # check, only one (or two) will be checked at a time.
44
      project_ids.each do |project_id|
Sean Arnold's avatar
Sean Arnold committed
45
        break if Time.current - start >= RUN_TIME
46

47
        next unless try_obtain_lease_for_project(project_id)
48

49 50 51
        SingleRepositoryWorker.new.perform(project_id)
      end
    end
52

53
    private
54

55 56 57 58 59
    # Project.find_each does not support WHERE clauses and
    # Project.find_in_batches does not support ordering. So we just build an
    # array of ID's. This is OK because we do it only once an hour, because
    # getting ID's from Postgres is not terribly slow, and because no user
    # has to sit and wait for this query to finish.
60 61
    def project_ids
      never_checked_project_ids(BATCH_SIZE) + old_checked_project_ids(BATCH_SIZE)
62 63
    end

64
    # rubocop: disable CodeReuse/ActiveRecord
65
    def never_checked_project_ids(batch_size)
66
      projects_on_shard.where(last_repository_check_at: nil)
67
        .where('created_at < ?', 24.hours.ago)
68 69
        .limit(batch_size).pluck(:id)
    end
70
    # rubocop: enable CodeReuse/ActiveRecord
71

72
    # rubocop: disable CodeReuse/ActiveRecord
73
    def old_checked_project_ids(batch_size)
74
      projects_on_shard.where.not(last_repository_check_at: nil)
75
        .where('last_repository_check_at < ?', 1.month.ago)
76 77
        .reorder(last_repository_check_at: :asc)
        .limit(batch_size).pluck(:id)
78
    end
79
    # rubocop: enable CodeReuse/ActiveRecord
80

81
    # rubocop: disable CodeReuse/ActiveRecord
82 83 84
    def projects_on_shard
      Project.where(repository_storage: shard_name)
    end
85
    # rubocop: enable CodeReuse/ActiveRecord
86

87
    def try_obtain_lease_for_project(id)
88 89 90 91 92 93 94 95 96
      # Use a 24-hour timeout because on servers/projects where 'git fsck' is
      # super slow we definitely do not want to run it twice in parallel.
      Gitlab::ExclusiveLease.new(
        "project_repository_check:#{id}",
        timeout: 24.hours
      ).try_obtain
    end
  end
end
97 98

RepositoryCheck::BatchWorker.prepend_if_ee('::EE::RepositoryCheck::BatchWorker')