Add a ReactiveCaching concern for use in the KubernetesService

app/models/concerns/reactive_caching.rb

+# The ReactiveCaching concern is used to fetch some data in the background and
+# store it in the Rails cache, keeping it up-to-date for as long as it is being
+# requested.  If the data hasn't been requested for +reactive_cache_lifetime+,
+# it stop being refreshed, and then be removed.
+#
+# Example of use:
+#
+#    class Foo < ActiveRecord::Base
+#      include ReactiveCaching
+#
+#      self.reactive_cache_key = ->(thing) { ["foo", thing.id] }
+#
+#      after_save :clear_reactive_cache!
+#
+#      def calculate_reactive_cache
+#        # Expensive operation here. The return value of this method is cached
+#      end
+#
+#      def result
+#        with_reactive_cache do |data|
+#          # ...
+#        end
+#      end
+#    end
+#
+# In this example, the first time `#result` is called, it will return `nil`.
+# However, it will enqueue a background worker to call `#calculate_reactive_cache`
+# and set an initial cache lifetime of ten minutes.
+#
+# Each time the background job completes, it stores the return value of
+# `#calculate_reactive_cache`. It is also re-enqueued to run again after
+# `reactive_cache_refresh_interval`, so keeping the stored value up to date.
+# Calculations are never run concurrently.
+#
+# Calling `#result` while a value is in the cache will call the block given to
+# `#with_reactive_cache`, yielding the cached value. It will also extend the
+# lifetime by `reactive_cache_lifetime`.
+#
+# Once the lifetime has expired, no more background jobs will be enqueued and
+# calling `#result` will again return `nil` - starting the process all over
+# again
+module ReactiveCaching
+  extend ActiveSupport::Concern
+
+  included do
+    class_attribute :reactive_cache_lease_timeout
+
+    class_attribute :reactive_cache_key
+    class_attribute :reactive_cache_lifetime
+    class_attribute :reactive_cache_refresh_interval
+
+    # defaults
+    self.reactive_cache_lease_timeout = 2.minutes
+
+    self.reactive_cache_refresh_interval = 1.minute
+    self.reactive_cache_lifetime = 10.minutes
+
+    def with_reactive_cache(&blk)
+      within_reactive_cache_lifetime do
+        data = Rails.cache.read(full_reactive_cache_key)
+        yield data if data.present?
+      end
+    ensure
+      Rails.cache.write(full_reactive_cache_key('alive'), true, expires_in: self.class.reactive_cache_lifetime)
+      ReactiveCachingWorker.perform_async(self.class, id)
+    end
+
+    def clear_reactive_cache!
+      Rails.cache.delete(full_reactive_cache_key)
+    end
+
+    def exclusively_update_reactive_cache!
+      locking_reactive_cache do
+        within_reactive_cache_lifetime do
+          enqueuing_update do
+            value = calculate_reactive_cache
+            Rails.cache.write(full_reactive_cache_key, value)
+          end
+        end
+      end
+    end
+
+    private
+
+    def full_reactive_cache_key(*qualifiers)
+      prefix = self.class.reactive_cache_key
+      prefix = prefix.call(self) if prefix.respond_to?(:call)
+
+      ([prefix].flatten + qualifiers).join(':')
+    end
+
+    def locking_reactive_cache
+      lease = Gitlab::ExclusiveLease.new(full_reactive_cache_key, timeout: reactive_cache_lease_timeout)
+      uuid = lease.try_obtain
+      yield if uuid
+    ensure
+      Gitlab::ExclusiveLease.cancel(full_reactive_cache_key, uuid)
+    end
+
+    def within_reactive_cache_lifetime
+      yield if Rails.cache.read(full_reactive_cache_key('alive'))
+    end
+
+    def enqueuing_update
+      yield
+    ensure
+      ReactiveCachingWorker.perform_in(self.class.reactive_cache_refresh_interval, self.class, id)
+    end
+  end
+end

app/workers/reactive_caching_worker.rb

+class ReactiveCachingWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  def perform(class_name, id)
+    klass = begin
+      Kernel.const_get(class_name)
+    rescue NameError
+      nil
+    end
+    return unless klass
+
+    klass.find_by(id: id).try(:exclusively_update_reactive_cache!)
+  end
+end

config/sidekiq_queues.yml

@@ -46,5 +46,6 @@
   - [repository_check, 1]
   - [system_hook, 1]
   - [git_garbage_collect, 1]
+  - [reactive_caching, 1]
   - [cronjob, 1]
   - [default, 1]

spec/models/concerns/reactive_caching_spec.rb

+require 'spec_helper'
+
+describe ReactiveCaching, caching: true do
+  include ReactiveCachingHelpers
+
+  class CacheTest
+    include ReactiveCaching
+
+    self.reactive_cache_key = ->(thing) { ["foo", thing.id] }
+
+    self.reactive_cache_lifetime = 5.minutes
+    self.reactive_cache_refresh_interval = 15.seconds
+
+    attr_reader :id
+
+    def initialize(id, &blk)
+      @id = id
+      @calculator = blk
+    end
+
+    def calculate_reactive_cache
+      @calculator.call
+    end
+
+    def result
+      with_reactive_cache do |data|
+        data / 2
+      end
+    end
+  end
+
+  let(:now) { Time.now.utc }
+
+  around(:each) do |example|
+    Timecop.freeze(now) { example.run }
+  end
+
+  let(:calculation) { -> { 2 + 2 } }
+  let(:cache_key) { "foo:666" }
+  let(:instance) { CacheTest.new(666, &calculation) }
+
+  describe '#with_reactive_cache' do
+    before { stub_reactive_cache }
+    subject(:go!) { instance.result }
+
+    context 'when cache is empty' do
+      it { is_expected.to be_nil }
+
+      it 'queues a background worker' do
+        expect(ReactiveCachingWorker).to receive(:perform_async).with(CacheTest, 666)
+
+        go!
+      end
+
+      it 'updates the cache lifespan' do
+        go!
+
+        expect(reactive_cache_alive?(instance)).to be_truthy
+      end
+    end
+
+    context 'when the cache is full' do
+      before { stub_reactive_cache(instance, 4) }
+
+      it { is_expected.to eq(2) }
+
+      context 'and expired' do
+        before { invalidate_reactive_cache(instance) }
+        it { is_expected.to be_nil }
+      end
+    end
+  end
+
+  describe '#clear_reactive_cache!' do
+    before do
+      stub_reactive_cache(instance, 4)
+      instance.clear_reactive_cache!
+    end
+
+    it { expect(instance.result).to be_nil }
+  end
+
+  describe '#exclusively_update_reactive_cache!' do
+    subject(:go!) { instance.exclusively_update_reactive_cache! }
+
+    context 'when the lease is free and lifetime is not exceeded' do
+      before { stub_reactive_cache(instance, "preexisting") }
+
+      it 'takes and releases the lease' do
+        expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return("000000")
+        expect(Gitlab::ExclusiveLease).to receive(:cancel).with(cache_key, "000000")
+
+        go!
+      end
+
+      it 'caches the result of #calculate_reactive_cache' do
+        go!
+
+        expect(read_reactive_cache(instance)).to eq(calculation.call)
+      end
+
+      it "enqueues a repeat worker" do
+        expect_reactive_cache_update_queued(instance)
+
+        go!
+      end
+
+      context 'and #calculate_reactive_cache raises an exception' do
+        before { stub_reactive_cache(instance, "preexisting") }
+        let(:calculation) { -> { raise "foo"} }
+
+        it 'leaves the cache untouched' do
+          expect { go! }.to raise_error("foo")
+          expect(read_reactive_cache(instance)).to eq("preexisting")
+        end
+
+        it 'enqueues a repeat worker' do
+          expect_reactive_cache_update_queued(instance)
+
+          expect { go! }.to raise_error("foo")
+        end
+      end
+    end
+
+    context 'when lifetime is exceeded' do
+      it 'skips the calculation' do
+        expect(instance).to receive(:calculate_reactive_cache).never
+
+        go!
+      end
+    end
+
+    context 'when the lease is already taken' do
+      before do
+        expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(nil)
+      end
+
+      it 'skips the calculation' do
+        expect(instance).to receive(:calculate_reactive_cache).never
+
+        go!
+      end
+    end
+  end
+end

spec/support/reactive_caching_helpers.rb

+module ReactiveCachingHelpers
+  def reactive_cache_key(subject, *qualifiers)
+    ([subject.class.reactive_cache_key.call(subject)].flatten + qualifiers).join(':')
+  end
+
+  def stub_reactive_cache(subject = nil, data = nil)
+    allow(ReactiveCachingWorker).to receive(:perform_async)
+    allow(ReactiveCachingWorker).to receive(:perform_in)
+    write_reactive_cache(subject, data) if data
+  end
+
+  def read_reactive_cache(subject)
+    Rails.cache.read(reactive_cache_key(subject))
+  end
+
+  def write_reactive_cache(subject, data)
+    start_reactive_cache_lifetime(subject)
+    Rails.cache.write(reactive_cache_key(subject), data)
+  end
+
+  def reactive_cache_alive?(subject)
+    Rails.cache.read(reactive_cache_key(subject, 'alive'))
+  end
+
+  def invalidate_reactive_cache(subject)
+    Rails.cache.delete(reactive_cache_key(subject, 'alive'))
+  end
+
+  def start_reactive_cache_lifetime(subject)
+    Rails.cache.write(reactive_cache_key(subject, 'alive'), true)
+  end
+
+  def expect_reactive_cache_update_queued(subject)
+    expect(ReactiveCachingWorker).
+      to receive(:perform_in).
+      with(subject.class.reactive_cache_refresh_interval, subject.class, subject.id)
+  end
+end

spec/workers/reactive_caching_worker_spec.rb

+require 'spec_helper'
+
+describe ReactiveCachingWorker do
+  let(:project) { create(:kubernetes_project) }
+  let(:service) { project.deployment_service }
+  subject { described_class.new.perform("KubernetesService", service.id) }
+
+  describe '#perform' do
+    it 'calls #exclusively_update_reactive_cache!' do
+      expect_any_instance_of(KubernetesService).to receive(:exclusively_update_reactive_cache!)
+
+      subject
+    end
+  end
+end