Skip to content

G
gitlab-ce
Commit ffb9b3ef authored by Yorick Peterse's avatar Yorick Peterse
Browse files
Options

Refactor cache refreshing/expiring 

This refactors repository caching so it's possible to selectively
refresh certain caches, instead of just expiring and refreshing
everything.

To allow this the various methods that were cached (e.g. "tag_count" and
"readme") use a similar pattern that makes expiring and refreshing
their data much easier.

In this new setup caches are refreshed as follows:

1. After a commit (but before running ProjectCacheWorker) we expire some
   basic caches such as the commit count and repository size.

2. ProjectCacheWorker will recalculate the commit count, repository
   size, then refresh a specific set of caches based on the list of
   files changed in a push payload.

This requires a bunch of changes to the various methods that may be
cached. For one, data should not be cached if a branch used or the
entire repository does not exist. To prevent all these methods from
handling this manually this is taken care of in
Repository#cache_method_output. Some methods still manually check for
the existence of a repository but this result is also cached.

With selective flushing implemented ProjectCacheWorker no longer uses an
exclusive lease for all of its work. Instead this worker only uses a
lease to limit the number of times the repository size is updated as
this is a fairly expensive operation.
parent 6f393877
Hide whitespace changes
Inline Side-by-side
Showing with 605 additions and 556 deletions
app/models/project.rb
View file @ ffb9b3ef
......@@ -1086,7 +1086,7 @@ class Project < ActiveRecord::Base
"refs/heads/#{branch}",
force: true)
repository.copy_gitattributes(branch)
repository.expire_avatar_cache(branch)
repository.expire_avatar_cache
reload_default_branch
end
......
app/models/repository.rb
View file @ ffb9b3ef
require 'securerandom'
class Repository
include Gitlab::ShellAdapter
attr_accessor :path_with_namespace, :project
class CommitError < StandardError; end
# Files to use as a project avatar in case no avatar was uploaded via the web
# UI.
AVATAR_FILES = %w{logo.png logo.jpg logo.gif}
# Methods that cache data from the Git repository.
#
# Each entry in this Array should have a corresponding method with the exact
# same name. The cache key used by those methods must also match method's
# name.
#
# For example, for entry `:readme` there's a method called `readme` which
# stores its data in the `readme` cache key.
CACHED_METHODS = %i(size commit_count readme version contribution_guide
changelog license_blob license_key gitignore koding_yml
gitlab_ci_yml branch_names tag_names branch_count
tag_count avatar exists? empty? root_ref)
# Certain method caches should be refreshed when certain types of files are
# changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
# the corresponding methods to call for refreshing caches.
METHOD_CACHES_FOR_FILE_TYPES = {
readme: :readme,
changelog: :changelog,
license: %i(license_blob license_key),
contributing: :contribution_guide,
version: :version,
gitignore: :gitignore,
koding: :koding_yml,
gitlab_ci: :gitlab_ci_yml,
avatar: :avatar
}
# Wraps around the given method and caches its output in Redis and an instance
# variable.
#
# This only works for methods that do not take any arguments.
def self.cache_method(name, fallback: nil)
original = :"_uncached_#{name}"
include Gitlab::ShellAdapter
alias_method(original, name)
attr_accessor :path_with_namespace, :project
define_method(name) do
cache_method_output(name, fallback: fallback) { __send__(original) }
end
end
def self.storages
Gitlab.config.repositories.storages
......@@ -37,20 +75,6 @@ class Repository
)
end
def exists?
return @exists unless @exists.nil?
@exists = cache.fetch(:exists?) do
refs_directory_exists?
end
end
def empty?
return @empty unless @empty.nil?
@empty = cache.fetch(:empty?) { raw_repository.empty? }
end
#
# Git repository can contains some hidden refs like:
# /refs/notes/*
......@@ -217,10 +241,6 @@ class Repository
branch_names + tag_names
end
def branch_names
@branch_names ||= cache.fetch(:branch_names) { branches.map(&:name) }
end
def branch_exists?(branch_name)
branch_names.include?(branch_name)
end
......@@ -270,34 +290,6 @@ class Repository
ref_exists?(keep_around_ref_name(sha))
end
def tag_names
cache.fetch(:tag_names) { raw_repository.tag_names }
end
def commit_count
cache.fetch(:commit_count) do
begin
raw_repository.commit_count(self.root_ref)
rescue
0
end
end
end
def branch_count
@branch_count ||= cache.fetch(:branch_count) { branches.size }
end
def tag_count
@tag_count ||= cache.fetch(:tag_count) { raw_repository.rugged.tags.count }
end
# Return repo size in megabytes
# Cached in redis
def size
cache.fetch(:size) { raw_repository.size }
end
def diverging_commit_counts(branch)
root_ref_hash = raw_repository.rev_parse_target(root_ref).oid
cache.fetch(:"diverging_commit_counts_#{branch.name}") do
......@@ -313,48 +305,55 @@ class Repository
end
end
# Keys for data that can be affected for any commit push.
def cache_keys
%i(size commit_count
readme version contribution_guide changelog
license_blob license_key gitignore koding_yml gitlab_ci_yml)
def expire_tags_cache
expire_method_caches(%i(tag_names tag_count))
@tags = nil
end
# Keys for data on branch/tag operations.
def cache_keys_for_branches_and_tags
%i(branch_names tag_names branch_count tag_count)
def expire_branches_cache
expire_method_caches(%i(branch_names branch_count))
@local_branches = nil
end
def build_cache
(cache_keys + cache_keys_for_branches_and_tags).each do |key|
unless cache.exist?(key)
send(key)
end
end
def expire_statistics_caches
expire_method_caches(%i(size commit_count))
end
def expire_tags_cache
cache.expire(:tag_names)
@tags = nil
def expire_all_method_caches
expire_method_caches(CACHED_METHODS)
end
def expire_branches_cache
cache.expire(:branch_names)
@branch_names = nil
@local_branches = nil
# Expires the caches of a specific set of methods
def expire_method_caches(methods)
methods.each do |key|
cache.expire(key)
ivar = cache_instance_variable_name(key)
remove_instance_variable(ivar) if instance_variable_defined?(ivar)
end
end
def expire_cache(branch_name = nil, revision = nil)
cache_keys.each do |key|
cache.expire(key)
def expire_avatar_cache
expire_method_caches(%i(avatar))
end
# Refreshes the method caches of this repository.
#
# types - An Array of file types (e.g. `:readme`) used to refresh extra
# caches.
def refresh_method_caches(types)
to_refresh = []
types.each do |type|
methods = METHOD_CACHES_FOR_FILE_TYPES[type.to_sym]
to_refresh.concat(Array(methods)) if methods
end
expire_branch_cache(branch_name)
expire_avatar_cache(branch_name, revision)
expire_method_caches(to_refresh)
# This ensures this particular cache is flushed after the first commit to a
# new repository.
expire_emptiness_caches if empty?
to_refresh.each { |method| send(method) }
end
def expire_branch_cache(branch_name = nil)
......@@ -373,15 +372,14 @@ class Repository
end
def expire_root_ref_cache
cache.expire(:root_ref)
@root_ref = nil
expire_method_caches(%i(root_ref))
end
# Expires the cache(s) used to determine if a repository is empty or not.
def expire_emptiness_caches
cache.expire(:empty?)
@empty = nil
return unless empty?
expire_method_caches(%i(empty?))
expire_has_visible_content_cache
end
......@@ -390,51 +388,22 @@ class Repository
@has_visible_content = nil
end
def expire_branch_count_cache
cache.expire(:branch_count)
@branch_count = nil
end
def expire_tag_count_cache
cache.expire(:tag_count)
@tag_count = nil
end
def lookup_cache
@lookup_cache ||= {}
end
def expire_avatar_cache(branch_name = nil, revision = nil)
# Avatars are pulled from the default branch, thus if somebody pushes to a
# different branch there's no need to expire anything.
return if branch_name && branch_name != root_ref
# We don't want to flush the cache if the commit didn't actually make any
# changes to any of the possible avatar files.
if revision && commit = self.commit(revision)
return unless commit.raw_diffs(deltas_only: true).
any? { |diff| AVATAR_FILES.include?(diff.new_path) }
end
cache.expire(:avatar)
@avatar = nil
end
def expire_exists_cache
cache.expire(:exists?)
@exists = nil
expire_method_caches(%i(exists?))
end
# expire cache that doesn't depend on repository data (when expiring)
def expire_content_cache
expire_tags_cache
expire_tag_count_cache
expire_branches_cache
expire_branch_count_cache
expire_root_ref_cache
expire_emptiness_caches
expire_exists_cache
expire_statistics_caches
end
# Runs code after a repository has been created.
......@@ -449,9 +418,8 @@ class Repository
# Runs code just before a repository is deleted.
def before_delete
expire_exists_cache
expire_cache if exists?
expire_all_method_caches
expire_branch_cache if exists?
expire_content_cache
repository_event(:remove_repository)
......@@ -468,9 +436,9 @@ class Repository
# Runs code before pushing (= creating or removing) a tag.
def before_push_tag
expire_cache
expire_statistics_caches
expire_emptiness_caches
expire_tags_cache
expire_tag_count_cache
repository_event(:push_tag)
end
......@@ -478,7 +446,7 @@ class Repository
# Runs code before removing a tag.
def before_remove_tag
expire_tags_cache
expire_tag_count_cache
expire_statistics_caches
repository_event(:remove_tag)
end
......@@ -490,12 +458,14 @@ class Repository
# Runs code after a repository has been forked/imported.
def after_import
expire_content_cache
build_cache
expire_tags_cache
expire_branches_cache
end
# Runs code after a new commit has been pushed.
def after_push_commit(branch_name, revision)
expire_cache(branch_name, revision)
def after_push_commit(branch_name)
expire_statistics_caches
expire_branch_cache(branch_name)
repository_event(:push_commit, branch: branch_name)
end
......@@ -504,7 +474,6 @@ class Repository
def after_create_branch
expire_branches_cache
expire_has_visible_content_cache
expire_branch_count_cache
repository_event(:push_branch)
end
......@@ -519,7 +488,6 @@ class Repository
# Runs code after an existing branch has been removed.
def after_remove_branch
expire_has_visible_content_cache
expire_branch_count_cache
expire_branches_cache
end
......@@ -546,82 +514,127 @@ class Repository
Gitlab::Git::Blob.raw(self, oid)
end
def root_ref
if raw_repository
raw_repository.root_ref
else
# When the repo does not exist we raise this error so no data is cached.
raise Rugged::ReferenceError
end
end
cache_method :root_ref
def exists?
refs_directory_exists?
end
cache_method :exists?
def empty?
raw_repository.empty?
end
cache_method :empty?
# The size of this repository in megabytes.
def size
exists? ? raw_repository.size : 0.0
end
cache_method :size, fallback: 0.0
def commit_count
root_ref ? raw_repository.commit_count(root_ref) : 0
end
cache_method :commit_count, fallback: 0
def branch_names
branches.map(&:name)
end
cache_method :branch_names, fallback: []
def tag_names
raw_repository.tag_names
end
cache_method :tag_names, fallback: []
def branch_count
branches.size
end
cache_method :branch_count, fallback: 0
def tag_count
raw_repository.rugged.tags.count
end
cache_method :tag_count, fallback: 0
def avatar
if tree = file_on_head(:avatar)
tree.path
end
end
cache_method :avatar
def readme
cache.fetch(:readme) { tree(:head).readme }
if head = tree(:head)
head.readme
end
end
cache_method :readme
def version
cache.fetch(:version) do
file_on_head(:version)
end
file_on_head(:version)
end
cache_method :version
def contribution_guide
cache.fetch(:contribution_guide) do
file_on_head(:contributing)
end
file_on_head(:contributing)
end
cache_method :contribution_guide
def changelog
cache.fetch(:changelog) do
file_on_head(:changelog)
end
file_on_head(:changelog)
end
cache_method :changelog
def license_blob
return nil unless head_exists?
cache.fetch(:license_blob) do
file_on_head(:license)
end
file_on_head(:license)
end
cache_method :license_blob
def license_key
return nil unless head_exists?
return unless exists?
cache.fetch(:license_key) do
Licensee.license(path).try(:key)
end
Licensee.license(path).try(:key)
end
cache_method :license_key
def gitignore
return nil if !exists? || empty?
cache.fetch(:gitignore) do
file_on_head(:gitignore)
end
file_on_head(:gitignore)
end
cache_method :gitignore
def koding_yml
return nil unless head_exists?
cache.fetch(:koding_yml) do
file_on_head(:koding)
end
file_on_head(:koding)
end
cache_method :koding_yml
def gitlab_ci_yml
return nil unless head_exists?
@gitlab_ci_yml ||= cache.fetch(:gitlab_ci_yml) do
file_on_head(:gitlab_ci)
end
rescue Rugged::ReferenceError
# For unknow reason spinach scenario "Scenario: I change project path"
# lead to "Reference 'HEAD' not found" exception from Repository#empty?
nil
file_on_head(:gitlab_ci)
end
cache_method :gitlab_ci_yml
def head_commit
@head_commit ||= commit(self.root_ref)
end
def head_tree
@head_tree ||= Tree.new(self, head_commit.sha, nil)
if head_commit
@head_tree ||= Tree.new(self, head_commit.sha, nil)
end
end
def tree(sha = :head, path = nil, recursive: false)
if sha == :head
return unless head_commit
if path.nil?
return head_tree
else
......@@ -771,10 +784,6 @@ class Repository
@tags ||= raw_repository.tags
end
def root_ref
@root_ref ||= cache.fetch(:root_ref) { raw_repository.root_ref }
end
def commit_dir(user, path, message, branch, author_email: nil, author_name: nil)
update_branch_with_hooks(user, branch) do |ref|
options = {
......@@ -1132,12 +1141,41 @@ class Repository
end
end
def avatar
return nil unless exists?
# Caches the supplied block both in a cache and in an instance variable.
#
# The cache key and instance variable are named the same way as the value of
# the `key` argument.
#
# This method will return `nil` if the corresponding instance variable is also
# set to `nil`. This ensures we don't keep yielding the block when it returns
# `nil`.
#
# key - The name of the key to cache the data in.
# fallback - A value to fall back to in the event of a Git error.
def cache_method_output(key, fallback: nil, &block)
ivar = cache_instance_variable_name(key)
if instance_variable_defined?(ivar)
instance_variable_get(ivar)
else
begin
instance_variable_set(ivar, cache.fetch(key, &block))
rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository
# if e.g. HEAD or the entire repository doesn't exist we want to
# gracefully handle this and not cache anything.
fallback
end
end
end
@avatar ||= cache.fetch(:avatar) do
AVATAR_FILES.find do |file|
blob_at_branch(root_ref, file)
def cache_instance_variable_name(key)
:"@#{key.to_s.tr('?!', '')}"
end
def file_on_head(type)
if head = tree(:head)
head.blobs.find do |file|
Gitlab::FileDetector.type_of(file.name) == type
end
end
end
......@@ -1154,16 +1192,6 @@ class Repository
@cache ||= RepositoryCache.new(path_with_namespace, @project.id)
end
def head_exists?
exists? && !empty? && !rugged.head_unborn?
end
def file_on_head(type)
tree(:head).blobs.find do |file|
Gitlab::FileDetector.type_of(file.name) == type
end
end
def tags_sorted_by_committed_date
tags.sort_by { |tag| tag.dereferenced_target.committed_date }
end
......
app/services/git_push_service.rb
View file @ ffb9b3ef
......@@ -18,7 +18,7 @@ class GitPushService < BaseService
#
def execute
@project.repository.after_create if @project.empty_repo?
@project.repository.after_push_commit(branch_name, params[:newrev])
@project.repository.after_push_commit(branch_name)
if push_remove_branch?
@project.repository.after_remove_branch
......@@ -51,12 +51,32 @@ class GitPushService < BaseService
execute_related_hooks
perform_housekeeping
update_caches
end
def update_gitattributes
@project.repository.copy_gitattributes(params[:ref])
end
def update_caches
if is_default_branch?
paths = Set.new
@push_commits.each do |commit|
commit.raw_diffs(deltas_only: true).each do |diff|
paths << diff.new_path
end
end
types = Gitlab::FileDetector.types_in_paths(paths.to_a)
else
types = []
end
ProjectCacheWorker.perform_async(@project.id, types)
end
protected
def execute_related_hooks
......@@ -70,7 +90,6 @@ class GitPushService < BaseService
@project.execute_hooks(build_push_data.dup, :push_hooks)
@project.execute_services(build_push_data.dup, :push_hooks)
Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute
ProjectCacheWorker.perform_async(@project.id)
if push_remove_branch?
AfterBranchDeleteService
......
app/workers/project_cache_worker.rb
View file @ ffb9b3ef
# Worker for updating any project specific caches.
#
# This worker runs at most once every 15 minutes per project. This is to ensure
# that multiple instances of jobs for this worker don't hammer the underlying
# storage engine as much.
class ProjectCacheWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
LEASE_TIMEOUT = 15.minutes.to_i
def self.lease_for(project_id)
Gitlab::ExclusiveLease.
new("project_cache_worker:#{project_id}", timeout: LEASE_TIMEOUT)
end
# project_id - The ID of the project for which to flush the cache.
# refresh - An Array containing extra types of data to refresh such as
# `:readme` to flush the README and `:changelog` to flush the
# CHANGELOG.
def perform(project_id, refresh = [])
project = Project.find_by(id: project_id)
# Overwrite Sidekiq's implementation so we only schedule when actually needed.
def self.perform_async(project_id)
# If a lease for this project is still being held there's no point in
# scheduling a new job.
super unless lease_for(project_id).exists?
end
return unless project && project.repository.exists?
def perform(project_id)
if try_obtain_lease_for(project_id)
Rails.logger.
info("Obtained ProjectCacheWorker lease for project #{project_id}")
else
Rails.logger.
info("Could not obtain ProjectCacheWorker lease for project #{project_id}")
return
end
update_repository_size(project)
project.update_commit_count
update_caches(project_id)
project.repository.refresh_method_caches(refresh.map(&:to_sym))
end
def update_caches(project_id)
project = Project.find(project_id)
def update_repository_size(project)
return unless try_obtain_lease_for(project.id, :update_repository_size)
return unless project.repository.exists?
Rails.logger.info("Updating repository size for project #{project.id}")
project.update_repository_size
project.update_commit_count
if project.repository.root_ref
project.repository.build_cache
end
end
def try_obtain_lease_for(project_id)
self.class.lease_for(project_id).try_obtain
private
def try_obtain_lease_for(project_id, section)
Gitlab::ExclusiveLease.
new("project_cache_worker:#{project_id}:#{section}", timeout: LEASE_TIMEOUT).
try_obtain
end
end
spec/models/project_spec.rb
View file @ ffb9b3ef
......@@ -1572,7 +1572,7 @@ describe Project, models: true do
end
it 'expires the avatar cache' do
expect(project.repository).to receive(:expire_avatar_cache).with(project.default_branch)
expect(project.repository).to receive(:expire_avatar_cache)
project.change_head(project.default_branch)
end
......
spec/models/repository_spec.rb
View file @ ffb9b3ef
......@@ -464,11 +464,7 @@ describe Repository, models: true do
end
end
describe "#changelog" do
before do
repository.send(:cache).expire(:changelog)
end
describe "#changelog", caching: true do
it 'accepts changelog' do
expect(repository.tree).to receive(:blobs).and_return([TestBlob.new('changelog')])
......@@ -500,17 +496,16 @@ describe Repository, models: true do
end
end
describe "#license_blob" do
describe "#license_blob", caching: true do
before do
repository.send(:cache).expire(:license_blob)
repository.remove_file(user, 'LICENSE', 'Remove LICENSE', 'master')
end
it 'handles when HEAD points to non-existent ref' do
repository.commit_file(user, 'LICENSE', 'Copyright!', 'Add LICENSE', 'master', false)
rugged = double('rugged')
expect(rugged).to receive(:head_unborn?).and_return(true)
expect(repository).to receive(:rugged).and_return(rugged)
allow(repository).to receive(:file_on_head).
and_raise(Rugged::ReferenceError)
expect(repository.license_blob).to be_nil
end
......@@ -537,22 +532,18 @@ describe Repository, models: true do
end
end
describe '#license_key' do
describe '#license_key', caching: true do
before do
repository.send(:cache).expire(:license_key)
repository.remove_file(user, 'LICENSE', 'Remove LICENSE', 'master')
end
it 'handles when HEAD points to non-existent ref' do
repository.commit_file(user, 'LICENSE', 'Copyright!', 'Add LICENSE', 'master', false)
rugged = double('rugged')
expect(rugged).to receive(:head_unborn?).and_return(true)
expect(repository).to receive(:rugged).and_return(rugged)
it 'returns nil when no license is detected' do
expect(repository.license_key).to be_nil
end
it 'returns nil when no license is detected' do
it 'returns nil when the repository does not exist' do
expect(repository).to receive(:exists?).and_return(false)
expect(repository.license_key).to be_nil
end
......@@ -569,7 +560,7 @@ describe Repository, models: true do
end
end
describe "#gitlab_ci_yml" do
describe "#gitlab_ci_yml", caching: true do
it 'returns valid file' do
files = [TestBlob.new('file'), TestBlob.new('.gitlab-ci.yml'), TestBlob.new('copying')]
expect(repository.tree).to receive(:blobs).and_return(files)
......@@ -583,7 +574,7 @@ describe Repository, models: true do
end
it 'returns nil for empty repository' do
expect(repository).to receive(:empty?).and_return(true)
allow(repository).to receive(:file_on_head).and_raise(Rugged::ReferenceError)
expect(repository.gitlab_ci_yml).to be_nil
end
end
......@@ -778,7 +769,6 @@ describe Repository, models: true do
expect(repository).not_to receive(:expire_emptiness_caches)
expect(repository).to receive(:expire_branches_cache)
expect(repository).to receive(:expire_has_visible_content_cache)
expect(repository).to receive(:expire_branch_count_cache)
repository.update_branch_with_hooks(user, 'new-feature') { new_rev }
end
......@@ -797,7 +787,6 @@ describe Repository, models: true do
expect(empty_repository).to receive(:expire_emptiness_caches)
expect(empty_repository).to receive(:expire_branches_cache)
expect(empty_repository).to receive(:expire_has_visible_content_cache)
expect(empty_repository).to receive(:expire_branch_count_cache)
empty_repository.commit_file(user, 'CHANGELOG', 'Changelog!',
'Updates file content', 'master', false)
......@@ -915,34 +904,6 @@ describe Repository, models: true do
end
end
describe '#expire_cache' do
it 'expires all caches' do
expect(repository).to receive(:expire_branch_cache)
repository.expire_cache
end
it 'expires the caches for a specific branch' do
expect(repository).to receive(:expire_branch_cache).with('master')
repository.expire_cache('master')
end
it 'expires the emptiness caches for an empty repository' do
expect(repository).to receive(:empty?).and_return(true)
expect(repository).to receive(:expire_emptiness_caches)
repository.expire_cache
end
it 'does not expire the emptiness caches for a non-empty repository' do
expect(repository).to receive(:empty?).and_return(false)
expect(repository).not_to receive(:expire_emptiness_caches)
repository.expire_cache
end
end
describe '#expire_root_ref_cache' do
it 'expires the root reference cache' do
repository.root_ref
......@@ -1002,12 +963,23 @@ describe Repository, models: true do
describe '#expire_emptiness_caches' do
let(:cache) { repository.send(:cache) }
it 'expires the caches' do
it 'expires the caches for an empty repository' do
allow(repository).to receive(:empty?).and_return(true)
expect(cache).to receive(:expire).with(:empty?)
expect(repository).to receive(:expire_has_visible_content_cache)
repository.expire_emptiness_caches
end
it 'does not expire the cache for a non-empty repository' do
allow(repository).to receive(:empty?).and_return(false)
expect(cache).not_to receive(:expire).with(:empty?)
expect(repository).not_to receive(:expire_has_visible_content_cache)
repository.expire_emptiness_caches
end
end
describe :skip_merged_commit do
......@@ -1119,24 +1091,12 @@ describe Repository, models: true do
repository.before_delete
end
it 'flushes the tag count cache' do
expect(repository).to receive(:expire_tag_count_cache)
repository.before_delete
end
it 'flushes the branches cache' do
expect(repository).to receive(:expire_branches_cache)
repository.before_delete
end
it 'flushes the branch count cache' do
expect(repository).to receive(:expire_branch_count_cache)
repository.before_delete
end
it 'flushes the root ref cache' do
expect(repository).to receive(:expire_root_ref_cache)
......@@ -1161,36 +1121,18 @@ describe Repository, models: true do
allow(repository).to receive(:exists?).and_return(true)
end
it 'flushes the caches that depend on repository data' do
expect(repository).to receive(:expire_cache)
repository.before_delete
end
it 'flushes the tags cache' do
expect(repository).to receive(:expire_tags_cache)
repository.before_delete
end
it 'flushes the tag count cache' do
expect(repository).to receive(:expire_tag_count_cache)
repository.before_delete
end
it 'flushes the branches cache' do
expect(repository).to receive(:expire_branches_cache)
repository.before_delete
end
it 'flushes the branch count cache' do
expect(repository).to receive(:expire_branch_count_cache)
repository.before_delete
end
it 'flushes the root ref cache' do
expect(repository).to receive(:expire_root_ref_cache)
......@@ -1221,8 +1163,9 @@ describe Repository, models: true do
describe '#before_push_tag' do
it 'flushes the cache' do
expect(repository).to receive(:expire_cache)
expect(repository).to receive(:expire_tag_count_cache)
expect(repository).to receive(:expire_statistics_caches)
expect(repository).to receive(:expire_emptiness_caches)
expect(repository).to receive(:expire_tags_cache)
repository.before_push_tag
end
......@@ -1239,17 +1182,23 @@ describe Repository, models: true do
describe '#after_import' do
it 'flushes and builds the cache' do
expect(repository).to receive(:expire_content_cache)
expect(repository).to receive(:build_cache)
expect(repository).to receive(:expire_tags_cache)
expect(repository).to receive(:expire_branches_cache)
repository.after_import
end
end
describe '#after_push_commit' do
it 'flushes the cache' do
expect(repository).to receive(:expire_cache).with('master', '123')
it 'expires statistics caches' do
expect(repository).to receive(:expire_statistics_caches).
and_call_original
expect(repository).to receive(:expire_branch_cache).
with('master').
and_call_original
repository.after_push_commit('master', '123')
repository.after_push_commit('master')
end
end
......@@ -1301,7 +1250,8 @@ describe Repository, models: true do
describe '#before_remove_tag' do
it 'flushes the tag cache' do
expect(repository).to receive(:expire_tag_count_cache)
expect(repository).to receive(:expire_tags_cache).and_call_original
expect(repository).to receive(:expire_statistics_caches).and_call_original
repository.before_remove_tag
end
......@@ -1319,23 +1269,23 @@ describe Repository, models: true do
end
end
describe '#expire_branch_count_cache' do
let(:cache) { repository.send(:cache) }
describe '#expire_branches_cache' do
it 'expires the cache' do
expect(cache).to receive(:expire).with(:branch_count)
expect(repository).to receive(:expire_method_caches).
with(%i(branch_names branch_count)).
and_call_original
repository.expire_branch_count_cache
repository.expire_branches_cache
end
end
describe '#expire_tag_count_cache' do
let(:cache) { repository.send(:cache) }
describe '#expire_tags_cache' do
it 'expires the cache' do
expect(cache).to receive(:expire).with(:tag_count)
expect(repository).to receive(:expire_method_caches).
with(%i(tag_names tag_count)).
and_call_original
repository.expire_tag_count_cache
repository.expire_tags_cache
end
end
......@@ -1411,84 +1361,27 @@ describe Repository, models: true do
describe '#avatar' do
it 'returns nil if repo does not exist' do
expect(repository).to receive(:exists?).and_return(false)
expect(repository).to receive(:file_on_head).
and_raise(Rugged::ReferenceError)
expect(repository.avatar).to eq(nil)
end
it 'returns the first avatar file found in the repository' do
expect(repository).to receive(:blob_at_branch).
with('master', 'logo.png').
and_return(true)
expect(repository).to receive(:file_on_head).
with(:avatar).
and_return(double(:tree, path: 'logo.png'))
expect(repository.avatar).to eq('logo.png')
end
it 'caches the output' do
allow(repository).to receive(:blob_at_branch).
with('master', 'logo.png').
and_return(true)
expect(repository.avatar).to eq('logo.png')
expect(repository).not_to receive(:blob_at_branch)
expect(repository.avatar).to eq('logo.png')
end
end
describe '#expire_avatar_cache' do
let(:cache) { repository.send(:cache) }
before do
allow(repository).to receive(:cache).and_return(cache)
end
context 'without a branch or revision' do
it 'flushes the cache' do
expect(cache).to receive(:expire).with(:avatar)
repository.expire_avatar_cache
end
end
context 'with a branch' do
it 'does not flush the cache if the branch is not the default branch' do
expect(cache).not_to receive(:expire)
repository.expire_avatar_cache('cats')
end
it 'flushes the cache if the branch equals the default branch' do
expect(cache).to receive(:expire).with(:avatar)
repository.expire_avatar_cache(repository.root_ref)
end
end
context 'with a branch and revision' do
let(:commit) { double(:commit) }
before do
allow(repository).to receive(:commit).and_return(commit)
end
it 'does not flush the cache if the commit does not change any logos' do
diff = double(:diff, new_path: 'test.txt')
expect(commit).to receive(:raw_diffs).and_return([diff])
expect(cache).not_to receive(:expire)
repository.expire_avatar_cache(repository.root_ref, '123')
end
it 'flushes the cache if the commit changes any of the logos' do
diff = double(:diff, new_path: Repository::AVATAR_FILES[0])
expect(commit).to receive(:raw_diffs).and_return([diff])
expect(cache).to receive(:expire).with(:avatar)
expect(repository).to receive(:file_on_head).
with(:avatar).
once.
and_return(double(:tree, path: 'logo.png'))
repository.expire_avatar_cache(repository.root_ref, '123')
end
2.times { expect(repository.avatar).to eq('logo.png') }
end
end
......@@ -1502,40 +1395,6 @@ describe Repository, models: true do
end
end
describe '#build_cache' do
let(:cache) { repository.send(:cache) }
it 'builds the caches if they do not already exist' do
cache_keys = repository.cache_keys + repository.cache_keys_for_branches_and_tags
expect(cache).to receive(:exist?).
exactly(cache_keys.length).
times.
and_return(false)
cache_keys.each do |key|
expect(repository).to receive(key)
end
repository.build_cache
end
it 'does not build any caches that already exist' do
cache_keys = repository.cache_keys + repository.cache_keys_for_branches_and_tags
expect(cache).to receive(:exist?).
exactly(cache_keys.length).
times.
and_return(true)
cache_keys.each do |key|
expect(repository).not_to receive(key)
end
repository.build_cache
end
end
describe "#keep_around" do
it "does not fail if we attempt to reference bad commit" do
expect(repository.kept_around?('abc1234')).to be_falsey
......@@ -1592,83 +1451,226 @@ describe Repository, models: true do
end
end
describe '#changelog', caching: true do
describe '#gitignore', caching: true do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head).
with(:changelog).
and_return(Gitlab::Git::Tree.new(path: 'CHANGELOG')).
with(:gitignore).
and_return(Gitlab::Git::Tree.new(path: '.gitignore')).
once
2.times do
expect(repository.changelog).to be_an_instance_of(Gitlab::Git::Tree)
expect(repository.gitignore).to be_an_instance_of(Gitlab::Git::Tree)
end
end
end
describe '#license_blob', caching: true do
describe '#koding_yml', caching: true do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head).
with(:license).
and_return(Gitlab::Git::Tree.new(path: 'LICENSE')).
with(:koding).
and_return(Gitlab::Git::Tree.new(path: '.koding.yml')).
once
2.times do
expect(repository.license_blob).to be_an_instance_of(Gitlab::Git::Tree)
expect(repository.koding_yml).to be_an_instance_of(Gitlab::Git::Tree)
end
end
end
describe '#license_key', caching: true do
it 'returns and caches the output' do
license = double(key: 'mit')
describe '#readme', caching: true do
context 'with a non-existing repository' do
it 'returns nil' do
expect(repository).to receive(:tree).with(:head).and_return(nil)
expect(Licensee).to receive(:license).
with(repository.path).
and_return(license).
once
expect(repository.readme).to be_nil
end
end
2.times do
expect(repository.license_key).to eq('mit')
context 'with an existing repository' do
it 'returns the README' do
expect(repository.readme).to be_an_instance_of(Gitlab::Git::Blob)
end
end
end
describe '#gitignore', caching: true do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head).
with(:gitignore).
and_return(Gitlab::Git::Tree.new(path: '.gitignore')).
once
describe '#expire_statistics_caches' do
it 'expires the caches' do
expect(repository).to receive(:expire_method_caches).
with(%i(size commit_count))
2.times do
expect(repository.gitignore).to be_an_instance_of(Gitlab::Git::Tree)
repository.expire_statistics_caches
end
end
describe '#expire_method_caches' do
it 'expires the caches of the given methods' do
expect_any_instance_of(RepositoryCache).to receive(:expire).with(:readme)
expect_any_instance_of(RepositoryCache).to receive(:expire).with(:gitignore)
repository.expire_method_caches(%i(readme gitignore))
end
end
describe '#expire_all_method_caches' do
it 'expires the caches of all methods' do
expect(repository).to receive(:expire_method_caches).
with(Repository::CACHED_METHODS)
repository.expire_all_method_caches
end
end
describe '#expire_avatar_cache' do
it 'expires the cache' do
expect(repository).to receive(:expire_method_caches).with(%i(avatar))
repository.expire_avatar_cache
end
end
describe '#file_on_head' do
context 'with a non-existing repository' do
it 'returns nil' do
expect(repository).to receive(:tree).with(:head).and_return(nil)
expect(repository.file_on_head(:readme)).to be_nil
end
end
context 'with a repository that has no blobs' do
it 'returns nil' do
expect_any_instance_of(Tree).to receive(:blobs).and_return([])
expect(repository.file_on_head(:readme)).to be_nil
end
end
context 'with an existing repository' do
it 'returns a Gitlab::Git::Tree' do
expect(repository.file_on_head(:readme)).
to be_an_instance_of(Gitlab::Git::Tree)
end
end
end
describe '#koding_yml', caching: true do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head).
with(:koding).
and_return(Gitlab::Git::Tree.new(path: '.koding.yml')).
once
describe '#head_tree' do
context 'with an existing repository' do
it 'returns a Tree' do
expect(repository.head_tree).to be_an_instance_of(Tree)
end
end
2.times do
expect(repository.koding_yml).to be_an_instance_of(Gitlab::Git::Tree)
context 'with a non-existing repository' do
it 'returns nil' do
expect(repository).to receive(:head_commit).and_return(nil)
expect(repository.head_tree).to be_nil
end
end
end
describe '#gitlab_ci_yml', caching: true do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head).
with(:gitlab_ci).
and_return(Gitlab::Git::Tree.new(path: '.gitlab-ci.yml')).
once
describe '#tree' do
context 'using a non-existing repository' do
before do
allow(repository).to receive(:head_commit).and_return(nil)
end
2.times do
expect(repository.gitlab_ci_yml).to be_an_instance_of(Gitlab::Git::Tree)
it 'returns nil' do
expect(repository.tree(:head)).to be_nil
end
it 'returns nil when using a path' do
expect(repository.tree(:head, 'README.md')).to be_nil
end
end
context 'using an existing repository' do
it 'returns a Tree' do
expect(repository.tree(:head)).to be_an_instance_of(Tree)
end
end
end
describe '#size' do
context 'with a non-existing repository' do
it 'returns 0' do
expect(repository).to receive(:exists?).and_return(false)
expect(repository.size).to eq(0.0)
end
end
context 'with an existing repository' do
it 'returns the repository size as a Float' do
expect(repository.size).to be_an_instance_of(Float)
end
end
end
describe '#commit_count' do
context 'with a non-existing repository' do
it 'returns 0' do
expect(repository).to receive(:root_ref).and_return(nil)
expect(repository.commit_count).to eq(0)
end
end
context 'with an existing repository' do
it 'returns the commit count' do
expect(repository.commit_count).to be_an_instance_of(Fixnum)
end
end
end
describe '#cache_method_output', caching: true do
context 'with a non-existing repository' do
let(:value) do
repository.cache_method_output(:cats, fallback: 10) do
raise Rugged::ReferenceError
end
end
it 'returns a fallback value' do
expect(value).to eq(10)
end
it 'does not cache the data' do
value
expect(repository.instance_variable_defined?(:@cats)).to eq(false)
expect(repository.send(:cache).exist?(:cats)).to eq(false)
end
end
context 'with an existing repository' do
it 'caches the output' do
object = double
expect(object).to receive(:number).once.and_return(10)
2.times do
val = repository.cache_method_output(:cats) { object.number }
expect(val).to eq(10)
end
expect(repository.send(:cache).exist?(:cats)).to eq(true)
expect(repository.instance_variable_get(:@cats)).to eq(10)
end
end
end
describe '#refresh_method_caches' do
it 'refreshes the caches of the given types' do
expect(repository).to receive(:expire_method_caches).
with(%i(readme license_blob license_key))
expect(repository).to receive(:readme)
expect(repository).to receive(:license_blob)
expect(repository).to receive(:license_key)
repository.refresh_method_caches(%i(readme license))
end
end
end
spec/requests/api/branches_spec.rb
View file @ ffb9b3ef
......@@ -14,7 +14,7 @@ describe API::API, api: true do
describe "GET /projects/:id/repository/branches" do
it "returns an array of project branches" do
project.repository.expire_cache
project.repository.expire_all_method_caches
get api("/projects/#{project.id}/repository/branches", user)
expect(response).to have_http_status(200)
......
spec/services/git_push_service_spec.rb
View file @ ffb9b3ef
......@@ -27,27 +27,14 @@ describe GitPushService, services: true do
it { is_expected.to be_truthy }
it 'flushes general cached data' do
expect(project.repository).to receive(:expire_cache).
with('master', newrev)
it 'calls the after_push_commit hook' do
expect(project.repository).to receive(:after_push_commit).with('master')
subject
end
it 'flushes the visible content cache' do
expect(project.repository).to receive(:expire_has_visible_content_cache)
subject
end
it 'flushes the branches cache' do
expect(project.repository).to receive(:expire_branches_cache)
subject
end
it 'flushes the branch count cache' do
expect(project.repository).to receive(:expire_branch_count_cache)
it 'calls the after_create_branch hook' do
expect(project.repository).to receive(:after_create_branch)
subject
end
......@@ -56,21 +43,8 @@ describe GitPushService, services: true do
context 'existing branch' do
it { is_expected.to be_truthy }
it 'flushes general cached data' do
expect(project.repository).to receive(:expire_cache).
with('master', newrev)
subject
end
it 'does not flush the branches cache' do
expect(project.repository).not_to receive(:expire_branches_cache)
subject
end
it 'does not flush the branch count cache' do
expect(project.repository).not_to receive(:expire_branch_count_cache)
it 'calls the after_push_commit hook' do
expect(project.repository).to receive(:after_push_commit).with('master')
subject
end
......@@ -81,27 +55,14 @@ describe GitPushService, services: true do
it { is_expected.to be_truthy }
it 'flushes the visible content cache' do
expect(project.repository).to receive(:expire_has_visible_content_cache)
subject
end
it 'flushes the branches cache' do
expect(project.repository).to receive(:expire_branches_cache)
subject
end
it 'flushes the branch count cache' do
expect(project.repository).to receive(:expire_branch_count_cache)
it 'calls the after_push_commit hook' do
expect(project.repository).to receive(:after_push_commit).with('master')
subject
end
it 'flushes general cached data' do
expect(project.repository).to receive(:expire_cache).
with('master', newrev)
it 'calls the after_remove_branch hook' do
expect(project.repository).to receive(:after_remove_branch)
subject
end
......@@ -598,6 +559,51 @@ describe GitPushService, services: true do
end
end
describe '#update_caches' do
let(:service) do
described_class.new(project,
user,
oldrev: sample_commit.parent_id,
newrev: sample_commit.id,
ref: 'refs/heads/master')
end
context 'on the default branch' do
before do
allow(service).to receive(:is_default_branch?).and_return(true)
end
it 'flushes the caches of any special files that have been changed' do
commit = double(:commit)
diff = double(:diff, new_path: 'README.md')
expect(commit).to receive(:raw_diffs).with(deltas_only: true).
and_return([diff])
service.push_commits = [commit]
expect(ProjectCacheWorker).to receive(:perform_async).
with(project.id, %i(readme))
service.update_caches
end
end
context 'on a non-default branch' do
before do
allow(service).to receive(:is_default_branch?).and_return(false)
end
it 'does not flush any conditional caches' do
expect(ProjectCacheWorker).to receive(:perform_async).
with(project.id, []).
and_call_original
service.update_caches
end
end
end
def execute_service(project, user, oldrev, newrev, ref)
service = described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref )
service.execute
......
spec/services/git_tag_push_service_spec.rb
View file @ ffb9b3ef
......@@ -18,7 +18,7 @@ describe GitTagPushService, services: true do
end
it 'flushes general cached data' do
expect(project.repository).to receive(:expire_cache)
expect(project.repository).to receive(:before_push_tag)
subject
end
......@@ -28,12 +28,6 @@ describe GitTagPushService, services: true do
subject
end
it 'flushes the tag count cache' do
expect(project.repository).to receive(:expire_tag_count_cache)
subject
end
end
describe "Git Tag Push Data" do
......
spec/workers/project_cache_worker_spec.rb
View file @ ffb9b3ef
......@@ -2,62 +2,78 @@ require 'spec_helper'
describe ProjectCacheWorker do
let(:project) { create(:project) }
let(:worker) { described_class.new }
subject { described_class.new }
describe '.perform_async' do
it 'schedules the job when no lease exists' do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:exists?).
and_return(false)
describe '#perform' do
before do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).
and_return(true)
end
expect_any_instance_of(described_class).to receive(:perform)
context 'with a non-existing project' do
it 'does nothing' do
expect(worker).not_to receive(:update_repository_size)
described_class.perform_async(project.id)
worker.perform(-1)
end
end
it 'does not schedule the job when a lease exists' do
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:exists?).
and_return(true)
context 'with an existing project without a repository' do
it 'does nothing' do
allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
expect_any_instance_of(described_class).not_to receive(:perform)
expect(worker).not_to receive(:update_repository_size)
described_class.perform_async(project.id)
worker.perform(project.id)
end
end
end
describe '#perform' do
context 'when an exclusive lease can be obtained' do
before do
allow(subject).to receive(:try_obtain_lease_for).with(project.id).
and_return(true)
end
context 'with an existing project' do
it 'updates the repository size' do
expect(worker).to receive(:update_repository_size).and_call_original
it 'updates project cache data' do
expect_any_instance_of(Repository).to receive(:size)
expect_any_instance_of(Repository).to receive(:commit_count)
worker.perform(project.id)
end
expect_any_instance_of(Project).to receive(:update_repository_size)
expect_any_instance_of(Project).to receive(:update_commit_count)
it 'updates the commit count' do
expect_any_instance_of(Project).to receive(:update_commit_count).
and_call_original
subject.perform(project.id)
worker.perform(project.id)
end
it 'handles missing repository data' do
expect_any_instance_of(Repository).to receive(:exists?).and_return(false)
expect_any_instance_of(Repository).not_to receive(:size)
it 'refreshes the method caches' do
expect_any_instance_of(Repository).to receive(:refresh_method_caches).
with(%i(readme)).
and_call_original
subject.perform(project.id)
worker.perform(project.id, %i(readme))
end
end
end
context 'when an exclusive lease can not be obtained' do
it 'does nothing' do
allow(subject).to receive(:try_obtain_lease_for).with(project.id).
describe '#update_repository_size' do
context 'when a lease could not be obtained' do
it 'does not update the repository size' do
allow(worker).to receive(:try_obtain_lease_for).
with(project.id, :update_repository_size).
and_return(false)
expect(subject).not_to receive(:update_caches)
expect(project).not_to receive(:update_repository_size)
worker.update_repository_size(project)
end
end
context 'when a lease could be obtained' do
it 'updates the repository size' do
allow(worker).to receive(:try_obtain_lease_for).
with(project.id, :update_repository_size).
and_return(true)
expect(project).to receive(:update_repository_size).and_call_original
subject.perform(project.id)
worker.update_repository_size(project)
end
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7