Commit 94c2516a authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'cache-improvement' into '7-13-stable'

Merge branch 'advanced-cache' into 'master'

Backport to stable from merge request !986

- - -

Advanced cache

Fixes #1993

* Build missing cache values in background job after each push
* Store commit_count in project table. Refresh in background job
* moved repository size calculation in background job


## Advantages

Every time push is triggered - we build cache for project even without user visiting project page. 
That means first visit is as fast as others now. This is especially good for active projects where people have some requests fast because of cache and some slow - because cache was build in their request. 

Between cache expired and cache built we we had gap when Linux repo can give 502 error because calculation commits count takes 30 seconds or even more. 
Using value from database fix this problem. Before cache is updated you see old value from database. After - you see new one. 

Basically this merge request is super win to GitLab. We don't do some heavy operations in user request but instead in background job. 

## Temporary problem

After this migration all projects will have `0` commits in database. It fill be replaced with real value on next push. I did not add recalculation to migration because it will last forever on big instances.  Can be fixed by adding rake task which will go in background on live instance without downtime



See merge request !989
parents ae44e5ec 2fa3acaa
...@@ -276,4 +276,18 @@ module ProjectsHelper ...@@ -276,4 +276,18 @@ module ProjectsHelper
def readme_cache_key def readme_cache_key
[@project.id, @project.commit.sha, "readme"].join('-') [@project.id, @project.commit.sha, "readme"].join('-')
end end
def round_commit_count(project)
count = project.commit_count
if count > 10000
'10000+'
elsif count > 5000
'5000+'
elsif count > 1000
'1000+'
else
count
end
end
end end
...@@ -683,6 +683,10 @@ class Project < ActiveRecord::Base ...@@ -683,6 +683,10 @@ class Project < ActiveRecord::Base
update_attribute(:repository_size, repository.size) update_attribute(:repository_size, repository.size)
end end
def update_commit_count
update_attribute(:commit_count, repository.commit_count)
end
def forks_count def forks_count
ForkedProjectLink.where(forked_from_project_id: self.id).count ForkedProjectLink.where(forked_from_project_id: self.id).count
end end
......
...@@ -94,18 +94,6 @@ class Repository ...@@ -94,18 +94,6 @@ class Repository
gitlab_shell.rm_tag(path_with_namespace, tag_name) gitlab_shell.rm_tag(path_with_namespace, tag_name)
end end
def round_commit_count
if commit_count > 10000
'10000+'
elsif commit_count > 5000
'5000+'
elsif commit_count > 1000
'1000+'
else
commit_count
end
end
def branch_names def branch_names
cache.fetch(:branch_names) { raw_repository.branch_names } cache.fetch(:branch_names) { raw_repository.branch_names }
end end
...@@ -130,10 +118,29 @@ class Repository ...@@ -130,10 +118,29 @@ class Repository
cache.fetch(:size) { raw_repository.size } cache.fetch(:size) { raw_repository.size }
end end
def expire_cache def cache_keys
%i(size branch_names tag_names commit_count graph_log %i(size branch_names tag_names commit_count graph_log
readme version contribution_guide changelog license).each do |key| readme version contribution_guide changelog license)
end
def build_cache
cache_keys.each do |key|
unless cache.exist?(key)
send(key)
end
end
end
def expire_cache
cache_keys.each do |key|
cache.expire(key)
end
end
def rebuild_cache
cache_keys.each do |key|
cache.expire(key) cache.expire(key)
send(key)
end end
end end
......
...@@ -21,7 +21,6 @@ class GitPushService ...@@ -21,7 +21,6 @@ class GitPushService
project.ensure_satellite_exists project.ensure_satellite_exists
project.repository.expire_cache project.repository.expire_cache
project.update_repository_size
if push_remove_branch?(ref, newrev) if push_remove_branch?(ref, newrev)
@push_commits = [] @push_commits = []
...@@ -61,6 +60,7 @@ class GitPushService ...@@ -61,6 +60,7 @@ class GitPushService
EventCreateService.new.push(project, user, @push_data) EventCreateService.new.push(project, user, @push_data)
project.execute_hooks(@push_data.dup, :push_hooks) project.execute_hooks(@push_data.dup, :push_hooks)
project.execute_services(@push_data.dup, :push_hooks) project.execute_services(@push_data.dup, :push_hooks)
ProjectCacheWorker.perform_async(project.id)
end end
protected protected
......
...@@ -2,15 +2,15 @@ class GitTagPushService ...@@ -2,15 +2,15 @@ class GitTagPushService
attr_accessor :project, :user, :push_data attr_accessor :project, :user, :push_data
def execute(project, user, oldrev, newrev, ref) def execute(project, user, oldrev, newrev, ref)
@project, @user = project, user project.repository.expire_cache
@project, @user = project, user
@push_data = build_push_data(oldrev, newrev, ref) @push_data = build_push_data(oldrev, newrev, ref)
EventCreateService.new.push(project, user, @push_data) EventCreateService.new.push(project, user, @push_data)
project.execute_hooks(@push_data.dup, :tag_push_hooks) project.execute_hooks(@push_data.dup, :tag_push_hooks)
project.execute_services(@push_data.dup, :tag_push_hooks) project.execute_services(@push_data.dup, :tag_push_hooks)
ProjectCacheWorker.perform_async(project.id)
project.repository.expire_cache
true true
end end
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
.repo-info .repo-info
- unless project.empty_repo? - unless project.empty_repo?
= link_to pluralize(project.repository.round_commit_count, 'commit'), namespace_project_commits_path(project.namespace, project, project.default_branch) = link_to pluralize(round_commit_count(project), 'commit'), namespace_project_commits_path(project.namespace, project, project.default_branch)
&middot; &middot;
= link_to pluralize(project.repository.branch_names.count, 'branch'), namespace_project_branches_path(project.namespace, project) = link_to pluralize(project.repository.branch_names.count, 'branch'), namespace_project_branches_path(project.namespace, project)
&middot; &middot;
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
%ul.nav.nav-pills %ul.nav.nav-pills
%li %li
= link_to namespace_project_commits_path(@project.namespace, @project, @ref || @repository.root_ref) do = link_to namespace_project_commits_path(@project.namespace, @project, @ref || @repository.root_ref) do
= pluralize(number_with_delimiter(@repository.commit_count), 'commit') = pluralize(number_with_delimiter(@project.commit_count), 'commit')
%li %li
= link_to namespace_project_branches_path(@project.namespace, @project) do = link_to namespace_project_branches_path(@project.namespace, @project) do
= pluralize(number_with_delimiter(@repository.branch_names.count), 'branch') = pluralize(number_with_delimiter(@repository.branch_names.count), 'branch')
......
class ProjectCacheWorker
include Sidekiq::Worker
sidekiq_options queue: :default
def perform(project_id)
project = Project.find(project_id)
project.update_repository_size
project.update_commit_count
if project.repository.root_ref
project.repository.build_cache
end
end
end
...@@ -28,7 +28,7 @@ class RepositoryImportWorker ...@@ -28,7 +28,7 @@ class RepositoryImportWorker
project.import_finish project.import_finish
project.save project.save
project.satellite.create unless project.satellite.exists? project.satellite.create unless project.satellite.exists?
project.update_repository_size ProjectCacheWorker.perform_async(project.id)
Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket' Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket'
end end
end end
...@@ -11,9 +11,42 @@ Sidekiq::Testing.inline! do ...@@ -11,9 +11,42 @@ Sidekiq::Testing.inline! do
'https://github.com/twitter/flight.git', 'https://github.com/twitter/flight.git',
'https://github.com/twitter/typeahead.js.git', 'https://github.com/twitter/typeahead.js.git',
'https://github.com/h5bp/html5-boilerplate.git', 'https://github.com/h5bp/html5-boilerplate.git',
'https://github.com/google/material-design-lite.git',
'https://github.com/jlevy/the-art-of-command-line.git',
'https://github.com/FreeCodeCamp/freecodecamp.git',
'https://github.com/google/deepdream.git',
'https://github.com/jtleek/datasharing.git',
'https://github.com/WebAssembly/design.git',
'https://github.com/airbnb/javascript.git',
'https://github.com/tessalt/echo-chamber-js.git',
'https://github.com/atom/atom.git',
'https://github.com/ipselon/react-ui-builder.git',
'https://github.com/mattermost/platform.git',
'https://github.com/purifycss/purifycss.git',
'https://github.com/facebook/nuclide.git',
'https://github.com/wbkd/awesome-d3.git',
'https://github.com/kilimchoi/engineering-blogs.git',
'https://github.com/gilbarbara/logos.git',
'https://github.com/gaearon/redux.git',
'https://github.com/awslabs/s2n.git',
'https://github.com/arkency/reactjs_koans.git',
'https://github.com/twbs/bootstrap.git',
'https://github.com/chjj/ttystudio.git',
'https://github.com/DrBoolean/mostly-adequate-guide.git',
'https://github.com/octocat/Spoon-Knife.git',
'https://github.com/opencontainers/runc.git',
'https://github.com/googlesamples/android-topeka.git'
] ]
project_urls.each_with_index do |url, i| # You can specify how many projects you need during seed execution
size = if ENV['SIZE'].present?
ENV['SIZE'].to_i
else
8
end
project_urls.first(size).each_with_index do |url, i|
group_path, project_path = url.split('/')[-2..-1] group_path, project_path = url.split('/')[-2..-1]
group = Group.find_by(path: group_path) group = Group.find_by(path: group_path)
......
class AddCommitsCountToProject < ActiveRecord::Migration
def change
add_column :projects, :commit_count, :integer, default: 0
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20150713160110) do ActiveRecord::Schema.define(version: 20150717130904) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -374,6 +374,7 @@ ActiveRecord::Schema.define(version: 20150713160110) do ...@@ -374,6 +374,7 @@ ActiveRecord::Schema.define(version: 20150713160110) do
t.integer "star_count", default: 0, null: false t.integer "star_count", default: 0, null: false
t.string "import_type" t.string "import_type"
t.string "import_source" t.string "import_source"
t.integer "commit_count", default: 0
end end
add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree
......
...@@ -18,4 +18,12 @@ class RepositoryCache ...@@ -18,4 +18,12 @@ class RepositoryCache
def fetch(key, &block) def fetch(key, &block)
backend.fetch(cache_key(key), &block) backend.fetch(cache_key(key), &block)
end end
def exist?(key)
backend.exist?(cache_key(key))
end
def read(key)
backend.read(cache_key(key))
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment