Merge branch 'cache-improvement' into '7-13-stable'

Merge branch 'advanced-cache' into 'master'

Backport to stable from merge request !986

- - -

Advanced cache

Fixes #1993

* Build missing cache values in background job after each push
* Store commit_count in project table. Refresh in background job
* moved repository size calculation in background job


## Advantages

Every time push is triggered - we build cache for project even without user visiting project page. 
That means first visit is as fast as others now. This is especially good for active projects where people have some requests fast because of cache and some slow - because cache was build in their request. 

Between cache expired and cache built we we had gap when Linux repo can give 502 error because calculation commits count takes 30 seconds or even more. 
Using value from database fix this problem. Before cache is updated you see old value from database. After - you see new one. 

Basically this merge request is super win to GitLab. We don't do some heavy operations in user request but instead in background job. 

## Temporary problem

After this migration all projects will have `0` commits in database. It fill be replaced with real value on next push. I did not add recalculation to migration because it will last forever on big instances.  Can be fixed by adding rake task which will go in background on live instance without downtime



See merge request !989

app/helpers/projects_helper.rb

@@ -276,4 +276,18 @@ module ProjectsHelper
   def readme_cache_key
     [@project.id, @project.commit.sha, "readme"].join('-')
   end
+
+  def round_commit_count(project)
+    count = project.commit_count
+
+    if count > 10000
+      '10000+'
+    elsif count > 5000
+      '5000+'
+    elsif count > 1000
+      '1000+'
+    else
+      count
+    end
+  end
 end

app/models/project.rb

@@ -683,6 +683,10 @@ class Project < ActiveRecord::Base
     update_attribute(:repository_size, repository.size)
   end
 
+  def update_commit_count
+    update_attribute(:commit_count, repository.commit_count)
+  end
+
   def forks_count
     ForkedProjectLink.where(forked_from_project_id: self.id).count
   end

app/models/repository.rb

@@ -94,18 +94,6 @@ class Repository
     gitlab_shell.rm_tag(path_with_namespace, tag_name)
   end
 
-  def round_commit_count
-    if commit_count > 10000
-      '10000+'
-    elsif commit_count > 5000
-      '5000+'
-    elsif commit_count > 1000
-      '1000+'
-    else
-      commit_count
-    end
-  end
-
   def branch_names
     cache.fetch(:branch_names) { raw_repository.branch_names }
   end
@@ -130,10 +118,29 @@ class Repository
     cache.fetch(:size) { raw_repository.size }
   end
 
-  def expire_cache
+  def cache_keys
     %i(size branch_names tag_names commit_count graph_log
-       readme version contribution_guide changelog license).each do |key|
+       readme version contribution_guide changelog license)
+  end
+
+  def build_cache
+    cache_keys.each do |key|
+      unless cache.exist?(key)
+        send(key)
+      end
+    end
+  end
+
+  def expire_cache
+    cache_keys.each do |key|
+      cache.expire(key)
+    end
+  end
+
+  def rebuild_cache
+    cache_keys.each do |key|
       cache.expire(key)
+      send(key)
     end
   end
 

app/services/git_push_service.rb

@@ -21,7 +21,6 @@ class GitPushService
 
     project.ensure_satellite_exists
     project.repository.expire_cache
-    project.update_repository_size
 
     if push_remove_branch?(ref, newrev)
       @push_commits = []
@@ -61,6 +60,7 @@ class GitPushService
     EventCreateService.new.push(project, user, @push_data)
     project.execute_hooks(@push_data.dup, :push_hooks)
     project.execute_services(@push_data.dup, :push_hooks)
+    ProjectCacheWorker.perform_async(project.id)
   end
 
   protected

app/services/git_tag_push_service.rb

@@ -2,15 +2,15 @@ class GitTagPushService
   attr_accessor :project, :user, :push_data
 
   def execute(project, user, oldrev, newrev, ref)
-    @project, @user = project, user
+    project.repository.expire_cache
 
+    @project, @user = project, user
     @push_data = build_push_data(oldrev, newrev, ref)
 
     EventCreateService.new.push(project, user, @push_data)
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
-
-    project.repository.expire_cache
+    ProjectCacheWorker.perform_async(project.id)
 
     true
   end

app/views/explore/projects/_project.html.haml

@@ -14,7 +14,7 @@
 
     .repo-info
       - unless project.empty_repo?
-        = link_to pluralize(project.repository.round_commit_count, 'commit'), namespace_project_commits_path(project.namespace, project, project.default_branch)
+        = link_to pluralize(round_commit_count(project), 'commit'), namespace_project_commits_path(project.namespace, project, project.default_branch)
         ·
         = link_to pluralize(project.repository.branch_names.count, 'branch'), namespace_project_branches_path(project.namespace, project)
         ·

app/views/projects/show.html.haml

@@ -13,7 +13,7 @@
   %ul.nav.nav-pills
     %li
       = link_to namespace_project_commits_path(@project.namespace, @project, @ref || @repository.root_ref) do
-        = pluralize(number_with_delimiter(@repository.commit_count), 'commit')
+        = pluralize(number_with_delimiter(@project.commit_count), 'commit')
     %li
       = link_to namespace_project_branches_path(@project.namespace, @project) do
         = pluralize(number_with_delimiter(@repository.branch_names.count), 'branch')

app/workers/project_cache_worker.rb

+class ProjectCacheWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :default
+
+  def perform(project_id)
+    project = Project.find(project_id)
+    project.update_repository_size
+    project.update_commit_count
+
+    if project.repository.root_ref
+      project.repository.build_cache
+    end
+  end
+end

app/workers/repository_import_worker.rb

@@ -28,7 +28,7 @@ class RepositoryImportWorker
     project.import_finish
     project.save
     project.satellite.create unless project.satellite.exists?
-    project.update_repository_size
+    ProjectCacheWorker.perform_async(project.id)
     Gitlab::BitbucketImport::KeyDeleter.new(project).execute if project.import_type == 'bitbucket'
   end
 end

db/fixtures/development/04_project.rb

@@ -11,9 +11,42 @@ Sidekiq::Testing.inline! do
       'https://github.com/twitter/flight.git',
       'https://github.com/twitter/typeahead.js.git',
       'https://github.com/h5bp/html5-boilerplate.git',
+      'https://github.com/google/material-design-lite.git',
+      'https://github.com/jlevy/the-art-of-command-line.git',
+      'https://github.com/FreeCodeCamp/freecodecamp.git',
+      'https://github.com/google/deepdream.git',
+      'https://github.com/jtleek/datasharing.git',
+      'https://github.com/WebAssembly/design.git',
+      'https://github.com/airbnb/javascript.git',
+      'https://github.com/tessalt/echo-chamber-js.git',
+      'https://github.com/atom/atom.git',
+      'https://github.com/ipselon/react-ui-builder.git',
+      'https://github.com/mattermost/platform.git',
+      'https://github.com/purifycss/purifycss.git',
+      'https://github.com/facebook/nuclide.git',
+      'https://github.com/wbkd/awesome-d3.git',
+      'https://github.com/kilimchoi/engineering-blogs.git',
+      'https://github.com/gilbarbara/logos.git',
+      'https://github.com/gaearon/redux.git',
+      'https://github.com/awslabs/s2n.git',
+      'https://github.com/arkency/reactjs_koans.git',
+      'https://github.com/twbs/bootstrap.git',
+      'https://github.com/chjj/ttystudio.git',
+      'https://github.com/DrBoolean/mostly-adequate-guide.git',
+      'https://github.com/octocat/Spoon-Knife.git',
+      'https://github.com/opencontainers/runc.git',
+      'https://github.com/googlesamples/android-topeka.git'
     ]
 
-    project_urls.each_with_index do |url, i|
+    # You can specify how many projects you need during seed execution
+    size = if ENV['SIZE'].present?
+             ENV['SIZE'].to_i
+           else
+             8
+           end
+
+
+    project_urls.first(size).each_with_index do |url, i|
       group_path, project_path = url.split('/')[-2..-1]
 
       group = Group.find_by(path: group_path)

db/migrate/20150717130904_add_commits_count_to_project.rb

+class AddCommitsCountToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :commit_count, :integer, default: 0
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150713160110) do
+ActiveRecord::Schema.define(version: 20150717130904) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -374,6 +374,7 @@ ActiveRecord::Schema.define(version: 20150713160110) do
     t.integer  "star_count",             default: 0,        null: false
     t.string   "import_type"
     t.string   "import_source"
+    t.integer  "commit_count",           default: 0
   end
 
   add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree

lib/repository_cache.rb

@@ -18,4 +18,12 @@ class RepositoryCache
   def fetch(key, &block)
     backend.fetch(cache_key(key), &block)
   end
+
+  def exist?(key)
+    backend.exist?(cache_key(key))
+  end
+
+  def read(key)
+    backend.read(cache_key(key))
+  end
 end