Merge branch 'ee-add-wiki-size-to-statistics' into 'master'

Add wiki size to project statistics (EE port of gitlab-org/gitlab-ce!25321)

See merge request gitlab-org/gitlab-ee!11984

app/helpers/storage_helper.rb

@@ -12,10 +12,11 @@ module StorageHelper
   def storage_counters_details(statistics)
     counters = {
       counter_repositories: storage_counter(statistics.repository_size),
+      counter_wikis: storage_counter(statistics.wiki_size),
       counter_build_artifacts: storage_counter(statistics.build_artifacts_size),
       counter_lfs_objects: storage_counter(statistics.lfs_objects_size)
     }
 
-    _("%{counter_repositories} repositories, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS") % counters
+    _("%{counter_repositories} repositories, %{counter_wikis} wikis, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS") % counters
   end
 end

app/models/namespace.rb

@@ -76,6 +76,7 @@ class Namespace < ApplicationRecord
         'namespaces.*',
         'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
         'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
+        'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size',
         'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
         'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
         'COALESCE(SUM(ps.packages_size), 0) AS packages_size'

app/models/project_statistics.rb

@@ -4,9 +4,16 @@ class ProjectStatistics < ApplicationRecord
   belongs_to :project
   belongs_to :namespace
 
+  default_value_for :wiki_size, 0
+
+  # older migrations fail due to non-existent attribute without this
+  def wiki_size
+    has_attribute?(:wiki_size) ? super : 0
+  end
+
   before_save :update_storage_size
 
-  COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze
+  COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count].freeze
   INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size] }.freeze
 
   def total_repository_size
@@ -27,11 +34,14 @@ class ProjectStatistics < ApplicationRecord
     self.commit_count = project.repository.commit_count
   end
 
-  # Repository#size needs to be converted from MB to Byte.
   def update_repository_size
     self.repository_size = project.repository.size * 1.megabyte
   end
 
+  def update_wiki_size
+    self.wiki_size = project.wiki.repository.size * 1.megabyte
+  end
+
   def update_lfs_objects_size
     self.lfs_objects_size = project.lfs_objects.sum(:size)
   end
@@ -42,7 +52,7 @@ class ProjectStatistics < ApplicationRecord
   end
 
   def update_storage_size
-    self.storage_size = repository_size + lfs_objects_size + build_artifacts_size + packages_size
+    self.storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size
   end
 
   # Since this incremental update method does not call update_storage_size above,

app/services/projects/update_statistics_service.rb

@@ -3,7 +3,7 @@
 module Projects
   class UpdateStatisticsService < BaseService
     def execute
-      return unless project && project.repository.exists?
+      return unless project
 
       Rails.logger.info("Updating statistics for project #{project.id}")
 

app/workers/post_receive.rb

@@ -74,6 +74,8 @@ class PostReceive
 
   def process_wiki_changes(post_received)
     post_received.project.touch(:last_activity_at, :last_repository_updated_at)
+    post_received.project.wiki.repository.expire_statistics_caches
+    ProjectCacheWorker.perform_async(post_received.project.id, [], [:wiki_size])
   end
 
   def log(message)

app/workers/project_cache_worker.rb

@@ -15,10 +15,12 @@ class ProjectCacheWorker
   def perform(project_id, files = [], statistics = [])
     project = Project.find_by(id: project_id)
 
-    return unless project && project.repository.exists?
+    return unless project
 
     update_statistics(project, statistics)
 
+    return unless project.repository.exists?
+
     project.repository.refresh_method_caches(files.map(&:to_sym))
 
     project.cleanup

changelogs/unreleased/add-wiki-size-to-statistics.yml

+---
+title: Add wiki size to project statistics
+merge_request: 25321
+author: Peter Marko
+type: added

db/migrate/20190527194830_add_wiki_size_to_statistics.rb

+# frozen_string_literal: true
+
+class AddWikiSizeToStatistics < ActiveRecord::Migration[5.0]
+  DOWNTIME = false
+
+  def change
+    add_column :project_statistics, :wiki_size, :bigint
+  end
+end

db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb

+# frozen_string_literal: true
+
+class ScheduleCalculateWikiSizes < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'CalculateWikiSizes'
+  BATCH_SIZE = 100000
+  BATCH_TIME = 5.minutes
+
+  class ProjectStatistics < ActiveRecord::Base
+    self.table_name = 'project_statistics'
+
+    scope :without_wiki_size, -> { where(wiki_size: nil) }
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      ::ScheduleCalculateWikiSizes::ProjectStatistics.without_wiki_size,
+      MIGRATION,
+      BATCH_TIME,
+      batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190524062810) do
+ActiveRecord::Schema.define(version: 20190527194900) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -2494,6 +2494,7 @@ ActiveRecord::Schema.define(version: 20190524062810) do
     t.bigint "shared_runners_seconds", default: 0, null: false
     t.datetime "shared_runners_seconds_last_reset"
     t.bigint "packages_size"
+    t.bigint "wiki_size"
     t.index ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
     t.index ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
   end

doc/api/groups.md

@@ -68,6 +68,7 @@ GET /groups?statistics=true
     "statistics": {
       "storage_size" : 212,
       "repository_size" : 33,
+      "wiki_size" : 100,
       "lfs_objects_size" : 123,
       "job_artifacts_size" : 57,
       "packages_size": 0

doc/api/projects.md

@@ -154,6 +154,7 @@ When the user is authenticated and `simple` is not set this returns something li
       "commit_count": 37,
       "storage_size": 1038090,
       "repository_size": 1038090,
+      "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
       "packages_size": 0
@@ -237,6 +238,7 @@ When the user is authenticated and `simple` is not set this returns something li
       "commit_count": 12,
       "storage_size": 2066080,
       "repository_size": 2066080,
+      "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
       "packages_size": 0
@@ -346,6 +348,7 @@ GET /users/:user_id/projects
       "commit_count": 37,
       "storage_size": 1038090,
       "repository_size": 1038090,
+      "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
       "packages_size": 0
@@ -428,6 +431,7 @@ GET /users/:user_id/projects
       "commit_count": 12,
       "storage_size": 2066080,
       "repository_size": 2066080,
+      "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
       "packages_size": 0
@@ -556,6 +560,7 @@ GET /projects/:id
     "commit_count": 37,
     "storage_size": 1038090,
     "repository_size": 1038090,
+    "wiki_size" : 0,
     "lfs_objects_size": 0,
     "job_artifacts_size": 0,
     "packages_size": 0

ee/spec/workers/post_receive_spec.rb

@@ -8,7 +8,7 @@ describe PostReceive do
   let(:gl_repository) { "project-#{project.id}" }
   let(:key) { create(:key, user: project.owner) }
   let(:key_id) { key.shell_id }
-  let(:project) { create(:project, :repository) }
+  let(:project) { create(:project, :repository, :wiki_repo) }
 
   describe "#process_project_changes" do
     before do
@@ -88,7 +88,7 @@ describe PostReceive do
 
       context 'when the project is not enabled specifically' do
         it 'does not trigger wiki index update' do
-          expect(ProjectWiki).not_to receive(:new)
+          expect_any_instance_of(ProjectWiki).not_to receive(:index_blobs)
 
           described_class.new.perform(gl_repository, key_id, base64_changes)
         end
@@ -100,9 +100,7 @@ describe PostReceive do
         end
 
         it 'triggers wiki index update' do
-          expect_next_instance_of(ProjectWiki) do |project_wiki|
-            expect(project_wiki).to receive(:index_blobs)
-          end
+          expect_any_instance_of(ProjectWiki).to receive(:index_blobs)
 
           described_class.new.perform(gl_repository, key_id, base64_changes)
         end
@@ -110,7 +108,7 @@ describe PostReceive do
 
       context 'when a group is enabled' do
         let(:group) { create(:group) }
-        let(:project) { create(:project, group: group) }
+        let(:project) { create(:project, :wiki_repo, group: group) }
         let(:key) { create(:key, user: group.owner) }
 
         before do
@@ -118,9 +116,7 @@ describe PostReceive do
         end
 
         it 'triggers wiki index update' do
-          expect_next_instance_of(ProjectWiki) do |project_wiki|
-            expect(project_wiki).to receive(:index_blobs)
-          end
+          expect_any_instance_of(ProjectWiki).to receive(:index_blobs)
 
           described_class.new.perform(gl_repository, key_id, base64_changes)
         end

lib/api/entities.rb

@@ -303,6 +303,7 @@ module API
       expose :commit_count
       expose :storage_size
       expose :repository_size
+      expose :wiki_size
       expose :lfs_objects_size
       expose :build_artifacts_size, as: :job_artifacts_size
     end
@@ -355,6 +356,7 @@ module API
         with_options format_with: -> (value) { value.to_i } do
           expose :storage_size
           expose :repository_size
+          expose :wiki_size
           expose :lfs_objects_size
           expose :build_artifacts_size, as: :job_artifacts_size
         end

lib/gitlab/background_migration/calculate_wiki_sizes.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class CalculateWikiSizes
+      def perform(start_id, stop_id)
+        ::ProjectStatistics.where(wiki_size: nil)
+          .where(id: start_id..stop_id)
+          .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics|
+          statistics.refresh!(only: [:wiki_size])
+        rescue => e
+          Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}"
+        end
+      end
+    end
+  end
+end

locale/gitlab.pot

@@ -139,7 +139,7 @@ msgstr ""
 msgid "%{commit_author_link} authored %{commit_timeago}"
 msgstr ""
 
-msgid "%{counter_repositories} repositories, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS"
+msgid "%{counter_repositories} repositories, %{counter_wikis} wikis, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS"
 msgstr ""
 
 msgid "%{count} approval required from %{name}"

spec/features/admin/admin_sees_project_statistics_spec.rb

@@ -15,7 +15,7 @@ describe "Admin > Admin sees project statistics" do
     let(:project) { create(:project, :repository) }
 
     it "shows project statistics" do
-      expect(page).to have_content("Storage: 0 Bytes (0 Bytes repositories, 0 Bytes build artifacts, 0 Bytes LFS)")
+      expect(page).to have_content("Storage: 0 Bytes (0 Bytes repositories, 0 Bytes wikis, 0 Bytes build artifacts, 0 Bytes LFS)")
     end
   end
 

spec/helpers/storage_helper_spec.rb

@@ -26,11 +26,12 @@ describe StorageHelper do
              namespace: namespace,
              statistics: build(:project_statistics,
                                repository_size:      10.kilobytes,
+                               wiki_size:            10.bytes,
                                lfs_objects_size:     20.gigabytes,
                                build_artifacts_size: 30.megabytes))
     end
 
-    let(:message) { '10 KB repositories, 30 MB build artifacts, 20 GB LFS' }
+    let(:message) { '10 KB repositories, 10 Bytes wikis, 30 MB build artifacts, 20 GB LFS' }
 
     it 'works on ProjectStatistics' do
       expect(helper.storage_counters_details(project.statistics)).to eq(message)

spec/lib/gitlab/background_migration/schedule_calculate_wiki_sizes_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20190527194900_schedule_calculate_wiki_sizes.rb')
+
+describe ScheduleCalculateWikiSizes, :migration, :sidekiq do
+  let(:migration_class) { Gitlab::BackgroundMigration::CalculateWikiSizes }
+  let(:migration_name)  { migration_class.to_s.demodulize }
+
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:project_statistics) { table(:project_statistics) }
+
+  context 'when missing wiki sizes exist' do
+    before do
+      namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration')
+      projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1)
+      projects.create!(id: 2, name: 'wiki-project-2', path: 'wiki-project-2', namespace_id: 1)
+      projects.create!(id: 3, name: 'wiki-project-3', path: 'wiki-project-3', namespace_id: 1)
+      project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000)
+      project_statistics.create!(id: 2, project_id: 2, namespace_id: 1, wiki_size: nil)
+      project_statistics.create!(id: 3, project_id: 3, namespace_id: 1, wiki_size: nil)
+    end
+
+    it 'schedules a background migration' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          expect(migration_name).to be_scheduled_delayed_migration(5.minutes, 2, 3)
+          expect(BackgroundMigrationWorker.jobs.size).to eq 1
+        end
+      end
+    end
+
+    it 'calculates missing wiki sizes' do
+      expect(project_statistics.find_by(id: 2).wiki_size).to be_nil
+      expect(project_statistics.find_by(id: 3).wiki_size).to be_nil
+
+      migrate!
+
+      expect(project_statistics.find_by(id: 2).wiki_size).not_to be_nil
+      expect(project_statistics.find_by(id: 3).wiki_size).not_to be_nil
+    end
+  end
+
+  context 'when missing wiki sizes do not exist' do
+    before do
+      namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration')
+      projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1)
+      project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000)
+    end
+
+    it 'does not schedule a background migration' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          expect(BackgroundMigrationWorker.jobs.size).to eq 0
+        end
+      end
+    end
+  end
+end

spec/models/namespace_spec.rb

@@ -147,6 +147,7 @@ describe Namespace do
              namespace: namespace,
              statistics: build(:project_statistics,
                                repository_size:      101,
+                               wiki_size:            505,
                                lfs_objects_size:     202,
                                build_artifacts_size: 303,
                                packages_size:        404))
@@ -157,6 +158,7 @@ describe Namespace do
              namespace: namespace,
              statistics: build(:project_statistics,
                                repository_size:      10,
+                               wiki_size:            50,
                                lfs_objects_size:     20,
                                build_artifacts_size: 30,
                                packages_size:        40))
@@ -167,8 +169,9 @@ describe Namespace do
       project2
       statistics = described_class.with_statistics.find(namespace.id)
 
-      expect(statistics.storage_size).to eq 1110
+      expect(statistics.storage_size).to eq 1665
       expect(statistics.repository_size).to eq 111
+      expect(statistics.wiki_size).to eq 555
       expect(statistics.lfs_objects_size).to eq 222
       expect(statistics.build_artifacts_size).to eq 333
       expect(statistics.packages_size).to eq 444
@@ -179,6 +182,7 @@ describe Namespace do
 
       expect(statistics.storage_size).to eq 0
       expect(statistics.repository_size).to eq 0
+      expect(statistics.wiki_size).to eq 0
       expect(statistics.lfs_objects_size).to eq 0
       expect(statistics.build_artifacts_size).to eq 0
       expect(statistics.packages_size).to eq 0

spec/models/project_statistics_spec.rb

@@ -16,16 +16,18 @@ describe ProjectStatistics do
       statistics.update!(
         commit_count: 8.exabytes - 1,
         repository_size: 2.exabytes,
+        wiki_size: 1.exabytes,
         lfs_objects_size: 2.exabytes,
-        build_artifacts_size: 4.exabytes - 1
+        build_artifacts_size: 3.exabytes - 1
       )
 
       statistics.reload
 
       expect(statistics.commit_count).to eq(8.exabytes - 1)
       expect(statistics.repository_size).to eq(2.exabytes)
+      expect(statistics.wiki_size).to eq(1.exabytes)
       expect(statistics.lfs_objects_size).to eq(2.exabytes)
-      expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
+      expect(statistics.build_artifacts_size).to eq(3.exabytes - 1)
       expect(statistics.storage_size).to eq(8.exabytes - 1)
     end
   end
@@ -33,6 +35,7 @@ describe ProjectStatistics do
   describe '#total_repository_size' do
     it "sums repository and LFS object size" do
       statistics.repository_size = 2
+      statistics.wiki_size = 6
       statistics.lfs_objects_size = 3
       statistics.build_artifacts_size = 4
 
@@ -40,10 +43,17 @@ describe ProjectStatistics do
     end
   end
 
+  describe '#wiki_size' do
+    it "is initialized with not null value" do
+      expect(statistics.wiki_size).to eq 0
+    end
+  end
+
   describe '#refresh!' do
     before do
       allow(statistics).to receive(:update_commit_count)
       allow(statistics).to receive(:update_repository_size)
+      allow(statistics).to receive(:update_wiki_size)
       allow(statistics).to receive(:update_lfs_objects_size)
       allow(statistics).to receive(:update_storage_size)
     end
@@ -56,6 +66,7 @@ describe ProjectStatistics do
       it "sums all counters" do
         expect(statistics).to have_received(:update_commit_count)
         expect(statistics).to have_received(:update_repository_size)
+        expect(statistics).to have_received(:update_wiki_size)
         expect(statistics).to have_received(:update_lfs_objects_size)
       end
     end
@@ -69,6 +80,45 @@ describe ProjectStatistics do
         expect(statistics).to have_received(:update_lfs_objects_size)
         expect(statistics).not_to have_received(:update_commit_count)
         expect(statistics).not_to have_received(:update_repository_size)
+        expect(statistics).not_to have_received(:update_wiki_size)
+      end
+    end
+
+    context 'without repositories' do
+      it 'does not crash' do
+        expect(project.repository.exists?).to be_falsey
+        expect(project.wiki.repository.exists?).to be_falsey
+
+        statistics.refresh!
+
+        expect(statistics).to have_received(:update_commit_count)
+        expect(statistics).to have_received(:update_repository_size)
+        expect(statistics).to have_received(:update_wiki_size)
+        expect(statistics.repository_size).to eq(0)
+        expect(statistics.commit_count).to eq(0)
+        expect(statistics.wiki_size).to eq(0)
+      end
+    end
+
+    context 'with deleted repositories' do
+      let(:project) { create(:project, :repository, :wiki_repo) }
+
+      before do
+        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          FileUtils.rm_rf(project.repository.path)
+          FileUtils.rm_rf(project.wiki.repository.path)
+        end
+      end
+
+      it 'does not crash' do
+        statistics.refresh!
+
+        expect(statistics).to have_received(:update_commit_count)
+        expect(statistics).to have_received(:update_repository_size)
+        expect(statistics).to have_received(:update_wiki_size)
+        expect(statistics.repository_size).to eq(0)
+        expect(statistics.commit_count).to eq(0)
+        expect(statistics.wiki_size).to eq(0)
       end
     end
   end
@@ -95,6 +145,17 @@ describe ProjectStatistics do
     end
   end
 
+  describe '#update_wiki_size' do
+    before do
+      allow(project.wiki.repository).to receive(:size).and_return(34)
+      statistics.update_wiki_size
+    end
+
+    it "stores the size of the wiki" do
+      expect(statistics.wiki_size).to eq 34.megabytes
+    end
+  end
+
   describe '#update_lfs_objects_size' do
     let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
     let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
@@ -114,12 +175,13 @@ describe ProjectStatistics do
     it "sums all storage counters" do
       statistics.update!(
         repository_size: 2,
+        wiki_size: 4,
         lfs_objects_size: 3
       )
 
       statistics.reload
 
-      expect(statistics.storage_size).to eq 5
+      expect(statistics.storage_size).to eq 9
     end
   end
 

spec/requests/api/groups_spec.rb

@@ -90,8 +90,9 @@ describe API::Groups do
 
       it "includes statistics if requested" do
         attributes = {
-          storage_size: 702,
+          storage_size: 1158,
           repository_size: 123,
+          wiki_size: 456,
           lfs_objects_size: 234,
           build_artifacts_size: 345
         }.stringify_keys

spec/services/projects/update_statistics_service_spec.rb

@@ -17,19 +17,9 @@ describe Projects::UpdateStatisticsService do
       end
     end
 
-    context 'with an existing project without a repository' do
+    context 'with an existing project' do
       let(:project) { create(:project) }
 
-      it 'does nothing' do
-        expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!)
-
-        service.execute
-      end
-    end
-
-    context 'with an existing project with a repository' do
-      let(:project) { create(:project, :repository) }
-
       it 'refreshes the project statistics' do
         expect_any_instance_of(ProjectStatistics).to receive(:refresh!)
           .with(only: statistics.map(&:to_sym))

spec/workers/project_cache_worker_spec.rb

@@ -25,10 +25,11 @@ describe ProjectCacheWorker do
     end
 
     context 'with an existing project without a repository' do
-      it 'does nothing' do
+      it 'updates statistics but does not refresh the method cashes' do
         allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
 
-        expect(worker).not_to receive(:update_statistics)
+        expect(worker).to receive(:update_statistics)
+        expect_any_instance_of(Repository).not_to receive(:refresh_method_caches)
 
         worker.perform(project.id)
       end