Merge branch 'feature/more-storage-statistics' into 'master'

Add more storage statistics

See merge request !7754

app/controllers/admin/groups_controller.rb

 class Admin::GroupsController < Admin::ApplicationController
-  before_action :group, only: [:edit, :show, :update, :destroy, :project_update, :members_update]
+  before_action :group, only: [:edit, :update, :destroy, :project_update, :members_update]
 
   def index
-    @groups = Group.all
+    @groups = Group.with_statistics
     @groups = @groups.sort(@sort = params[:sort])
     @groups = @groups.search(params[:name]) if params[:name].present?
     @groups = @groups.page(params[:page])
   end
 
   def show
+    @group = Group.with_statistics.find_by_full_path(params[:id])
     @members = @group.members.order("access_level DESC").page(params[:members_page])
     @requesters = AccessRequestsFinder.new(@group).execute(current_user)
-    @projects = @group.projects.page(params[:projects_page])
+    @projects = @group.projects.with_statistics.page(params[:projects_page])
   end
 
   def new

app/controllers/admin/projects_controller.rb

@@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController
   before_action :group, only: [:show, :transfer]
 
   def index
-    @projects = Project.all
+    @projects = Project.with_statistics
     @projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present?
     @projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present?
     @projects = @projects.with_push if params[:with_push].present?

app/controllers/groups_controller.rb

@@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController
   end
 
   def projects
-    @projects = @group.projects.page(params[:page])
+    @projects = @group.projects.with_statistics.page(params[:page])
   end
 
   def update

app/helpers/projects_helper.rb

@@ -246,11 +246,6 @@ module ProjectsHelper
     end
   end
 
-  def repository_size(project = @project)
-    size_in_bytes = project.repository_size * 1.megabyte
-    number_to_human_size(size_in_bytes, delimiter: ',', precision: 2)
-  end
-
   def default_url_to_repo(project = @project)
     case default_clone_protocol
     when 'ssh'
@@ -398,20 +393,6 @@ module ProjectsHelper
     [@project.path_with_namespace, sha, "readme"].join('-')
   end
 
-  def round_commit_count(project)
-    count = project.commit_count
-
-    if count > 10000
-      '10000+'
-    elsif count > 5000
-      '5000+'
-    elsif count > 1000
-      '1000+'
-    else
-      count
-    end
-  end
-
   def current_ref
     @ref || @repository.try(:root_ref)
   end

app/helpers/sorting_helper.rb

@@ -11,6 +11,7 @@ module SortingHelper
       sort_value_due_date_soon => sort_title_due_date_soon,
       sort_value_due_date_later => sort_title_due_date_later,
       sort_value_largest_repo => sort_title_largest_repo,
+      sort_value_largest_group => sort_title_largest_group,
       sort_value_recently_signin => sort_title_recently_signin,
       sort_value_oldest_signin => sort_title_oldest_signin,
       sort_value_downvotes => sort_title_downvotes,
@@ -92,6 +93,10 @@ module SortingHelper
     'Largest repository'
   end
 
+  def sort_title_largest_group
+    'Largest group'
+  end
+
   def sort_title_recently_signin
     'Recent sign in'
   end
@@ -193,7 +198,11 @@ module SortingHelper
   end
 
   def sort_value_largest_repo
-    'repository_size_desc'
+    'storage_size_desc'
+  end
+
+  def sort_value_largest_group
+    'storage_size_desc'
   end
 
   def sort_value_recently_signin

app/helpers/storage_helper.rb

+module StorageHelper
+  def storage_counter(size_in_bytes)
+    precision = size_in_bytes < 1.megabyte ? 0 : 1
+
+    number_to_human_size(size_in_bytes, delimiter: ',', precision: precision, significant: false)
+  end
+end

app/models/ci/build.rb

@@ -43,6 +43,8 @@ module Ci
     before_destroy { project }
 
     after_create :execute_hooks
+    after_save :update_project_statistics, if: :artifacts_size_changed?
+    after_destroy :update_project_statistics
 
     class << self
       def first_pending
@@ -584,5 +586,9 @@ module Ci
       Ci::MaskSecret.mask!(trace, token)
       trace
     end
+
+    def update_project_statistics
+      ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
+    end
   end
 end

app/models/group.rb

@@ -48,8 +48,14 @@ class Group < Namespace
     end
 
     def sort(method)
+      if method == 'storage_size_desc'
+        # storage_size is a virtual column so we need to
+        # pass a string to avoid AR adding the table name
+        reorder('storage_size DESC, namespaces.id DESC')
+      else
         order_by(method)
       end
+    end
 
     def reference_prefix
       User.reference_prefix

app/models/lfs_objects_project.rb

@@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base
   validates :lfs_object_id, presence: true
   validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" }
   validates :project_id, presence: true
+
+  after_create :update_project_statistics
+  after_destroy :update_project_statistics
+
+  private
+
+  def update_project_statistics
+    ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size])
+  end
 end

app/models/namespace.rb

@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base
   cache_markdown_field :description, pipeline: :description
 
   has_many :projects, dependent: :destroy
+  has_many :project_statistics
   belongs_to :owner, class_name: "User"
 
   belongs_to :parent, class_name: "Namespace"
@@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base
 
   scope :root, -> { where('type IS NULL') }
 
+  scope :with_statistics, -> do
+    joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id')
+      .group('namespaces.id')
+      .select(
+        'namespaces.*',
+        'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
+        'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
+        'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
+        'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
+      )
+  end
+
   class << self
     def by_path(path)
       find_by('lower(path) = :value', value: path.downcase)

app/models/project.rb

@@ -44,6 +44,7 @@ class Project < ActiveRecord::Base
   after_create :ensure_dir_exist
   after_create :create_project_feature, unless: :project_feature
   after_save :ensure_dir_exist, if: :namespace_id_changed?
+  after_save :update_project_statistics, if: :namespace_id_changed?
 
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
@@ -151,6 +152,7 @@ class Project < ActiveRecord::Base
 
   has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
   has_one :project_feature, dependent: :destroy
+  has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
 
   has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id
   has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
@@ -220,6 +222,7 @@ class Project < ActiveRecord::Base
   scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
 
   scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') }
+  scope :with_statistics, -> { includes(:statistics) }
 
   # "enabled" here means "not disabled". It includes private features!
   scope :with_feature_enabled, ->(feature) {
@@ -332,8 +335,10 @@ class Project < ActiveRecord::Base
     end
 
     def sort(method)
-      if method == 'repository_size_desc'
-        reorder(repository_size: :desc, id: :desc)
+      if method == 'storage_size_desc'
+        # storage_size is a joined column so we need to
+        # pass a string to avoid AR adding the table name
+        reorder('project_statistics.storage_size DESC, projects.id DESC')
       else
         order_by(method)
       end
@@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base
     forked? && project == forked_from_project
   end
 
-  def update_repository_size
-    update_attribute(:repository_size, repository.size)
-  end
-
-  def update_commit_count
-    update_attribute(:commit_count, repository.commit_count)
-  end
-
   def forks_count
     forks.count
   end
@@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base
   def full_path_changed?
     path_changed? || namespace_id_changed?
   end
+
+  def update_project_statistics
+    stats = statistics || build_statistics
+    stats.update(namespace_id: namespace_id)
+  end
 end

app/models/project_statistics.rb

+class ProjectStatistics < ActiveRecord::Base
+  belongs_to :project
+  belongs_to :namespace
+
+  before_save :update_storage_size
+
+  STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size]
+  STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS
+
+  def total_repository_size
+    repository_size + lfs_objects_size
+  end
+
+  def refresh!(only: nil)
+    STATISTICS_COLUMNS.each do |column, generator|
+      if only.blank? || only.include?(column)
+        public_send("update_#{column}")
+      end
+    end
+
+    save!
+  end
+
+  def update_commit_count
+    self.commit_count = project.repository.commit_count
+  end
+
+  def update_repository_size
+    self.repository_size = project.repository.size
+  end
+
+  def update_lfs_objects_size
+    self.lfs_objects_size = project.lfs_objects.sum(:size)
+  end
+
+  def update_build_artifacts_size
+    self.build_artifacts_size = project.builds.sum(:artifacts_size)
+  end
+
+  def update_storage_size
+    self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute))
+  end
+end

app/services/git_push_service.rb

@@ -77,7 +77,7 @@ class GitPushService < BaseService
       types = []
     end
 
-    ProjectCacheWorker.perform_async(@project.id, types)
+    ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
   end
 
   # Schedules processing of commit messages.

app/services/git_tag_push_service.rb

@@ -12,7 +12,7 @@ class GitTagPushService < BaseService
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
     Ci::CreatePipelineService.new(project, current_user, @push_data).execute
-    ProjectCacheWorker.perform_async(project.id)
+    ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
 
     true
   end

app/views/admin/groups/_group.html.haml

@@ -5,6 +5,9 @@
     = link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn'
     = link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove'
   .stats
+    %span.badge
+      = storage_counter(group.storage_size)
+
     %span
       = icon('bookmark')
       = number_with_delimiter(group.projects.count)

app/views/admin/groups/index.html.haml

@@ -27,6 +27,8 @@
                   = sort_title_recently_updated
                 = link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do
                   = sort_title_oldest_updated
+                = link_to admin_groups_path(sort: sort_value_largest_group, name: project_name) do
+                  = sort_title_largest_group
           = link_to new_admin_group_path, class: "btn btn-new" do
             New Group
   %ul.content-list

app/views/admin/groups/show.html.haml

@@ -38,6 +38,18 @@
           %strong
             = @group.created_at.to_s(:medium)
 
+        %li
+          %span.light Storage:
+          %strong= storage_counter(@group.storage_size)
+          (
+          = storage_counter(@group.repository_size)
+          repositories,
+          = storage_counter(@group.build_artifacts_size)
+          build artifacts,
+          = storage_counter(@group.lfs_objects_size)
+          LFS
+          )
+
         %li
           %span.light Group Git LFS status:
           %strong
@@ -55,8 +67,8 @@
           %li
             %strong
               = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
-              %span.label.label-gray
-                = repository_size(project)
+              %span.badge
+                = storage_counter(project.statistics.storage_size)
             %span.pull-right.light
               %span.monospace= project.path_with_namespace + ".git"
       .panel-footer
@@ -73,8 +85,8 @@
             %li
               %strong
                 = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
-                %span.label.label-gray
-                  = repository_size(project)
+                %span.badge
+                  = storage_counter(project.statistics.storage_size)
               %span.pull-right.light
                 %span.monospace= project.path_with_namespace + ".git"
 

app/views/admin/projects/index.html.haml

@@ -69,8 +69,8 @@
             .controls
               - if project.archived
                 %span.label.label-warning archived
-              %span.label.label-gray
-                = repository_size(project)
+              %span.badge
+                = storage_counter(project.statistics.storage_size)
               = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn"
               = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove"
             .title

app/views/admin/projects/show.html.haml

@@ -65,9 +65,16 @@
               = @project.repository.path_to_repo
 
           %li
-            %span.light Size
-            %strong
-              = repository_size(@project)
+            %span.light Storage:
+            %strong= storage_counter(@project.statistics.storage_size)
+            (
+            = storage_counter(@project.statistics.repository_size)
+            repository,
+            = storage_counter(@project.statistics.build_artifacts_size)
+            build artifacts,
+            = storage_counter(@project.statistics.lfs_objects_size)
+            LFS
+            )
 
           %li
             %span.light last commit:

app/views/groups/projects.html.haml

@@ -18,8 +18,8 @@
         .pull-right
           - if project.archived
             %span.label.label-warning archived
-          %span.label.label-gray
-            = repository_size(project)
+          %span.badge
+            = storage_counter(project.statistics.storage_size)
           = link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
           = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
           = link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove"

app/views/projects/show.html.haml

@@ -17,10 +17,10 @@
     %ul.nav
       %li
         = link_to project_files_path(@project) do
-          Files (#{repository_size})
+          Files (#{storage_counter(@project.statistics.total_repository_size)})
       %li
         = link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
-          #{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)})
+          #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
       %li
         = link_to namespace_project_branches_path(@project.namespace, @project) do
           #{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})

app/workers/project_cache_worker.rb

@@ -6,26 +6,27 @@ class ProjectCacheWorker
   LEASE_TIMEOUT = 15.minutes.to_i
 
   # project_id - The ID of the project for which to flush the cache.
-  # refresh - An Array containing extra types of data to refresh such as
+  # files - An Array containing extra types of files to refresh such as
   #         `:readme` to flush the README and `:changelog` to flush the
   #         CHANGELOG.
-  def perform(project_id, refresh = [])
+  # statistics - An Array containing columns from ProjectStatistics to
+  #              refresh, if empty all columns will be refreshed
+  def perform(project_id, files = [], statistics = [])
     project = Project.find_by(id: project_id)
 
     return unless project && project.repository.exists?
 
-    update_repository_size(project)
-    project.update_commit_count
+    update_statistics(project, statistics.map(&:to_sym))
 
-    project.repository.refresh_method_caches(refresh.map(&:to_sym))
+    project.repository.refresh_method_caches(files.map(&:to_sym))
   end
 
-  def update_repository_size(project)
-    return unless try_obtain_lease_for(project.id, :update_repository_size)
+  def update_statistics(project, statistics = [])
+    return unless try_obtain_lease_for(project.id, :update_statistics)
 
-    Rails.logger.info("Updating repository size for project #{project.id}")
+    Rails.logger.info("Updating statistics for project #{project.id}")
 
-    project.update_repository_size
+    project.statistics.refresh!(only: statistics)
   end
 
   private

changelogs/unreleased/feature-more-storage-statistics.yml

+---
+title: Add more storage statistics
+merge_request: 7754
+author: Markus Koller

config/initializers/inflections.rb

@@ -10,5 +10,5 @@
 # end
 #
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w(award_emoji)
+  inflect.uncountable %w(award_emoji project_statistics)
 end

db/migrate/20161201155511_create_project_statistics.rb

+class CreateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    # use bigint columns to support values >2GB
+    counter_column = { limit: 8, null: false, default: 0 }
+
+    create_table :project_statistics do |t|
+      t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.references :namespace, null: false, index: true
+      t.integer :commit_count, counter_column
+      t.integer :storage_size, counter_column
+      t.integer :repository_size, counter_column
+      t.integer :lfs_objects_size, counter_column
+      t.integer :build_artifacts_size, counter_column
+    end
+  end
+end

db/migrate/20161201160452_migrate_project_statistics.rb

+class MigrateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Removes two columns from the projects table'
+
+  def up
+    # convert repository_size in float (megabytes) to integer (bytes),
+    # initialize total storage_size with repository_size
+    execute <<-EOF
+      INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
+        SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
+    EOF
+
+    remove_column :projects, :repository_size
+    remove_column :projects, :commit_count
+  end
+
+  def down
+    add_column_with_default :projects, :repository_size, :float, default: 0.0
+    add_column_with_default :projects, :commit_count, :integer, default: 0
+  end
+end

db/schema.rb

@@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161221140236) do
 
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
+  create_table "project_statistics", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "namespace_id", null: false
+    t.integer "commit_count", limit: 8, default: 0, null: false
+    t.integer "storage_size", limit: 8, default: 0, null: false
+    t.integer "repository_size", limit: 8, default: 0, null: false
+    t.integer "lfs_objects_size", limit: 8, default: 0, null: false
+    t.integer "build_artifacts_size", limit: 8, default: 0, null: false
+  end
+
+  add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
+  add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
+
   create_table "projects", force: :cascade do |t|
     t.string "name"
     t.string "path"
@@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161221140236) do
     t.boolean "archived", default: false, null: false
     t.string "avatar"
     t.string "import_status"
-    t.float "repository_size", default: 0.0
     t.integer "star_count", default: 0, null: false
     t.string "import_type"
     t.string "import_source"
-    t.integer "commit_count", default: 0
     t.text "import_error"
     t.integer "ci_id"
     t.boolean "shared_runners_enabled", default: true, null: false
@@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161221140236) do
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
+  add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
   add_foreign_key "protected_branch_push_access_levels", "protected_branches"
   add_foreign_key "subscriptions", "projects", on_delete: :cascade

doc/administration/build_artifacts.md

@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo
 
 [reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
 [restart gitlab]: restart_gitlab.md "How to restart GitLab"
+
+## Storage statistics
+
+You can see the total storage used for build artifacts on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).

doc/api/groups.md

@@ -13,6 +13,7 @@ Parameters:
 | `search` | string | no | Return list of authorized groups matching the search criteria |
 | `order_by` | string | no | Order groups by `name` or `path`. Default is `name` |
 | `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` |
+| `statistics` | boolean | no | Include group statistics (admins only) |
 
 ```
 GET /groups
@@ -31,7 +32,6 @@ GET /groups
 
 You can search for groups by name or path, see below.
 
-=======
 ## List owned groups
 
 Get a list of groups which are owned by the authenticated user.
@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user.
 GET /groups/owned
 ```
 
+Parameters:
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `statistics` | boolean | no | Include group statistics |
+
 ## List a group's projects
 
 Get a list of projects in this group.

doc/api/projects.md

@@ -307,6 +307,8 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
+| `statistics` | boolean | no | Include project statistics |
 
 ### List starred projects
 
@@ -325,6 +327,7 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
 
 ### List ALL projects
 
@@ -343,6 +346,7 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `statistics` | boolean | no | Include project statistics |
 
 ### Get single project
 

doc/workflow/lfs/lfs_administration.md

@@ -40,6 +40,12 @@ In `config/gitlab.yml`:
     storage_path: /mnt/storage/lfs-objects
 ```
 
+## Storage statistics
+
+You can see the total storage used for LFS objects on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).
+
 ## Known limitations
 
 * Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
@@ -47,3 +53,5 @@ In `config/gitlab.yml`:
 * Currently, removing LFS objects from GitLab Git LFS storage is not supported
 * LFS authentications via SSH was added with GitLab 8.12
 * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
+* The storage statistics currently count each LFS object multiple times for
+  every project linking to it

lib/api/entities.rb

@@ -78,21 +78,21 @@ module API
       expose :container_registry_enabled
 
       # Expose old field names with the new permissions methods to keep API compatible
-      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) }
-      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) }
-      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) }
-      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) }
-      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
+      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) }
+      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) }
+      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) }
+      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
+      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
 
       expose :created_at, :last_activity_at
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace
+      expose :namespace, using: 'API::Entities::Namespace'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
       expose :avatar_url
       expose :star_count, :forks_count
-      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? }
+      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds
       expose :shared_with_groups do |project, options|
@@ -101,6 +101,16 @@ module API
       expose :only_allow_merge_if_build_succeeds
       expose :request_access_enabled
       expose :only_allow_merge_if_all_discussions_are_resolved
+
+      expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+    end
+
+    class ProjectStatistics < Grape::Entity
+      expose :commit_count
+      expose :storage_size
+      expose :repository_size
+      expose :lfs_objects_size
+      expose :build_artifacts_size
     end
 
     class Member < UserBasic
@@ -127,6 +137,15 @@ module API
       expose :avatar_url
       expose :web_url
       expose :request_access_enabled
+
+      expose :statistics, if: :statistics do
+        with_options format_with: -> (value) { value.to_i } do
+          expose :storage_size
+          expose :repository_size
+          expose :lfs_objects_size
+          expose :build_artifacts_size
+        end
+      end
     end
 
     class GroupDetail < Group
@@ -391,7 +410,7 @@ module API
     end
 
     class Namespace < Grape::Entity
-      expose :id, :path, :kind
+      expose :id, :name, :path, :kind
     end
 
     class MemberAccess < Grape::Entity
@@ -440,12 +459,12 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:user].id)
+          project.project_members.find_by(user_id: options[:current_user].id)
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:user].id)
+            project.group.group_members.find_by(user_id: options[:current_user].id)
           end
         end
       end

lib/api/groups.rb

@@ -11,6 +11,20 @@ module API
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
       end
+
+      params :statistics_params do
+        optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+      end
+
+      def present_groups(groups, options = {})
+        options = options.reverse_merge(
+          with: Entities::Group,
+          current_user: current_user,
+        )
+
+        groups = groups.with_statistics if options[:statistics]
+        present paginate(groups), options
+      end
     end
 
     resource :groups do
@@ -18,6 +32,7 @@ module API
         success Entities::Group
       end
       params do
+        use :statistics_params
         optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
         optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
         optional :search, type: String, desc: 'Search for a specific group'
@@ -38,7 +53,7 @@ module API
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         groups = groups.reorder(params[:order_by] => params[:sort])
 
-        present paginate(groups), with: Entities::Group
+        present_groups groups, statistics: params[:statistics] && current_user.is_admin?
       end
 
       desc 'Get list of owned groups for authenticated user' do
@@ -46,10 +61,10 @@ module API
       end
       params do
         use :pagination
+        use :statistics_params
       end
       get '/owned' do
-        groups = current_user.owned_groups
-        present paginate(groups), with: Entities::Group, user: current_user
+        present_groups current_user.owned_groups, statistics: params[:statistics]
       end
 
       desc 'Create a group. Available only for users who can create groups.' do
@@ -66,7 +81,7 @@ module API
         group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute
 
         if group.persisted?
-          present group, with: Entities::Group
+          present group, with: Entities::Group, current_user: current_user
         else
           render_api_error!("Failed to save group #{group.errors.messages}", 400)
         end
@@ -92,7 +107,7 @@ module API
         authorize! :admin_group, group
 
         if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute
-          present group, with: Entities::GroupDetail
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_validation_error!(group)
         end
@@ -103,7 +118,7 @@ module API
       end
       get ":id" do
         group = find_group!(params[:id])
-        present group, with: Entities::GroupDetail
+        present group, with: Entities::GroupDetail, current_user: current_user
       end
 
       desc 'Remove a group.'
@@ -134,7 +149,7 @@ module API
         projects = GroupProjectsFinder.new(group).execute(current_user)
         projects = filter_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
-        present paginate(projects), with: entity, user: current_user
+        present paginate(projects), with: entity, current_user: current_user
       end
 
       desc 'Transfer a project to the group namespace. Available only for admin.' do
@@ -150,7 +165,7 @@ module API
         result = ::Projects::TransferService.new(project, current_user).execute(group)
 
         if result
-          present group, with: Entities::GroupDetail
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
         end

lib/api/helpers.rb

@@ -248,7 +248,7 @@ module API
       rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
     end
 
-    # Projects helpers
+    # project helpers
 
     def filter_projects(projects)
       if params[:search].present?

lib/api/projects.rb

@@ -40,6 +40,15 @@ module API
 
     resource :projects do
       helpers do
+        params :collection_params do
+          use :sort_params
+          use :filter_params
+          use :pagination
+
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+        end
+
         params :sort_params do
           optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
                               default: 'created_at', desc: 'Return projects ordered by field'
@@ -52,97 +61,94 @@ module API
           optional :visibility, type: String, values: %w[public internal private],
                                 desc: 'Limit by visibility'
           optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
-          use :sort_params
+        end
+
+        params :statistics_params do
+          optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
         end
 
         params :create_params do
           optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
+
+        def present_projects(projects, options = {})
+          options = options.reverse_merge(
+            with: Entities::Project,
+            current_user: current_user,
+            simple: params[:simple],
+          )
+
+          projects = filter_projects(projects)
+          projects = projects.with_statistics if options[:statistics]
+          options[:with] = Entities::BasicProjectDetails if options[:simple]
+
+          present paginate(projects), options
+        end
       end
 
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/visible' do
-        projects = ProjectsFinder.new.execute(current_user)
-        projects = filter_projects(projects)
-        entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, user: current_user
+        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+        present_projects ProjectsFinder.new.execute(current_user), with: entity
       end
 
       desc 'Get a projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get do
         authenticate!
 
-        projects = current_user.authorized_projects
-        projects = filter_projects(projects)
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, user: current_user
+        present_projects current_user.authorized_projects,
+          with: Entities::ProjectWithAccess
       end
 
       desc 'Get an owned projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/owned' do
         authenticate!
 
-        projects = current_user.owned_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+        present_projects current_user.owned_projects,
+          with: Entities::ProjectWithAccess,
+          statistics: params[:statistics]
       end
 
       desc 'Gets starred project for the authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/starred' do
         authenticate!
 
-        projects = current_user.viewable_starred_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::Project, user: current_user
+        present_projects current_user.viewable_starred_projects
       end
 
       desc 'Get all projects for admin user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/all' do
         authenticated_as_admin!
 
-        projects = Project.all
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+        present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
       end
 
       desc 'Search for projects the current user has access to' do
@@ -221,7 +227,7 @@ module API
       end
       get ":id" do
         entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present user_project, with: entity, user: current_user,
+        present user_project, with: entity, current_user: current_user,
                               user_can_admin_project: can?(current_user, :admin_project, user_project)
       end
 

lib/tasks/gitlab/import.rake

@@ -63,8 +63,7 @@ namespace :gitlab do
 
             if project.persisted?
               puts " * Created #{project.name} (#{repo_path})".color(:green)
-              project.update_repository_size
-              project.update_commit_count
+              ProjectCacheWorker.perform(project.id)
             else
               puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
               puts "   Errors: #{project.errors.messages}".color(:red)

lib/tasks/gitlab/update_commit_count.rake

-namespace :gitlab do
-  desc "GitLab | Update commit count for projects"
-  task update_commit_count: :environment do
-    projects = Project.where(commit_count: 0)
-    puts "#{projects.size} projects need to be updated. This might take a while."
-    ask_to_continue unless ENV['force'] == 'yes'
-
-    projects.find_each(batch_size: 100) do |project|
-      print "#{project.name_with_namespace.color(:yellow)} ... "
-
-      unless project.repo_exists?
-        puts "skipping, because the repo is empty".color(:magenta)
-        next
-      end
-
-      project.update_commit_count
-      puts project.commit_count.to_s.color(:green)
-    end
-  end
-end

spec/factories/lfs_objects.rb

@@ -2,7 +2,7 @@ include ActionDispatch::TestProcess
 
 FactoryGirl.define do
   factory :lfs_object do
-    oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
+    sequence(:oid) { |n| "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a%05x" % n }
     size 499013
   end
 

spec/factories/project_statistics.rb

+FactoryGirl.define do
+  factory :project_statistics do
+    project { create :project }
+    namespace { project.namespace }
+  end
+end

spec/helpers/storage_helper_spec.rb

+require 'spec_helper'
+
+describe StorageHelper do
+  describe '#storage_counter' do
+    it 'formats bytes to one decimal place' do
+      expect(helper.storage_counter(1.23.megabytes)).to eq '1.2 MB'
+    end
+
+    it 'does not add decimals for sizes < 1 MB' do
+      expect(helper.storage_counter(23.5.kilobytes)).to eq '24 KB'
+    end
+
+    it 'does not add decimals for zeroes' do
+      expect(helper.storage_counter(2.megabytes)).to eq '2 MB'
+    end
+
+    it 'uses commas as thousands separator' do
+      expect(helper.storage_counter(100_000_000_000_000_000)).to eq '90,949.5 TB'
+    end
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -192,6 +192,7 @@ project:
 - authorized_users
 - project_authorizations
 - route
+- statistics
 award_emoji:
 - awardable
 - user

spec/models/ci/build_spec.rb

@@ -85,4 +85,30 @@ describe Ci::Build, models: true do
       it { expect(build.trace_file_path).to eq(build.old_path_to_trace) }
     end
   end
+
+  describe '#update_project_statistics' do
+    let!(:build) { create(:ci_build, artifacts_size: 23) }
+
+    it 'updates project statistics when the artifact size changes' do
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(build.project_id, [], [:build_artifacts_size])
+
+      build.artifacts_size = 42
+      build.save!
+    end
+
+    it 'does not update project statistics when the artifact size stays the same' do
+      expect(ProjectCacheWorker).not_to receive(:perform_async)
+
+      build.name = 'changed'
+      build.save!
+    end
+
+    it 'updates project statistics when the build is destroyed' do
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(build.project_id, [], [:build_artifacts_size])
+
+      build.destroy
+    end
+  end
 end

spec/models/lfs_objects_project_spec.rb

+require 'spec_helper'
+
+describe LfsObjectsProject, models: true do
+  subject { create(:lfs_objects_project, project: project) }
+  let(:project) { create(:empty_project) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to belong_to(:lfs_object) }
+  end
+
+  describe 'validation' do
+    it { is_expected.to validate_presence_of(:lfs_object_id) }
+    it { is_expected.to validate_uniqueness_of(:lfs_object_id).scoped_to(:project_id).with_message("already exists in project") }
+
+    it { is_expected.to validate_presence_of(:project_id) }
+  end
+
+  describe '#update_project_statistics' do
+    it 'updates project statistics when the object is added' do
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(project.id, [], [:lfs_objects_size])
+
+      subject.save!
+    end
+
+    it 'updates project statistics when the object is removed' do
+      subject.save!
+
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(project.id, [], [:lfs_objects_size])
+
+      subject.destroy
+    end
+  end
+end

spec/models/namespace_spec.rb

@@ -4,6 +4,7 @@ describe Namespace, models: true do
   let!(:namespace) { create(:namespace) }
 
   it { is_expected.to have_many :projects }
+  it { is_expected.to have_many :project_statistics }
 
   it { is_expected.to validate_presence_of(:name) }
   it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) }
@@ -57,6 +58,50 @@ describe Namespace, models: true do
     end
   end
 
+  describe '.with_statistics' do
+    let(:namespace) { create :namespace }
+
+    let(:project1) do
+      create(:empty_project,
+             namespace: namespace,
+             statistics: build(:project_statistics,
+                               storage_size:         606,
+                               repository_size:      101,
+                               lfs_objects_size:     202,
+                               build_artifacts_size: 303))
+    end
+
+    let(:project2) do
+      create(:empty_project,
+             namespace: namespace,
+             statistics: build(:project_statistics,
+                               storage_size:         60,
+                               repository_size:      10,
+                               lfs_objects_size:     20,
+                               build_artifacts_size: 30))
+    end
+
+    it "sums all project storage counters in the namespace" do
+      project1
+      project2
+      statistics = Namespace.with_statistics.find(namespace.id)
+
+      expect(statistics.storage_size).to eq 666
+      expect(statistics.repository_size).to eq 111
+      expect(statistics.lfs_objects_size).to eq 222
+      expect(statistics.build_artifacts_size).to eq 333
+    end
+
+    it "correctly handles namespaces without projects" do
+      statistics = Namespace.with_statistics.find(namespace.id)
+
+      expect(statistics.storage_size).to eq 0
+      expect(statistics.repository_size).to eq 0
+      expect(statistics.lfs_objects_size).to eq 0
+      expect(statistics.build_artifacts_size).to eq 0
+    end
+  end
+
   describe '#move_dir' do
     before do
       @namespace = create :namespace

spec/models/project_spec.rb

@@ -49,6 +49,7 @@ describe Project, models: true do
     it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) }
     it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
     it { is_expected.to have_one(:project_feature).dependent(:destroy) }
+    it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
     it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) }
     it { is_expected.to have_one(:last_event).class_name('Event') }
     it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
@@ -1729,6 +1730,26 @@ describe Project, models: true do
     end
   end
 
+  describe '#update_project_statistics' do
+    let(:project) { create(:empty_project) }
+
+    it "is called after creation" do
+      expect(project.statistics).to be_a ProjectStatistics
+      expect(project.statistics).to be_persisted
+    end
+
+    it "copies the namespace_id" do
+      expect(project.statistics.namespace_id).to eq project.namespace_id
+    end
+
+    it "updates the namespace_id when changed" do
+      namespace = create(:namespace)
+      project.update(namespace: namespace)
+
+      expect(project.statistics.namespace_id).to eq namespace.id
+    end
+  end
+
   def enable_lfs
     allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
   end

spec/models/project_statistics_spec.rb

+require 'rails_helper'
+
+describe ProjectStatistics, models: true do
+  let(:project) { create :empty_project }
+  let(:statistics) { project.statistics }
+
+  describe 'constants' do
+    describe 'STORAGE_COLUMNS' do
+      it 'is an array of symbols' do
+        expect(described_class::STORAGE_COLUMNS).to be_kind_of Array
+        expect(described_class::STORAGE_COLUMNS.map(&:class).uniq).to eq [Symbol]
+      end
+    end
+
+    describe 'STATISTICS_COLUMNS' do
+      it 'is an array of symbols' do
+        expect(described_class::STATISTICS_COLUMNS).to be_kind_of Array
+        expect(described_class::STATISTICS_COLUMNS.map(&:class).uniq).to eq [Symbol]
+      end
+
+      it 'includes all storage columns' do
+        expect(described_class::STATISTICS_COLUMNS & described_class::STORAGE_COLUMNS).to eq described_class::STORAGE_COLUMNS
+      end
+    end
+  end
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to belong_to(:namespace) }
+  end
+
+  describe 'statistics columns' do
+    it "support values up to 8 exabytes" do
+      statistics.update!(
+        commit_count: 8.exabytes - 1,
+        repository_size: 2.exabytes,
+        lfs_objects_size: 2.exabytes,
+        build_artifacts_size: 4.exabytes - 1,
+      )
+
+      statistics.reload
+
+      expect(statistics.commit_count).to eq(8.exabytes - 1)
+      expect(statistics.repository_size).to eq(2.exabytes)
+      expect(statistics.lfs_objects_size).to eq(2.exabytes)
+      expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
+      expect(statistics.storage_size).to eq(8.exabytes - 1)
+    end
+  end
+
+  describe '#total_repository_size' do
+    it "sums repository and LFS object size" do
+      statistics.repository_size = 2
+      statistics.lfs_objects_size = 3
+      statistics.build_artifacts_size = 4
+
+      expect(statistics.total_repository_size).to eq 5
+    end
+  end
+
+  describe '#refresh!' do
+    before do
+      allow(statistics).to receive(:update_commit_count)
+      allow(statistics).to receive(:update_repository_size)
+      allow(statistics).to receive(:update_lfs_objects_size)
+      allow(statistics).to receive(:update_build_artifacts_size)
+      allow(statistics).to receive(:update_storage_size)
+    end
+
+    context "without arguments" do
+      before do
+        statistics.refresh!
+      end
+
+      it "sums all counters" do
+        expect(statistics).to have_received(:update_commit_count)
+        expect(statistics).to have_received(:update_repository_size)
+        expect(statistics).to have_received(:update_lfs_objects_size)
+        expect(statistics).to have_received(:update_build_artifacts_size)
+      end
+    end
+
+    context "when passing an only: argument" do
+      before do
+        statistics.refresh! only: [:lfs_objects_size]
+      end
+
+      it "only updates the given columns" do
+        expect(statistics).to have_received(:update_lfs_objects_size)
+        expect(statistics).not_to have_received(:update_commit_count)
+        expect(statistics).not_to have_received(:update_repository_size)
+        expect(statistics).not_to have_received(:update_build_artifacts_size)
+      end
+    end
+  end
+
+  describe '#update_commit_count' do
+    before do
+      allow(project.repository).to receive(:commit_count).and_return(23)
+      statistics.update_commit_count
+    end
+
+    it "stores the number of commits in the repository" do
+      expect(statistics.commit_count).to eq 23
+    end
+  end
+
+  describe '#update_repository_size' do
+    before do
+      allow(project.repository).to receive(:size).and_return(12.megabytes)
+      statistics.update_repository_size
+    end
+
+    it "stores the size of the repository" do
+      expect(statistics.repository_size).to eq 12.megabytes
+    end
+  end
+
+  describe '#update_lfs_objects_size' do
+    let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
+    let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
+    let!(:lfs_objects_project1) { create(:lfs_objects_project, project: project, lfs_object: lfs_object1) }
+    let!(:lfs_objects_project2) { create(:lfs_objects_project, project: project, lfs_object: lfs_object2) }
+
+    before do
+      statistics.update_lfs_objects_size
+    end
+
+    it "stores the size of related LFS objects" do
+      expect(statistics.lfs_objects_size).to eq 57.megabytes
+    end
+  end
+
+  describe '#update_build_artifacts_size' do
+    let!(:pipeline) { create(:ci_pipeline, project: project) }
+    let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) }
+    let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) }
+
+    before do
+      statistics.update_build_artifacts_size
+    end
+
+    it "stores the size of related build artifacts" do
+      expect(statistics.build_artifacts_size).to eq 101.megabytes
+    end
+  end
+
+  describe '#update_storage_size' do
+    it "sums all storage counters" do
+      statistics.update!(
+        repository_size: 2,
+        lfs_objects_size: 3,
+      )
+
+      statistics.reload
+
+      expect(statistics.storage_size).to eq 5
+    end
+  end
+end

spec/requests/api/groups_spec.rb

@@ -35,6 +35,14 @@ describe API::Groups, api: true  do
         expect(json_response.length).to eq(1)
         expect(json_response.first['name']).to eq(group1.name)
       end
+
+      it "does not include statistics" do
+        get api("/groups", user1), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include 'statistics'
+      end
     end
 
     context "when authenticated as admin" do
@@ -44,6 +52,31 @@ describe API::Groups, api: true  do
         expect(json_response).to be_an Array
         expect(json_response.length).to eq(2)
       end
+
+      it "does not include statistics by default" do
+        get api("/groups", admin)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include('statistics')
+      end
+
+      it "includes statistics if requested" do
+        attributes = {
+          storage_size: 702,
+          repository_size: 123,
+          lfs_objects_size: 234,
+          build_artifacts_size: 345,
+        }
+
+        project1.statistics.update!(attributes)
+
+        get api("/groups", admin), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first['statistics']).to eq attributes.stringify_keys
+      end
     end
 
     context "when using skip_groups in request" do

spec/requests/api/projects_spec.rb

@@ -49,7 +49,7 @@ describe API::Projects, api: true  do
       end
     end
 
-    context 'when authenticated' do
+    context 'when authenticated as regular user' do
       it 'returns an array of projects' do
         get api('/projects', user)
         expect(response).to have_http_status(200)
@@ -172,6 +172,22 @@ describe API::Projects, api: true  do
           end
         end
       end
+
+      it "does not include statistics by default" do
+        get api('/projects/all', admin)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include('statistics')
+      end
+
+      it "includes statistics if requested" do
+        get api('/projects/all', admin), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).to include 'statistics'
+      end
     end
   end
 
@@ -196,6 +212,32 @@ describe API::Projects, api: true  do
         expect(json_response.first['name']).to eq(project4.name)
         expect(json_response.first['owner']['username']).to eq(user4.username)
       end
+
+      it "does not include statistics by default" do
+        get api('/projects/owned', user4)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include('statistics')
+      end
+
+      it "includes statistics if requested" do
+        attributes = {
+          commit_count: 23,
+          storage_size: 702,
+          repository_size: 123,
+          lfs_objects_size: 234,
+          build_artifacts_size: 345,
+        }
+
+        project4.statistics.update!(attributes)
+
+        get api('/projects/owned', user4), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first['statistics']).to eq attributes.stringify_keys
+      end
     end
   end
 
@@ -630,6 +672,18 @@ describe API::Projects, api: true  do
         expect(json_response['name']).to eq(project.name)
       end
 
+      it 'exposes namespace fields' do
+        get api("/projects/#{project.id}", user)
+
+        expect(response).to have_http_status(200)
+        expect(json_response['namespace']).to eq({
+          'id' => user.namespace.id,
+          'name' => user.namespace.name,
+          'path' => user.namespace.path,
+          'kind' => user.namespace.kind,
+        })
+      end
+
       describe 'permissions' do
         context 'all projects' do
           before { project.team << [user, :master] }

spec/services/git_push_service_spec.rb

@@ -583,7 +583,7 @@ describe GitPushService, services: true do
         service.push_commits = [commit]
 
         expect(ProjectCacheWorker).to receive(:perform_async).
-          with(project.id, %i(readme))
+          with(project.id, %i(readme), %i(commit_count repository_size))
 
         service.update_caches
       end
@@ -596,7 +596,7 @@ describe GitPushService, services: true do
 
       it 'does not flush any conditional caches' do
         expect(ProjectCacheWorker).to receive(:perform_async).
-          with(project.id, []).
+          with(project.id, [], %i(commit_count repository_size)).
           and_call_original
 
         service.update_caches

spec/workers/project_cache_worker_spec.rb

 require 'spec_helper'
 
 describe ProjectCacheWorker do
-  let(:project) { create(:project) }
   let(:worker) { described_class.new }
+  let(:project) { create(:project) }
+  let(:statistics) { project.statistics }
 
   describe '#perform' do
     before do
@@ -12,7 +13,7 @@ describe ProjectCacheWorker do
 
     context 'with a non-existing project' do
       it 'does nothing' do
-        expect(worker).not_to receive(:update_repository_size)
+        expect(worker).not_to receive(:update_statistics)
 
         worker.perform(-1)
       end
@@ -22,24 +23,19 @@ describe ProjectCacheWorker do
       it 'does nothing' do
         allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
 
-        expect(worker).not_to receive(:update_repository_size)
+        expect(worker).not_to receive(:update_statistics)
 
         worker.perform(project.id)
       end
     end
 
     context 'with an existing project' do
-      it 'updates the repository size' do
-        expect(worker).to receive(:update_repository_size).and_call_original
-
-        worker.perform(project.id)
-      end
-
-      it 'updates the commit count' do
-        expect_any_instance_of(Project).to receive(:update_commit_count).
-          and_call_original
+      it 'updates the project statistics' do
+        expect(worker).to receive(:update_statistics)
+          .with(kind_of(Project), %i(repository_size))
+          .and_call_original
 
-        worker.perform(project.id)
+        worker.perform(project.id, [], %w(repository_size))
       end
 
       it 'refreshes the method caches' do
@@ -47,33 +43,35 @@ describe ProjectCacheWorker do
           with(%i(readme)).
           and_call_original
 
-        worker.perform(project.id, %i(readme))
+        worker.perform(project.id, %w(readme))
       end
     end
   end
 
-  describe '#update_repository_size' do
+  describe '#update_statistics' do
     context 'when a lease could not be obtained' do
       it 'does not update the repository size' do
         allow(worker).to receive(:try_obtain_lease_for).
-          with(project.id, :update_repository_size).
+          with(project.id, :update_statistics).
           and_return(false)
 
-        expect(project).not_to receive(:update_repository_size)
+        expect(statistics).not_to receive(:refresh!)
 
-        worker.update_repository_size(project)
+        worker.update_statistics(project)
       end
     end
 
     context 'when a lease could be obtained' do
-      it 'updates the repository size' do
+      it 'updates the project statistics' do
         allow(worker).to receive(:try_obtain_lease_for).
-          with(project.id, :update_repository_size).
+          with(project.id, :update_statistics).
           and_return(true)
 
-        expect(project).to receive(:update_repository_size).and_call_original
+        expect(statistics).to receive(:refresh!)
+          .with(only: %i(repository_size))
+          .and_call_original
 
-        worker.update_repository_size(project)
+        worker.update_statistics(project, %i(repository_size))
       end
     end
   end