Merge branch 'storage-statistics-api' into 'master'

storage-statistics API project/group, add missing fields

See merge request gitlab-org/gitlab!72075

app/models/namespace.rb

@@ -140,6 +140,7 @@ class Namespace < ApplicationRecord
         'COALESCE(SUM(ps.snippets_size), 0) AS snippets_size',
         'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
         'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
+        'COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size',
         'COALESCE(SUM(ps.packages_size), 0) AS packages_size',
         'COALESCE(SUM(ps.uploads_size), 0) AS uploads_size'
       )

doc/api/groups.md

@@ -100,13 +100,15 @@ GET /groups?statistics=true
     "parent_id": null,
     "created_at": "2020-01-15T12:36:29.590Z",
     "statistics": {
-      "storage_size" : 363,
-      "repository_size" : 33,
-      "wiki_size" : 100,
-      "lfs_objects_size" : 123,
-      "job_artifacts_size" : 57,
+      "storage_size": 363,
+      "repository_size": 33,
+      "wiki_size": 100,
+      "lfs_objects_size": 123,
+      "job_artifacts_size": 57,
+      "pipeline_artifacts_size": 0,
       "packages_size": 0,
-      "snippets_size" : 50
+      "snippets_size": 50,
+      "uploads_size": 0
     }
   }
 ]

doc/api/projects.md

@@ -191,8 +191,10 @@ When the user is authenticated and `simple` is not set this returns something li
       "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
+      "pipeline_artifacts_size": 0,
       "packages_size": 0,
-      "snippets_size": 0
+      "snippets_size": 0,
+      "uploads_size": 0
     },
     "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-client",
     "_links": {
@@ -303,8 +305,10 @@ When the user is authenticated and `simple` is not set this returns something li
       "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
+      "pipeline_artifacts_size": 0,
       "packages_size": 0,
-      "snippets_size": 0
+      "snippets_size": 0,
+      "uploads_size": 0
     },
     "container_registry_image_prefix": "registry.example.com/brightbox/puppet",
     "_links": {
@@ -469,8 +473,10 @@ GET /users/:user_id/projects
       "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
+      "pipeline_artifacts_size": 0,
       "packages_size": 0,
-      "snippets_size": 0
+      "snippets_size": 0,
+      "uploads_size": 0
     },
     "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-client",
     "_links": {
@@ -581,8 +587,10 @@ GET /users/:user_id/projects
       "wiki_size" : 0,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0,
+      "pipeline_artifacts_size": 0,
       "packages_size": 0,
-      "snippets_size": 0
+      "snippets_size": 0,
+      "uploads_size": 0
     },
     "container_registry_image_prefix": "registry.example.com/brightbox/puppet",
     "_links": {
@@ -704,6 +712,10 @@ Example response:
       "repository_size": 1038090,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0
+      "pipeline_artifacts_size": 0,
+      "packages_size": 0,
+      "snippets_size": 0,
+      "uploads_size": 0
     },
     "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-client",
     "_links": {
@@ -811,6 +823,10 @@ Example response:
       "repository_size": 2066080,
       "lfs_objects_size": 0,
       "job_artifacts_size": 0
+      "pipeline_artifacts_size": 0,
+      "packages_size": 0,
+      "snippets_size": 0,
+      "uploads_size": 0
     },
     "container_registry_image_prefix": "registry.example.com/brightbox/puppet",
     "_links": {
@@ -978,8 +994,10 @@ GET /projects/:id
     "wiki_size" : 0,
     "lfs_objects_size": 0,
     "job_artifacts_size": 0,
+    "pipeline_artifacts_size": 0,
     "packages_size": 0,
-    "snippets_size": 0
+    "snippets_size": 0,
+    "uploads_size": 0
   },
   "container_registry_image_prefix": "registry.example.com/diaspora/diaspora-client",
   "_links": {

doc/development/namespaces_storage_statistics.md

@@ -53,7 +53,10 @@ SELECT split_part("rs".path, '/', 1) as root_path,
         COALESCE(SUM(ps.wiki_size), 0) AS wiki_size,
         COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size,
         COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size,
-        COALESCE(SUM(ps.packages_size), 0) AS packages_size
+        COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size,
+        COALESCE(SUM(ps.packages_size), 0) AS packages_size,
+        COALESCE(SUM(ps.snippets_size), 0) AS snippets_size,
+        COALESCE(SUM(ps.uploads_size), 0) AS uploads_size
 FROM "projects"
     INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project'
     INNER JOIN project_statistics ps ON ps.project_id  = projects.id
@@ -83,7 +86,10 @@ WITH refresh AS (
         COALESCE(SUM(ps.wiki_size), 0) AS wiki_size,
         COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size,
         COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size,
-        COALESCE(SUM(ps.packages_size), 0) AS packages_size
+        COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size,
+        COALESCE(SUM(ps.packages_size), 0) AS packages_size,
+        COALESCE(SUM(ps.snippets_size), 0) AS snippets_size,
+        COALESCE(SUM(ps.uploads_size), 0) AS uploads_size
   FROM "projects"
         INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project'
         INNER JOIN project_statistics ps ON ps.project_id  = projects.id
@@ -94,7 +100,10 @@ SET storage_size = refresh.storage_size,
     wiki_size = refresh.wiki_size,
     lfs_objects_size = refresh.lfs_objects_size,
     build_artifacts_size = refresh.build_artifacts_size,
-    packages_size  = refresh.packages_size
+    pipeline_artifacts_size = refresh.pipeline_artifacts_size,
+    packages_size  = refresh.packages_size,
+    snippets_size  = refresh.snippets_size,
+    uploads_size  = refresh.uploads_size
 FROM refresh
     INNER JOIN routes rs ON rs.path = refresh.root_path AND rs.source_type = 'Namespace'
 WHERE namespace_storage_statistics.namespace_id = rs.source_id

lib/api/entities/group.rb

@@ -31,7 +31,10 @@ module API
           expose :wiki_size
           expose :lfs_objects_size
           expose :build_artifacts_size, as: :job_artifacts_size
+          expose :pipeline_artifacts_size
+          expose :packages_size
           expose :snippets_size
+          expose :uploads_size
         end
       end
     end

lib/api/entities/project_statistics.rb

@@ -9,8 +9,10 @@ module API
       expose :wiki_size
       expose :lfs_objects_size
       expose :build_artifacts_size, as: :job_artifacts_size
-      expose :snippets_size
+      expose :pipeline_artifacts_size
       expose :packages_size
+      expose :snippets_size
+      expose :uploads_size
     end
   end
 end

spec/models/namespace_spec.rb

@@ -568,26 +568,30 @@ RSpec.describe Namespace do
       create(:project,
              namespace: namespace,
              statistics: build(:project_statistics,
-                               namespace:            namespace,
-                               repository_size:      101,
-                               wiki_size:            505,
-                               lfs_objects_size:     202,
-                               build_artifacts_size: 303,
-                               packages_size:        404,
-                               snippets_size:        605))
+                               namespace:               namespace,
+                               repository_size:         101,
+                               wiki_size:               505,
+                               lfs_objects_size:        202,
+                               build_artifacts_size:    303,
+                               pipeline_artifacts_size: 707,
+                               packages_size:           404,
+                               snippets_size:           605,
+                               uploads_size:            808))
     end
 
     let(:project2) do
       create(:project,
              namespace: namespace,
              statistics: build(:project_statistics,
-                               namespace:            namespace,
-                               repository_size:      10,
-                               wiki_size:            50,
-                               lfs_objects_size:     20,
-                               build_artifacts_size: 30,
-                               packages_size:        40,
-                               snippets_size:        60))
+                               namespace:               namespace,
+                               repository_size:         10,
+                               wiki_size:               50,
+                               lfs_objects_size:        20,
+                               build_artifacts_size:    30,
+                               pipeline_artifacts_size: 70,
+                               packages_size:           40,
+                               snippets_size:           60,
+                               uploads_size:            80))
     end
 
     it "sums all project storage counters in the namespace" do
@@ -595,13 +599,15 @@ RSpec.describe Namespace do
       project2
       statistics = described_class.with_statistics.find(namespace.id)
 
-      expect(statistics.storage_size).to eq 2330
+      expect(statistics.storage_size).to eq 3995
       expect(statistics.repository_size).to eq 111
       expect(statistics.wiki_size).to eq 555
       expect(statistics.lfs_objects_size).to eq 222
       expect(statistics.build_artifacts_size).to eq 333
+      expect(statistics.pipeline_artifacts_size).to eq 777
       expect(statistics.packages_size).to eq 444
       expect(statistics.snippets_size).to eq 665
+      expect(statistics.uploads_size).to eq 888
     end
 
     it "correctly handles namespaces without projects" do
@@ -612,8 +618,10 @@ RSpec.describe Namespace do
       expect(statistics.wiki_size).to eq 0
       expect(statistics.lfs_objects_size).to eq 0
       expect(statistics.build_artifacts_size).to eq 0
+      expect(statistics.pipeline_artifacts_size).to eq 0
       expect(statistics.packages_size).to eq 0
       expect(statistics.snippets_size).to eq 0
+      expect(statistics.uploads_size).to eq 0
     end
   end
 

spec/models/project_statistics_spec.rb

@@ -325,12 +325,14 @@ RSpec.describe ProjectStatistics do
         lfs_objects_size: 3,
         snippets_size: 2,
         pipeline_artifacts_size: 3,
+        build_artifacts_size: 3,
+        packages_size: 6,
         uploads_size: 5
       )
 
       statistics.reload
 
-      expect(statistics.storage_size).to eq 19
+      expect(statistics.storage_size).to eq 28
     end
 
     it 'works during wiki_size backfill' do

spec/requests/api/groups_spec.rb

@@ -319,12 +319,15 @@ RSpec.describe API::Groups do
 
       it "includes statistics if requested" do
         attributes = {
-          storage_size: 2392,
+          storage_size: 4093,
           repository_size: 123,
           wiki_size: 456,
           lfs_objects_size: 234,
           build_artifacts_size: 345,
-          snippets_size: 1234
+          pipeline_artifacts_size: 456,
+          packages_size: 567,
+          snippets_size: 1234,
+          uploads_size: 678
         }.stringify_keys
         exposed_attributes = attributes.dup
         exposed_attributes['job_artifacts_size'] = exposed_attributes.delete('build_artifacts_size')

spec/requests/api/projects_spec.rb

@@ -359,7 +359,7 @@ RSpec.describe API::Projects do
 
         statistics = json_response.find { |p| p['id'] == project.id }['statistics']
         expect(statistics).to be_present
-        expect(statistics).to include('commit_count', 'storage_size', 'repository_size', 'wiki_size', 'lfs_objects_size', 'job_artifacts_size', 'snippets_size', 'packages_size')
+        expect(statistics).to include('commit_count', 'storage_size', 'repository_size', 'wiki_size', 'lfs_objects_size', 'job_artifacts_size', 'pipeline_artifacts_size', 'snippets_size', 'packages_size', 'uploads_size')
       end
 
       it "does not include license by default" do