Commit 5ae9a44a authored by Jacopo's avatar Jacopo

Add project http fetch statistics API

The API get projects/:id/traffic/fetches allows user with write
access to the repository to get the number of clones for the
last 30 days.
parent 6fa88ed7
...@@ -20,6 +20,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController ...@@ -20,6 +20,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
# POST /foo/bar.git/git-upload-pack (git pull) # POST /foo/bar.git/git-upload-pack (git pull)
def git_upload_pack def git_upload_pack
enqueue_fetch_statistics_update
render_ok render_ok
end end
...@@ -67,6 +69,13 @@ class Projects::GitHttpController < Projects::GitHttpClientController ...@@ -67,6 +69,13 @@ class Projects::GitHttpController < Projects::GitHttpClientController
render plain: exception.message, status: :service_unavailable render plain: exception.message, status: :service_unavailable
end end
def enqueue_fetch_statistics_update
return if wiki?
return unless project.daily_statistics_enabled?
ProjectDailyStatisticsWorker.perform_async(project.id)
end
def access def access
@access ||= access_klass.new(access_actor, project, @access ||= access_klass.new(access_actor, project,
'http', authentication_abilities: authentication_abilities, 'http', authentication_abilities: authentication_abilities,
......
# frozen_string_literal: true
module Projects
class DailyStatisticsFinder
attr_reader :project
def initialize(project)
@project = project
end
def fetches
ProjectDailyStatistic.of_project(project)
.of_last_30_days
.sorted_by_date_desc
end
def total_fetch_count
fetches.sum_fetch_count
end
end
end
...@@ -629,6 +629,10 @@ class Project < ActiveRecord::Base ...@@ -629,6 +629,10 @@ class Project < ActiveRecord::Base
auto_devops&.enabled.nil? && !(Gitlab::CurrentSettings.auto_devops_enabled? || Feature.enabled?(:force_autodevops_on_by_default, self)) auto_devops&.enabled.nil? && !(Gitlab::CurrentSettings.auto_devops_enabled? || Feature.enabled?(:force_autodevops_on_by_default, self))
end end
def daily_statistics_enabled?
Feature.enabled?(:project_daily_statistics, self, default_enabled: true)
end
def empty_repo? def empty_repo?
repository.empty? repository.empty?
end end
......
# frozen_string_literal: true
class ProjectDailyStatistic < ActiveRecord::Base
belongs_to :project
scope :of_project, -> (project) { where(project: project) }
scope :of_last_30_days, -> { where('date >= ?', 29.days.ago.utc.to_date) }
scope :sorted_by_date_desc, -> { order(project_id: :desc, date: :desc) }
scope :sum_fetch_count, -> { sum(:fetch_count) }
end
...@@ -278,6 +278,7 @@ class ProjectPolicy < BasePolicy ...@@ -278,6 +278,7 @@ class ProjectPolicy < BasePolicy
enable :admin_cluster enable :admin_cluster
enable :create_environment_terminal enable :create_environment_terminal
enable :destroy_release enable :destroy_release
enable :daily_statistics
end end
rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
......
# frozen_string_literal: true
module Projects
class FetchStatisticsIncrementService
attr_reader :project
def initialize(project)
@project = project
end
def execute
increment_fetch_count_sql = <<~SQL
INSERT INTO #{table_name} (project_id, date, fetch_count)
VALUES (#{project.id}, '#{Date.today}', 1)
SQL
increment_fetch_count_sql += if Gitlab::Database.postgresql?
"ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1"
else
"ON DUPLICATE KEY UPDATE fetch_count = #{table_name}.fetch_count + 1"
end
ActiveRecord::Base.connection.execute(increment_fetch_count_sql)
end
private
def table_name
ProjectDailyStatistic.table_name
end
end
end
...@@ -146,3 +146,4 @@ ...@@ -146,3 +146,4 @@
- repository_cleanup - repository_cleanup
- delete_stored_files - delete_stored_files
- import_issues_csv - import_issues_csv
- project_daily_statistics
# frozen_string_literal: true
class ProjectDailyStatisticsWorker
include ApplicationWorker
def perform(project_id)
project = Project.find_by_id(project_id)
return unless project&.repository&.exists?
Projects::FetchStatisticsIncrementService.new(project).execute
end
end
---
title: Add project fetch statistics
merge_request: 23596
author: Jacopo Beschi @jacopo-beschi
type: added
...@@ -85,5 +85,6 @@ ...@@ -85,5 +85,6 @@
- [repository_cleanup, 1] - [repository_cleanup, 1]
- [delete_stored_files, 1] - [delete_stored_files, 1]
- [remote_mirror_notification, 2] - [remote_mirror_notification, 2]
- [project_daily_statistics, 1]
- [import_issues_csv, 2] - [import_issues_csv, 2]
- [chat_notification, 2] - [chat_notification, 2]
# frozen_string_literal: true
class CreateProjectDailyStatistics < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :project_daily_statistics, id: :bigserial do |t|
t.integer :project_id, null: false
t.integer :fetch_count, null: false
t.date :date
t.index [:project_id, :date], unique: true, order: { date: :desc }
t.foreign_key :projects, on_delete: :cascade
end
end
end
...@@ -1578,6 +1578,13 @@ ActiveRecord::Schema.define(version: 20190204115450) do ...@@ -1578,6 +1578,13 @@ ActiveRecord::Schema.define(version: 20190204115450) do
t.index ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree t.index ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree
end end
create_table "project_daily_statistics", id: :bigserial, force: :cascade do |t|
t.integer "project_id", null: false
t.integer "fetch_count", null: false
t.date "date"
t.index ["project_id", "date"], name: "index_project_daily_statistics_on_project_id_and_date", unique: true, order: { date: :desc }, using: :btree
end
create_table "project_deploy_tokens", force: :cascade do |t| create_table "project_deploy_tokens", force: :cascade do |t|
t.integer "project_id", null: false t.integer "project_id", null: false
t.integer "deploy_token_id", null: false t.integer "deploy_token_id", null: false
...@@ -2461,6 +2468,7 @@ ActiveRecord::Schema.define(version: 20190204115450) do ...@@ -2461,6 +2468,7 @@ ActiveRecord::Schema.define(version: 20190204115450) do
add_foreign_key "project_auto_devops", "projects", on_delete: :cascade add_foreign_key "project_auto_devops", "projects", on_delete: :cascade
add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade
add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade
add_foreign_key "project_daily_statistics", "projects", on_delete: :cascade
add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade
add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade
......
# Project statistics API
Every API call to [project](../user/project/index.md) statistics must be authenticated.
## Get the statistics of the last 30 days
Retrieving the statistics requires write access to the repository.
Currently only HTTP fetches statistics are returned.
Fetches statistics includes both clones and pulls count and are HTTP only, SSH fetches are not included.
```
GET /projects/:id/statistics
```
| Attribute | Type | Required | Description |
| ---------- | ------ | -------- | ----------- |
| `id ` | integer / string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
Example response:
```json
{
"fetches": {
"total": 50,
"days": [
{
"count": 10,
"date": "2018-01-10"
},
{
"count": 10,
"date": "2018-01-09"
},
{
"count": 10,
"date": "2018-01-08"
},
{
"count": 10,
"date": "2018-01-07"
},
{
"count": 10,
"date": "2018-01-06"
}
]
}
}
```
...@@ -111,6 +111,7 @@ The following table depicts the various user permission levels in a project. ...@@ -111,6 +111,7 @@ The following table depicts the various user permission levels in a project.
| Force push to protected branches [^4] | | | | | | | Force push to protected branches [^4] | | | | | |
| Remove protected branches [^4] | | | | | | | Remove protected branches [^4] | | | | | |
| View project Audit Events | | | | ✓ | ✓ | | View project Audit Events | | | | ✓ | ✓ |
| View project statistics | | | | ✓ | ✓ |
## Project features permissions ## Project features permissions
......
...@@ -8,7 +8,7 @@ Your projects can be [available](../../public_access/public_access.md) ...@@ -8,7 +8,7 @@ Your projects can be [available](../../public_access/public_access.md)
publicly, internally, or privately, at your choice. GitLab does not limit publicly, internally, or privately, at your choice. GitLab does not limit
the number of private projects you create. the number of private projects you create.
## Project's features ## Project features
When you create a project in GitLab, you'll have access to a large number of When you create a project in GitLab, you'll have access to a large number of
[features](https://about.gitlab.com/features/): [features](https://about.gitlab.com/features/):
...@@ -82,7 +82,7 @@ your code blocks, overriding GitLab's default choice of language. ...@@ -82,7 +82,7 @@ your code blocks, overriding GitLab's default choice of language.
the source, build output, and other metadata or artifacts the source, build output, and other metadata or artifacts
associated with a released version of your code. associated with a released version of your code.
### Project's integrations ### Project integrations
[Integrate your project](integrations/index.md) with Jira, Mattermost, [Integrate your project](integrations/index.md) with Jira, Mattermost,
Kubernetes, Slack, and a lot more. Kubernetes, Slack, and a lot more.
...@@ -116,7 +116,7 @@ Read through the documentation on [project settings](settings/index.md). ...@@ -116,7 +116,7 @@ Read through the documentation on [project settings](settings/index.md).
- [Export a project from GitLab](settings/import_export.md#exporting-a-project-and-its-data) - [Export a project from GitLab](settings/import_export.md#exporting-a-project-and-its-data)
- [Importing and exporting projects between GitLab instances](settings/import_export.md) - [Importing and exporting projects between GitLab instances](settings/import_export.md)
## Project's members ## Project members
Learn how to [add members to your projects](members/index.md). Learn how to [add members to your projects](members/index.md).
...@@ -170,3 +170,23 @@ password <personal_access_token> ...@@ -170,3 +170,23 @@ password <personal_access_token>
To quickly access a project from the GitLab UI using the project ID, To quickly access a project from the GitLab UI using the project ID,
visit the `/projects/:id` URL in your browser or other tool accessing the project. visit the `/projects/:id` URL in your browser or other tool accessing the project.
## Project APIs
There are numerous [APIs](../../api/README.md) to use with your projects:
- [Badges](../../api/project_badges.md)
- [Clusters](../../api/project_clusters.md)
- [Discussions](../../api/discussions.md)
- [General](../../api/projects.md)
- [Import/export](../../api/project_import_export.md)
- [Issue Board](../../api/boards.md)
- [Labels](../../api/labels.md)
- [Markdown](../../api/markdown.md)
- [Merge Requests](../../api/merge_requests.md)
- [Milestones](../../api/milestones.md)
- [Services](../../api/services.md)
- [Snippets](../../api/project_snippets.md)
- [Templates](../../api/project_templates.md)
- [Traffic](../../api/project_statistics.md)
- [Variables](../../api/project_level_variables.md)
...@@ -141,6 +141,7 @@ module API ...@@ -141,6 +141,7 @@ module API
mount ::API::Projects mount ::API::Projects
mount ::API::ProjectSnapshots mount ::API::ProjectSnapshots
mount ::API::ProjectSnippets mount ::API::ProjectSnippets
mount ::API::ProjectStatistics
mount ::API::ProjectTemplates mount ::API::ProjectTemplates
mount ::API::ProtectedBranches mount ::API::ProtectedBranches
mount ::API::ProtectedTags mount ::API::ProtectedTags
......
...@@ -300,6 +300,18 @@ module API ...@@ -300,6 +300,18 @@ module API
expose :build_artifacts_size, as: :job_artifacts_size expose :build_artifacts_size, as: :job_artifacts_size
end end
class ProjectDailyFetches < Grape::Entity
expose :fetch_count, as: :count
expose :date
end
class ProjectDailyStatistics < Grape::Entity
expose :fetches do
expose :total_fetch_count, as: :total
expose :fetches, as: :days, using: ProjectDailyFetches
end
end
class Member < Grape::Entity class Member < Grape::Entity
expose :user, merge: true, using: UserBasic expose :user, merge: true, using: UserBasic
expose :access_level expose :access_level
......
# frozen_string_literal: true
module API
class ProjectStatistics < Grape::API
before do
authenticate!
not_found! unless user_project.daily_statistics_enabled?
authorize! :daily_statistics, user_project
end
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get the list of project fetch statistics for the last 30 days'
get ":id/statistics" do
statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project)
present statistic_finder, with: Entities::ProjectDailyStatistics
end
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :project_daily_statistic do
project
fetch_count 1
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProjectDailyStatistic do
it { is_expected.to belong_to(:project) }
end
...@@ -2335,6 +2335,18 @@ describe Project do ...@@ -2335,6 +2335,18 @@ describe Project do
end end
end end
describe '#daily_statistics_enabled?' do
it { is_expected.to be_daily_statistics_enabled }
context 'when :project_daily_statistics is disabled for the project' do
before do
stub_feature_flags(project_daily_statistics: { thing: subject, enabled: false })
end
it { is_expected.not_to be_daily_statistics_enabled }
end
end
describe '#change_head' do describe '#change_head' do
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
......
...@@ -49,6 +49,7 @@ describe ProjectPolicy do ...@@ -49,6 +49,7 @@ describe ProjectPolicy do
admin_project_member admin_note admin_wiki admin_project admin_project_member admin_note admin_wiki admin_project
admin_commit_status admin_build admin_container_image admin_commit_status admin_build admin_container_image
admin_pipeline admin_environment admin_deployment destroy_release add_cluster admin_pipeline admin_environment admin_deployment destroy_release add_cluster
daily_statistics
] ]
end end
......
# frozen_string_literal: true
require 'spec_helper'
describe API::ProjectStatistics do
let(:maintainer) { create(:user) }
let(:public_project) { create(:project, :public) }
before do
public_project.add_maintainer(maintainer)
end
describe 'GET /projects/:id/statistics' do
let!(:fetch_statistics1) { create(:project_daily_statistic, project: public_project, fetch_count: 30, date: 29.days.ago) }
let!(:fetch_statistics2) { create(:project_daily_statistic, project: public_project, fetch_count: 4, date: 3.days.ago) }
let!(:fetch_statistics3) { create(:project_daily_statistic, project: public_project, fetch_count: 3, date: 2.days.ago) }
let!(:fetch_statistics4) { create(:project_daily_statistic, project: public_project, fetch_count: 2, date: 1.day.ago) }
let!(:fetch_statistics5) { create(:project_daily_statistic, project: public_project, fetch_count: 1, date: Date.today) }
let!(:fetch_statistics_other_project) { create(:project_daily_statistic, project: create(:project), fetch_count: 29, date: 29.days.ago) }
it 'returns the fetch statistics of the last 30 days' do
get api("/projects/#{public_project.id}/statistics", maintainer)
expect(response).to have_gitlab_http_status(200)
fetches = json_response['fetches']
expect(fetches['total']).to eq(40)
expect(fetches['days'].length).to eq(5)
expect(fetches['days'].first).to eq({ 'count' => fetch_statistics5.fetch_count, 'date' => fetch_statistics5.date.to_s })
expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s })
end
it 'excludes the fetch statistics older than 30 days' do
create(:project_daily_statistic, fetch_count: 31, project: public_project, date: 30.days.ago)
get api("/projects/#{public_project.id}/statistics", maintainer)
expect(response).to have_gitlab_http_status(200)
fetches = json_response['fetches']
expect(fetches['total']).to eq(40)
expect(fetches['days'].length).to eq(5)
expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s })
end
it 'responds with 403 when the user is not a maintainer of the repository' do
developer = create(:user)
public_project.add_developer(developer)
get api("/projects/#{public_project.id}/statistics", developer)
expect(response).to have_gitlab_http_status(403)
expect(json_response['message']).to eq('403 Forbidden')
end
it 'responds with 404 when daily_statistics_enabled? is false' do
stub_feature_flags(project_daily_statistics: { thing: public_project, enabled: false })
get api("/projects/#{public_project.id}/statistics", maintainer)
expect(response).to have_gitlab_http_status(404)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
module Projects
describe FetchStatisticsIncrementService do
let(:project) { create(:project) }
describe '#execute' do
subject { described_class.new(project).execute }
it 'creates a new record for today with count == 1' do
expect { subject }.to change { ProjectDailyStatistic.count }.by(1)
created_stat = ProjectDailyStatistic.last
expect(created_stat.fetch_count).to eq(1)
expect(created_stat.project).to eq(project)
expect(created_stat.date).to eq(Date.today)
end
it "doesn't increment previous days statistics" do
yesterday_stat = create(:project_daily_statistic, fetch_count: 5, project: project, date: 1.day.ago)
expect { subject }.not_to change { yesterday_stat.reload.fetch_count }
end
context 'when the record already exists for today' do
let!(:project_daily_stat) { create(:project_daily_statistic, fetch_count: 5, project: project, date: Date.today) }
it 'increments the today record count by 1' do
expect { subject }.to change { project_daily_stat.reload.fetch_count }.to(6)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProjectDailyStatisticsWorker, '#perform' do
let(:worker) { described_class.new }
let(:project) { create(:project) }
describe '#perform' do
context 'with a non-existing project' do
it 'does nothing' do
expect(Projects::FetchStatisticsIncrementService).not_to receive(:new)
worker.perform(-1)
end
end
context 'with an existing project without a repository' do
it 'does nothing' do
expect(Projects::FetchStatisticsIncrementService).not_to receive(:new)
worker.perform(project.id)
end
end
it 'calls daily_statistics_service with the given project' do
project = create(:project, :repository)
expect_next_instance_of(Projects::FetchStatisticsIncrementService, project) do |service|
expect(service).to receive(:execute)
end
worker.perform(project.id)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment