Merge branch 'feature/fetch_git_stream_event' into 'master'

Add git audit streaming events

See merge request gitlab-org/gitlab!76719

doc/administration/audit_event_streaming.md

@@ -78,3 +78,184 @@ token is generated when the event destination is created and cannot be changed.
 
 Each streamed event contains a random alphanumeric identifier for the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against
 the destination's value when [listing streaming destinations](#list-currently-enabled-streaming-destinations).
+
+## Audit event streaming on Git operations
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `audit_event_streaming_git_operations`. On GitLab.com, this feature is not available.
+
+Streaming audit events can be sent when signed-in users push or pull a project's remote Git repositories:
+
+- [Using SSH](../ssh/index.md).
+- Using HTTP or HTTPS.
+- Using the **Download** button (**{download}**) in GitLab UI.
+
+Audit events are not captured for users that are not signed in. For example, when downloading a public project.
+
+To configure streaming audit events for Git operations, see [Add a new event streaming destination](#add-a-new-event-streaming-destination).
+
+### Request headers
+
+Request headers are formatted as follows:
+
+```plaintext
+POST /logs HTTP/1.1
+Host: <DESTINATION_HOST>
+Content-Type: application/x-www-form-urlencoded
+X-Gitlab-Event-Streaming-Token: <DESTINATION_TOKEN>
+```
+
+### Example responses for SSH events
+
+Fetch:
+
+```json
+{
+  "id": 1,
+  "author_id": 1,
+  "entity_id": 29,
+  "entity_type": "Project",
+  "details": {
+    "author_name": "Administrator",
+    "target_id": 29,
+    "target_type": "Project",
+    "target_details": "example-project",
+    "custom_message": {
+      "protocol": "ssh",
+      "action": "git-upload-pack"
+    },
+    "ip_address": "127.0.0.1",
+    "entity_path": "example-group/example-project"
+  },
+  "ip_address": "127.0.0.1",
+  "author_name": "Administrator",
+  "entity_path": "example-group/example-project",
+  "target_details": "example-project",
+  "created_at": "2022-02-23T06:21:05.283Z",
+  "target_type": "Project",
+  "target_id": 29
+}
+```
+
+Push:
+
+```json
+{
+  "id": 1,
+  "author_id": 1,
+  "entity_id": 29,
+  "entity_type": "Project",
+  "details": {
+    "author_name": "Administrator",
+    "target_id": 29,
+    "target_type": "Project",
+    "target_details": "example-project",
+    "custom_message": {
+      "protocol": "ssh",
+      "action": "git-receive-pack"
+    },
+    "ip_address": "127.0.0.1",
+    "entity_path": "example-group/example-project"
+  },
+  "ip_address": "127.0.0.1",
+  "author_name": "Administrator",
+  "entity_path": "example-group/example-project",
+  "target_details": "example-project",
+  "created_at": "2022-02-23T06:23:08.746Z",
+  "target_type": "Project",
+  "target_id": 29
+}
+```
+
+### Example responses for HTTP and HTTPS events
+
+Fetch:
+
+```json
+{
+  "id": 1,
+  "author_id": 1,
+  "entity_id": 29,
+  "entity_type": "Project",
+  "details": {
+    "author_name": "Administrator",
+    "target_id": 29,
+    "target_type": "Project",
+    "target_details": "example-project",
+    "custom_message": {
+      "protocol": "http",
+      "action": "git-upload-pack"
+    },
+    "ip_address": "127.0.0.1",
+    "entity_path": "example-group/example-project"
+  },
+  "ip_address": "127.0.0.1",
+  "author_name": "Administrator",
+  "entity_path": "example-group/example-project",
+  "target_details": "example-project",
+  "created_at": "2022-02-23T06:25:43.938Z",
+  "target_type": "Project",
+  "target_id": 29
+}
+```
+
+Push:
+
+```json
+{
+  "id": 1,
+  "author_id": 1,
+  "entity_id": 29,
+  "entity_type": "Project",
+  "details": {
+    "author_name": "Administrator",
+    "target_id": 29,
+    "target_type": "Project",
+    "target_details": "example-project",
+    "custom_message": {
+      "protocol": "http",
+      "action": "git-receive-pack"
+    },
+    "ip_address": "127.0.0.1",
+    "entity_path": "example-group/example-project"
+  },
+  "ip_address": "127.0.0.1",
+  "author_name": "Administrator",
+  "entity_path": "example-group/example-project",
+  "target_details": "example-project",
+  "created_at": "2022-02-23T06:26:29.294Z",
+  "target_type": "Project",
+  "target_id": 29
+}
+```
+
+### Example responses for events from GitLab UI download button
+
+Fetch:
+
+```json
+{
+  "id": 1,
+  "author_id": 99,
+  "entity_id": 29,
+  "entity_type": "Project",
+  "details": {
+    "custom_message": "Repository Download Started",
+    "author_name": "example_username",
+    "target_id": 29,
+    "target_type": "Project",
+    "target_details": "example-group/example-project",
+    "ip_address": "127.0.0.1",
+    "entity_path": "example-group/example-project"
+  },
+  "ip_address": "127.0.0.1",
+  "author_name": "example_username",
+  "entity_path": "example-group/example-project",
+  "target_details": "example-group/example-project",
+  "created_at": "2022-02-23T06:27:17.873Z",
+  "target_type": "Project",
+  "target_id": 29
+}
+```

ee/app/controllers/ee/repositories/git_http_controller.rb

@@ -81,6 +81,12 @@ module EE
         render_bad_geo_jwt("Invalid signature time ")
       end
 
+      override :update_fetch_statistics
+      def update_fetch_statistics
+        send_git_audit_streaming_event
+        super
+      end
+
       def jwt_scope_valid?
         decoded_authorization[:scope] == repository_path.delete_suffix('.git')
       end
@@ -102,6 +108,18 @@ module EE
       def ip_allowed?
         ::Gitlab::Geo.allowed_ip?(request.ip)
       end
+
+      def send_git_audit_streaming_event
+        return if user.blank? || project.blank?
+        return unless ::Feature.enabled?(:audit_event_streaming_git_operations, project.group)
+
+        AuditEvents::BuildService.new(
+          author: user,
+          scope: project,
+          target: project,
+          message: { protocol: 'http', action: 'git-upload-pack' }
+        ).execute.stream_to_external_destinations(use_json: true)
+      end
     end
   end
 end

ee/app/models/ee/audit_event.rb

@@ -54,11 +54,12 @@ module EE
         end
     end
 
-    def stream_to_external_destinations
+    def stream_to_external_destinations(use_json: false)
       return if entity.nil?
       return unless group_entity&.licensed_feature_available?(:external_audit_events)
 
-      AuditEvents::AuditEventStreamingWorker.perform_async(id)
+      perform_params = use_json ? [nil, self.to_json] : [id, nil]
+      AuditEvents::AuditEventStreamingWorker.perform_async(*perform_params)
     end
 
     def entity_is_group_or_project?

ee/app/workers/audit_events/audit_event_streaming_worker.rb

@@ -46,7 +46,7 @@ module AuditEvents
     end
 
     def parse_audit_event_json(audit_event_json)
-      audit_event_json = Gitlab::Json.parse(audit_event_json)
+      audit_event_json = Gitlab::Json.parse(audit_event_json).with_indifferent_access
       audit_event = AuditEvent.new(audit_event_json)
       # We want to have created_at as unique id for deduplication if audit_event id is not present
       audit_event.id = audit_event.created_at.to_i if audit_event.id.blank?

ee/config/feature_flags/development/audit_event_streaming_git_operations.yml

+---
+name: audit_event_streaming_git_operations
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76719
+rollout_issue_url:
+milestone: '14.9'
+type: development
+group: group::compliance
+default_enabled: false

ee/lib/ee/api/internal/base.rb

@@ -23,6 +23,19 @@ module EE
               end
             end
 
+            override :send_git_audit_streaming_event
+            def send_git_audit_streaming_event(msg)
+              return if actor.user.blank? || @project.blank?
+              return unless ::Feature.enabled?(:audit_event_streaming_git_operations, @project.group)
+
+              ::AuditEvents::BuildService.new(
+                author: actor.user,
+                scope: @project,
+                target: @project,
+                message: msg
+              ).execute.stream_to_external_destinations(use_json: true)
+            end
+
             override :two_factor_otp_check
             def two_factor_otp_check
               return { success: false, message: 'Feature is not available' } unless ::License.feature_available?(:git_two_factor_enforcement)

ee/spec/controllers/projects/repositories_controller_spec.rb

@@ -3,14 +3,22 @@
 require "spec_helper"
 
 RSpec.describe Projects::RepositoriesController do
-  let(:project) { create(:project, :repository) }
+  let(:group) { create(:group) }
+  let(:project) { create(:project, :repository, namespace: group) }
 
   describe "GET archive" do
+    subject(:get_archive) do
+      get :archive, params: { namespace_id: project.namespace, project_id: project, id: "master" }, format: "zip"
+    end
+
+    def set_group_destination
+      group.external_audit_event_destinations.create!(destination_url: 'http://example.com')
+      stub_licensed_features(external_audit_events: true)
+    end
+
     shared_examples 'logs the audit event' do
       it 'logs the audit event' do
-        expect do
-          get :archive, params: { namespace_id: project.namespace, project_id: project, id: "master" }, format: "zip"
-        end.to change { AuditEvent.count }.by(1)
+        expect { get_archive }.to change { AuditEvent.count }.by(1)
       end
     end
 
@@ -18,6 +26,16 @@ RSpec.describe Projects::RepositoriesController do
       it_behaves_like 'logs the audit event' do
         let(:project) { create(:project, :repository, :public) }
       end
+
+      context 'when group sets event destination' do
+        before do
+          set_group_destination
+        end
+        it "doesn't send the streaming audit event" do
+          expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async)
+          get_archive
+        end
+      end
     end
 
     context 'when authenticated', 'as a developer' do
@@ -29,6 +47,18 @@ RSpec.describe Projects::RepositoriesController do
       it_behaves_like 'logs the audit event' do
         let(:user) { create(:user) }
       end
+
+      context 'when group sets event destination' do
+        let(:user) { create(:user) }
+
+        before do
+          set_group_destination
+        end
+        it "sends the streaming audit event" do
+          expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async)
+          get_archive
+        end
+      end
     end
   end
 end

ee/spec/controllers/repositories/git_http_controller_spec.rb

@@ -22,4 +22,14 @@ RSpec.describe Repositories::GitHttpController do
       let(:access_checker_class) { Gitlab::GitAccessWiki }
     end
   end
+
+  context 'git audit streaming event' do
+    include GitHttpHelpers
+
+    it_behaves_like 'sends git audit streaming event' do
+      subject do
+        post :git_upload_pack, params: { repository_path: "#{project.full_path}.git" }
+      end
+    end
+  end
 end

ee/spec/requests/api/internal/base_spec.rb

@@ -277,6 +277,12 @@ RSpec.describe API::Internal::Base do
         end
       end
     end
+
+    context 'git audit streaming event' do
+      it_behaves_like 'sends git audit streaming event' do
+        subject { pull(key, project) }
+      end
+    end
   end
 
   describe "POST /internal/lfs_authenticate", :geo do

lib/api/internal/base.rb

@@ -92,6 +92,8 @@ module API
               payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}"
             end
 
+            send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action])
+
             response_with_status(**payload)
           when ::Gitlab::GitAccessResult::CustomAction
             response_with_status(code: 300, payload: check_result.payload, gl_console_messages: check_result.console_messages)
@@ -100,6 +102,10 @@ module API
           end
         end
 
+        def send_git_audit_streaming_event(msg)
+          # Defined in EE
+        end
+
         def access_check!(actor, params)
           access_checker = access_checker_for(actor, params[:protocol])
           access_checker.check(params[:action], params[:changes]).tap do |result|

spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'sends git audit streaming event' do
+  let_it_be(:user) { create(:user) }
+
+  before do
+    stub_licensed_features(external_audit_events: true)
+  end
+
+  subject {}
+
+  context 'for public groups and projects' do
+    let(:group) { create(:group, :public) }
+    let(:project) { create(:project, :public, :repository, namespace: group) }
+
+    before do
+      group.external_audit_event_destinations.create!(destination_url: 'http://example.com')
+      project.add_developer(user)
+    end
+
+    context 'when user not logged in' do
+      let(:key) { create(:key) }
+
+      before do
+        if request
+          request.headers.merge! auth_env(user.username, nil, nil)
+        end
+      end
+      it 'sends the audit streaming event' do
+        expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async)
+        subject
+      end
+    end
+  end
+
+  context 'for private groups and projects' do
+    let(:group) { create(:group, :private) }
+    let(:project) { create(:project, :private, :repository, namespace: group) }
+
+    before do
+      group.external_audit_event_destinations.create!(destination_url: 'http://example.com')
+      project.add_developer(user)
+      sign_in(user)
+    end
+
+    context 'when user logged in' do
+      let(:key) { create(:key, user: user) }
+
+      before do
+        if request
+          password = user.try(:password) || user.try(:token)
+          request.headers.merge! auth_env(user.username, password, nil)
+        end
+      end
+      it 'sends the audit streaming event' do
+        expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async).once
+        subject
+      end
+    end
+  end
+end