Add error tracking collector

Add API endpoint that receives error tracking event data from error
tracking client sdk.

Changelog: added
Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

app/models/error_tracking/error.rb

@@ -9,4 +9,15 @@ class ErrorTracking::Error < ApplicationRecord
   validates :name, presence: true
   validates :description, presence: true
   validates :actor, presence: true
+
+  def self.report_error(name:, description:, actor:, platform:, timestamp:)
+    safe_find_or_create_by(
+      name: name,
+      description: description,
+      actor: actor,
+      platform: platform
+    ) do |error|
+      error.update!(last_seen_at: timestamp)
+    end
+  end
 end

app/models/project.rb

@@ -377,6 +377,8 @@ class Project < ApplicationRecord
   has_one :operations_feature_flags_client, class_name: 'Operations::FeatureFlagsClient'
   has_many :operations_feature_flags_user_lists, class_name: 'Operations::FeatureFlags::UserList'
 
+  has_many :error_tracking_errors, inverse_of: :project, class_name: 'ErrorTracking::Error'
+
   has_many :timelogs
 
   accepts_nested_attributes_for :variables, allow_destroy: true

app/services/error_tracking/collect_error_service.rb

+# frozen_string_literal: true
+
+module ErrorTracking
+  class CollectErrorService < ::BaseService
+    def execute
+      # Error is a way to group events based on common data like name or cause
+      # of exception. We need to keep a sane balance here between taking too little
+      # and too much data into group logic.
+      error = project.error_tracking_errors.report_error(
+        name: exception['type'], # Example: ActionView::MissingTemplate
+        description: exception['value'], # Example: Missing template posts/show in...
+        actor: event['transaction'], # Example: PostsController#show
+        platform: event['platform'], # Example: ruby
+        timestamp: event['timestamp']
+      )
+
+      # The payload field contains all the data on error including stacktrace in jsonb.
+      # Together with occured_at these are 2 main attributes that we need to save here.
+      error.events.create!(
+        environment: event['environment'],
+        description: exception['type'],
+        level: event['level'],
+        occurred_at: event['timestamp'],
+        payload: event
+      )
+    end
+
+    private
+
+    def event
+      params[:event]
+    end
+
+    def exception
+      event['exception']['values'].first
+    end
+  end
+end

config/feature_flags/development/integrated_error_tracking.yml

+---
+name: integrated_error_tracking
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65767
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/335846
+milestone: '14.1'
+type: development
+group: group::monitor
+default_enabled: false

lib/api/api.rb

@@ -167,6 +167,7 @@ module API
       mount ::API::Deployments
       mount ::API::Environments
       mount ::API::ErrorTracking
+      mount ::API::ErrorTrackingCollector
       mount ::API::Events
       mount ::API::FeatureFlags
       mount ::API::FeatureFlagsUserLists

lib/api/error_tracking_collector.rb

+# frozen_string_literal: true
+
+module API
+  # This API is responsible for collecting error tracking information
+  # from sentry client. It allows us to use GitLab as an alternative to
+  # sentry backend. For more details see https://gitlab.com/gitlab-org/gitlab/-/issues/329596.
+  class ErrorTrackingCollector < ::API::Base
+    feature_category :error_tracking
+
+    content_type :envelope, 'application/x-sentry-envelope'
+    default_format :envelope
+
+    before do
+      not_found!('Project') unless project
+      not_found! unless feature_enabled?
+    end
+
+    helpers do
+      def project
+        @project ||= find_project(params[:id])
+      end
+
+      def feature_enabled?
+        ::Feature.enabled?(:integrated_error_tracking, project) &&
+          project.error_tracking_setting&.enabled?
+      end
+    end
+
+    desc 'Submit error tracking event to the project' do
+      detail 'This feature was introduced in GitLab 14.1.'
+    end
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    post 'error_tracking/collector/api/:id/envelope' do
+      # There is a reason why we have such uncommon path.
+      # We depend on a client side error tracking software which
+      # modifies URL for its own reasons.
+      #
+      # When we give user a URL like this
+      #   HOST/api/v4/error_tracking/collector/123
+      #
+      # Then error tracking software will convert it like this:
+      #   HOST/api/v4/error_tracking/collector/api/123/envelope/
+
+      begin
+        parsed_request = ::ErrorTracking::Collector::SentryRequestParser.parse(request)
+      rescue StandardError
+        render_api_error!('Failed to parse sentry request', 400)
+      end
+
+      type = parsed_request[:request_type]
+
+      # Sentry sends 2 requests on each exception: transaction and event.
+      # Everything else is not a desired behavior.
+      unless type == 'transaction' || type == 'event'
+        render_api_error!('400 Bad Request', 400)
+
+        break
+      end
+
+      # We don't have use for transaction request yet,
+      # so we record only event one.
+      if type == 'event'
+        ::ErrorTracking::CollectErrorService
+          .new(project, nil, event: parsed_request[:event])
+          .execute
+      end
+
+      no_content!
+    end
+  end
+end

lib/error_tracking/collector/sentry_request_parser.rb

+# frozen_string_literal: true
+
+module ErrorTracking
+  module Collector
+    class SentryRequestParser
+      def self.parse(request)
+        # Request body can be "" or "gzip".
+        # If later then body was compressed with Zlib.gzip
+        encoding = request.headers['Content-Encoding']
+
+        body = if encoding == 'gzip'
+                 Zlib.gunzip(request.body.read)
+               else
+                 request.body.read
+               end
+
+        # Request body contains 3 json objects merged together in one StringIO.
+        # We need to separate and parse them into array of hash objects.
+        json_objects = []
+        parser = Yajl::Parser.new
+
+        parser.parse(body) do |json_object|
+          json_objects << json_object
+        end
+
+        # The request contains 3 objects: sentry metadata, type data and event data.
+        # We need only last two. Type to decide what to do with the request.
+        # And event data as it contains all information about the exception.
+        _, type, event = json_objects
+
+        {
+          request_type: type['type'],
+          event: event
+        }
+      end
+    end
+  end
+end

spec/fixtures/error_tracking/event.txt

+{"event_id":"7c9ae6e58f03442b9203bbdcf6ae904c","dsn":"http://1fedb514e17f4b958435093deb03048c@localhost:3000/api/v4/projects/7/error_tracking/collector/7","sdk":{"name":"sentry.ruby","version":"4.5.1"},"sent_at":"2021-07-08T12:59:16Z"}
+{"type":"event","content_type":"application/json"}
+{"event_id":"7c9ae6e58f03442b9203bbdcf6ae904c","level":"error","timestamp":"2021-07-08T12:59:16Z","release":"db853d7","environment":"development","server_name":"MacBook.local","modules":{"rake":"13.0.3","concurrent-ruby":"1.1.9","i18n":"1.8.10","minitest":"5.14.4","thread_safe":"0.3.6","tzinfo":"1.2.9","uglifier":"4.2.0","web-console":"3.7.0"},"message":"","user":{},"tags":{"request_id":"4253dcd9-5e48-474a-89b4-0e945ab825af"},"contexts":{"os":{"name":"Darwin","version":"Darwin Kernel Version 20.5.0: Sat May  8 05:10:33 PDT 2021; root:xnu-7195.121.3~9/RELEASE_X86_64","build":"20.5.0","kernel_version":"Darwin Kernel Version 20.5.0: Sat May  8 05:10:33 PDT 2021; root:xnu-7195.121.3~9/RELEASE_X86_64"},"runtime":{"name":"ruby","version":"ruby 2.5.1p57 (2018-03-29 revision 63029) [x86_64-darwin19]"},"trace":{"trace_id":"d82b93fbc39e4d13b85762afa2e3ff36","span_id":"4a3ed8701e7f4ea4","parent_span_id":null,"description":null,"op":"rails.request","status":null}},"extra":{},"fingerprint":[],"breadcrumbs":{"values":[{"category":"start_processing.action_controller","data":{"controller":"PostsController","action":"error2","params":{"controller":"posts","action":"error2"},"format":"html","method":"GET","path":"/posts/error2","start_timestamp":1625749156.5553},"level":null,"message":"","timestamp":1625749156,"type":null},{"category":"process_action.action_controller","data":{"controller":"PostsController","action":"error2","params":{"controller":"posts","action":"error2"},"format":"html","method":"GET","path":"/posts/error2","start_timestamp":1625749156.55539,"view_runtime":null,"db_runtime":0},"level":null,"message":"","timestamp":1625749156,"type":null}]},"transaction":"PostsController#error2","platform":"ruby","sdk":{"name":"sentry.ruby.rails","version":"4.5.1"},"request":{"url":"http://localhost/posts/error2","method":"GET","headers":{},"env":{"SERVER_NAME":"localhost","SERVER_PORT":"4444"}},"exception":{"values":[{"type":"ActionView::MissingTemplate","value":"Missing template posts/error2, application/error2 with {:locale=>[:en], :formats=>[:html], :variants=>[], :handlers=>[:raw, :erb, :html, :builder, :ruby, :coffee, :jbuilder]}. Searched in:\n  * \"/Users/developer/rails-project/app/views\"\n","module":"ActionView","thread_id":70254489510160,"stacktrace":{"frames":[{"project_root":"/Users/developer/rails-project","abs_path":"/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/puma-3.12.6/lib/puma/thread_pool.rb","function":"block in spawn_thread","lineno":135,"in_app":false,"filename":"puma/thread_pool.rb","pre_context":["          end\n","\n","          begin\n"],"context_line":"            block.call(work, *extra)\n","post_context":["          rescue Exception => e\n","            STDERR.puts \"Error reached top of thread-pool: #{e.message} (#{e.class})\"\n","          end\n"]},{"project_root":"/Users/developer/rails-project","abs_path":"/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/puma-3.12.6/lib/puma/server.rb","function":"block in run","lineno":334,"in_app":false,"filename":"puma/server.rb","pre_context":["          client.close\n","        else\n","          if process_now\n"],"context_line":"            process_client client, buffer\n","post_context":["          else\n","            client.set_timeout @first_data_timeout\n","            @reactor.add client\n"]},{"project_root":"/Users/developer/rails-project","abs_path":"/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/actionview-5.2.6/lib/action_view/path_set.rb","function":"find","lineno":48,"in_app":false,"filename":"action_view/path_set.rb","pre_context":["    end\n","\n","    def find(*args)\n"],"context_line":"      find_all(*args).first || raise(MissingTemplate.new(self, *args))\n","post_context":["    end\n","\n","    def find_file(path, prefixes = [], *args)\n"]}]}}]}}
\ No newline at end of file

spec/fixtures/error_tracking/parsed_event.json

+{"event_id":"7c9ae6e58f03442b9203bbdcf6ae904c","level":"error","timestamp":"2021-07-08T12:59:16Z","release":"db853d7","environment":"development","server_name":"MacBook.local","modules":{"rake":"13.0.3","concurrent-ruby":"1.1.9","i18n":"1.8.10","minitest":"5.14.4","thread_safe":"0.3.6","tzinfo":"1.2.9","uglifier":"4.2.0","web-console":"3.7.0"},"message":"","user":{},"tags":{"request_id":"4253dcd9-5e48-474a-89b4-0e945ab825af"},"contexts":{"os":{"name":"Darwin","version":"Darwin Kernel Version 20.5.0: Sat May  8 05:10:33 PDT 2021; root:xnu-7195.121.3~9/RELEASE_X86_64","build":"20.5.0","kernel_version":"Darwin Kernel Version 20.5.0: Sat May  8 05:10:33 PDT 2021; root:xnu-7195.121.3~9/RELEASE_X86_64"},"runtime":{"name":"ruby","version":"ruby 2.5.1p57 (2018-03-29 revision 63029) [x86_64-darwin19]"},"trace":{"trace_id":"d82b93fbc39e4d13b85762afa2e3ff36","span_id":"4a3ed8701e7f4ea4","parent_span_id":null,"description":null,"op":"rails.request","status":null}},"extra":{},"fingerprint":[],"breadcrumbs":{"values":[{"category":"start_processing.action_controller","data":{"controller":"PostsController","action":"error2","params":{"controller":"posts","action":"error2"},"format":"html","method":"GET","path":"/posts/error2","start_timestamp":1625749156.5553},"level":null,"message":"","timestamp":1625749156,"type":null},{"category":"process_action.action_controller","data":{"controller":"PostsController","action":"error2","params":{"controller":"posts","action":"error2"},"format":"html","method":"GET","path":"/posts/error2","start_timestamp":1625749156.55539,"view_runtime":null,"db_runtime":0},"level":null,"message":"","timestamp":1625749156,"type":null}]},"transaction":"PostsController#error2","platform":"ruby","sdk":{"name":"sentry.ruby.rails","version":"4.5.1"},"request":{"url":"http://localhost/posts/error2","method":"GET","headers":{},"env":{"SERVER_NAME":"localhost","SERVER_PORT":"4444"}},"exception":{"values":[{"type":"ActionView::MissingTemplate","value":"Missing template posts/error2, application/error2 with {:locale=>[:en], :formats=>[:html], :variants=>[], :handlers=>[:raw, :erb, :html, :builder, :ruby, :coffee, :jbuilder]}. Searched in:\n  * \"/Users/developer/rails-project/app/views\"\n","module":"ActionView","thread_id":70254489510160,"stacktrace":{"frames":[{"project_root":"/Users/developer/rails-project","abs_path":"/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/puma-3.12.6/lib/puma/thread_pool.rb","function":"block in spawn_thread","lineno":135,"in_app":false,"filename":"puma/thread_pool.rb","pre_context":["          end\n","\n","          begin\n"],"context_line":"            block.call(work, *extra)\n","post_context":["          rescue Exception => e\n","            STDERR.puts \"Error reached top of thread-pool: #{e.message} (#{e.class})\"\n","          end\n"]},{"project_root":"/Users/developer/rails-project","abs_path":"/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/puma-3.12.6/lib/puma/server.rb","function":"block in run","lineno":334,"in_app":false,"filename":"puma/server.rb","pre_context":["          client.close\n","        else\n","          if process_now\n"],"context_line":"            process_client client, buffer\n","post_context":["          else\n","            client.set_timeout @first_data_timeout\n","            @reactor.add client\n"]},{"project_root":"/Users/developer/rails-project","abs_path":"/Users/developer/.asdf/installs/ruby/2.5.1/lib/ruby/gems/2.5.0/gems/actionview-5.2.6/lib/action_view/path_set.rb","function":"find","lineno":48,"in_app":false,"filename":"action_view/path_set.rb","pre_context":["    end\n","\n","    def find(*args)\n"],"context_line":"      find_all(*args).first || raise(MissingTemplate.new(self, *args))\n","post_context":["    end\n","\n","    def find_file(path, prefixes = [], *args)\n"]}]}}]}}
\ No newline at end of file

spec/fixtures/error_tracking/transaction.txt

+{"event_id":"4a304dbdf3404e87962e99bced2f6c8b","dsn":"","sdk":{"name":"sentry.ruby","version":"4.5.1"},"sent_at":"2021-07-08T12:58:29Z"}
+{"type":"transaction","content_type":"application/json"}
+{}
\ No newline at end of file

spec/fixtures/error_tracking/unknown.txt

+{"event_id":"7c9ae6e58f03442b9203bbdcf6ae904c","dsn":"","sdk":{"name":"sentry.ruby","version":"4.5.1"},"sent_at":"2021-07-08T12:59:16Z"}
+{"type":"unknown","content_type":"application/json"}
+{}
\ No newline at end of file

spec/lib/error_tracking/collector/sentry_request_parser_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ErrorTracking::Collector::SentryRequestParser do
+  describe '.parse' do
+    let_it_be(:raw_event) { fixture_file('error_tracking/event.txt') }
+    let_it_be(:parsed_event) { Gitlab::Json.parse(fixture_file('error_tracking/parsed_event.json')) }
+
+    let(:body) { raw_event }
+    let(:headers) { { 'Content-Encoding' => '' } }
+    let(:request) { double('request', headers: headers, body: StringIO.new(body)) }
+
+    subject { described_class.parse(request) }
+
+    RSpec.shared_examples 'valid parser' do
+      it 'returns a valid hash' do
+        parsed_request = subject
+
+        expect(parsed_request[:request_type]).to eq('event')
+        expect(parsed_request[:event]).to eq(parsed_event)
+      end
+    end
+
+    context 'empty body content' do
+      let(:body) { '' }
+
+      it 'fails with exception' do
+        expect { subject }.to raise_error(StandardError)
+      end
+    end
+
+    context 'plain text sentry request' do
+      it_behaves_like 'valid parser'
+    end
+
+    context 'gzip encoded sentry request' do
+      let(:headers) { { 'Content-Encoding' => 'gzip' } }
+      let(:body) { Zlib.gzip(raw_event) }
+
+      it_behaves_like 'valid parser'
+    end
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -578,6 +578,7 @@ project:
 - merge_request_metrics
 - security_orchestration_policy_configuration
 - timelogs
+- error_tracking_errors
 award_emoji:
 - awardable
 - user

spec/requests/api/error_tracking_collector_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::ErrorTrackingCollector do
+  let_it_be(:project) { create(:project, :private) }
+  let_it_be(:setting) { create(:project_error_tracking_setting, project: project) }
+
+  describe "POST /error_tracking/collector/api/:id/envelope" do
+    let_it_be(:raw_event) { fixture_file('error_tracking/event.txt') }
+    let_it_be(:url) { "/error_tracking/collector/api/#{project.id}/envelope" }
+
+    let(:params) { raw_event }
+
+    subject { post api(url), params: params }
+
+    RSpec.shared_examples 'not found' do
+      it 'reponds with 404' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    RSpec.shared_examples 'bad request' do
+      it 'responds with 400' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:bad_request)
+      end
+    end
+
+    context 'error tracking feature is disabled' do
+      before do
+        setting.update!(enabled: false)
+      end
+
+      it_behaves_like 'not found'
+    end
+
+    context 'feature flag is disabled' do
+      before do
+        stub_feature_flags(integrated_error_tracking: false)
+      end
+
+      it_behaves_like 'not found'
+    end
+
+    context 'empty body' do
+      let(:params) { '' }
+
+      it_behaves_like 'bad request'
+    end
+
+    context 'unknown request type' do
+      let(:params) { fixture_file('error_tracking/unknown.txt') }
+
+      it_behaves_like 'bad request'
+    end
+
+    context 'transaction request type' do
+      let(:params) { fixture_file('error_tracking/transaction.txt') }
+
+      it 'does nothing and returns no content' do
+        expect { subject }.not_to change { ErrorTracking::ErrorEvent.count }
+
+        expect(response).to have_gitlab_http_status(:no_content)
+      end
+    end
+
+    it 'writes to the database and returns no content' do
+      expect { subject }.to change { ErrorTracking::ErrorEvent.count }.by(1)
+
+      expect(response).to have_gitlab_http_status(:no_content)
+    end
+  end
+end

spec/services/error_tracking/collect_error_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ErrorTracking::CollectErrorService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:parsed_event) { Gitlab::Json.parse(fixture_file('error_tracking/parsed_event.json')) }
+
+  subject { described_class.new(project, nil, event: parsed_event) }
+
+  describe '#execute' do
+    it 'creates Error and creates ErrorEvent' do
+      expect { subject.execute }
+        .to change { ErrorTracking::Error.count }.by(1)
+        .and change { ErrorTracking::ErrorEvent.count }.by(1)
+    end
+
+    it 'updates Error and created ErrorEvent on second hit' do
+      subject.execute
+
+      expect { subject.execute }.not_to change { ErrorTracking::Error.count }
+      expect { subject.execute }.to change { ErrorTracking::ErrorEvent.count }.by(1)
+    end
+
+    it 'has correct values set' do
+      subject.execute
+
+      event = ErrorTracking::ErrorEvent.last
+      error = event.error
+
+      expect(error.name).to eq 'ActionView::MissingTemplate'
+      expect(error.description).to start_with 'Missing template posts/error2'
+      expect(error.actor).to eq 'PostsController#error2'
+      expect(error.platform).to eq 'ruby'
+      expect(error.last_seen_at).to eq '2021-07-08T12:59:16Z'
+
+      expect(event.description).to eq 'ActionView::MissingTemplate'
+      expect(event.occurred_at).to eq '2021-07-08T12:59:16Z'
+      expect(event.level).to eq 'error'
+      expect(event.environment).to eq 'development'
+      expect(event.payload).to eq parsed_event
+    end
+  end
+end