Merge branch '3780-service-desk-e-mail-text-truncated-by-markdown' into 'master'

Append replies in email generated notes for issues

See merge request gitlab-org/gitlab!67615

lib/gitlab/email/handler/create_note_handler.rb

@@ -5,6 +5,7 @@ require 'gitlab/email/handler/reply_processing'
 
 # handles note/reply creation emails with these formats:
 #   incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com
+# Quoted material is _not_ stripped but appended as a `details` section
 module Gitlab
   module Email
     module Handler
@@ -24,7 +25,7 @@ module Gitlab
           validate_permission!(:create_note)
 
           raise NoteableNotFoundError unless noteable
-          raise EmptyEmailError if message.blank?
+          raise EmptyEmailError if note_message.blank?
 
           verify_record!(
             record: create_note,
@@ -47,7 +48,13 @@ module Gitlab
         end
 
         def create_note
-          sent_notification.create_reply(message)
+          sent_notification.create_reply(note_message)
+        end
+
+        def note_message
+          return message unless sent_notification.noteable_type == "Issue"
+
+          message_with_appended_reply
         end
       end
     end

lib/gitlab/email/handler/reply_processing.rb

@@ -35,13 +35,20 @@ module Gitlab
           @message_with_reply ||= process_message(trim_reply: false)
         end
 
+        def message_with_appended_reply
+          @message_with_appended_reply ||= process_message(append_reply: true)
+        end
+
         def process_message(**kwargs)
-          message = ReplyParser.new(mail, **kwargs).execute.strip
+          message, stripped_text = ReplyParser.new(mail, **kwargs).execute
+          message = message.strip
+
           message_with_attachments = add_attachments(message)
+          # Support bot is specifically forbidden from using slash commands.
+          message = strip_quick_actions(message_with_attachments)
+          return message unless kwargs[:append_reply]
 
-          # Support bot is specifically forbidden
-          # from using slash commands.
-          strip_quick_actions(message_with_attachments)
+          append_reply(message, stripped_text)
         end
 
         def add_attachments(reply)
@@ -92,10 +99,22 @@ module Gitlab
         def strip_quick_actions(content)
           return content unless author.support_bot?
 
+          quick_actions_extractor.redact_commands(content)
+        end
+
+        def quick_actions_extractor
           command_definitions = ::QuickActions::InterpretService.command_definitions
-          extractor = ::Gitlab::QuickActions::Extractor.new(command_definitions)
+          ::Gitlab::QuickActions::Extractor.new(command_definitions)
+        end
+
+        def append_reply(message, reply)
+          return message if message.blank? || reply.blank?
+
+          # Do not append if message only contains slash commands
+          body, _commands = quick_actions_extractor.extract_commands(message)
+          return message if body.empty?
 
-          extractor.redact_commands(content)
+          message + "\n\n<details><summary>...</summary>\n\n#{reply}\n\n</details>"
         end
       end
     end

lib/gitlab/email/reply_parser.rb

@@ -6,20 +6,17 @@ module Gitlab
     class ReplyParser
       attr_accessor :message
 
-      def initialize(message, trim_reply: true)
+      def initialize(message, trim_reply: true, append_reply: false)
         @message = message
         @trim_reply = trim_reply
+        @append_reply = append_reply
       end
 
       def execute
         body = select_body(message)
 
         encoding = body.encoding
-
-        if @trim_reply
-          body = EmailReplyTrimmer.trim(body)
-        end
-
+        body, stripped_text = EmailReplyTrimmer.trim(body, @append_reply) if @trim_reply
         return '' unless body
 
         # not using /\s+$/ here because that deletes empty lines
@@ -30,7 +27,10 @@ module Gitlab
         # so we detect it manually here.
         return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') }
 
-        body.force_encoding(encoding).encode("UTF-8")
+        encoded_body = body.force_encoding(encoding).encode("UTF-8")
+        return encoded_body unless @append_reply
+
+        [encoded_body, stripped_text.force_encoding(encoding).encode("UTF-8")]
       end
 
       private

spec/fixtures/emails/no_content_with_quote.eml

+Return-Path: <jake@adventuretime.ooo>
+Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Date: Thu, 13 Jun 2013 17:03:48 -0400
+From: Jake the Dog <jake@adventuretime.ooo>
+To: reply+59d8df8370b7e95c5a49fbf86aeb2c93@appmail.adventuretime.ooo
+Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
+Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
+Mime-Version: 1.0
+Content-Type: text/plain;
+ charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+X-Sieve: CMU Sieve 2.2
+X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
+ 13 Jun 2013 14:03:48 -0700 (PDT)
+X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
+
+--
+> quote line 1
+> quote line 2
+> quote line 3

spec/lib/gitlab/email/handler/create_note_handler_spec.rb

@@ -110,6 +110,60 @@ RSpec.describe Gitlab::Email::Handler::CreateNoteHandler do
     end
   end
 
+  context 'when email contains reply' do
+    shared_examples 'no content message' do
+      context 'when email contains quoted text only' do
+        let(:email_raw) { fixture_file('emails/no_content_with_quote.eml') }
+
+        it 'raises an EmptyEmailError' do
+          expect { receiver.execute }.to raise_error(Gitlab::Email::EmptyEmailError)
+        end
+      end
+
+      context 'when email contains quoted text and quick commands only' do
+        let(:email_raw) { fixture_file('emails/commands_only_reply.eml') }
+
+        it 'does not create a discussion' do
+          expect { receiver.execute }.not_to change { noteable.notes.count }
+        end
+      end
+    end
+
+    context 'when noteable is not an issue' do
+      let_it_be(:note) { create(:note_on_merge_request, project: project) }
+
+      it_behaves_like 'no content message'
+
+      context 'when email contains text, quoted text and quick commands' do
+        let(:email_raw) { fixture_file('emails/commands_in_reply.eml') }
+
+        it 'creates a discussion without appended reply' do
+          expect { receiver.execute }.to change { noteable.notes.count }.by(1)
+          new_note = noteable.notes.last
+
+          expect(new_note.note).not_to include('<details><summary>...</summary>')
+        end
+      end
+    end
+
+    context 'when noteable is an issue' do
+      let_it_be(:note) { create(:note_on_issue, project: project) }
+
+      it_behaves_like 'no content message'
+
+      context 'when email contains text, quoted text and quick commands' do
+        let(:email_raw) { fixture_file('emails/commands_in_reply.eml') }
+
+        it 'creates a discussion with appended reply' do
+          expect { receiver.execute }.to change { noteable.notes.count }.by(1)
+          new_note = noteable.notes.last
+
+          expect(new_note.note).to include('<details><summary>...</summary>')
+        end
+      end
+    end
+  end
+
   context 'when note is not a discussion' do
     let(:note) { create(:note_on_merge_request, project: project) }
 

spec/lib/gitlab/email/reply_parser_spec.rb

@@ -228,5 +228,21 @@ RSpec.describe Gitlab::Email::ReplyParser do
           BODY
         )
     end
+
+    it "appends trimmed reply when when append_reply option is true" do
+      body = <<-BODY.strip_heredoc.chomp
+      The reply by email functionality should be extended to allow creating a new issue by email.
+      even when the email is forwarded to the project which may include lines that begin with ">"
+
+      there should be a quote below this line:
+      BODY
+
+      reply = <<-BODY.strip_heredoc.chomp
+      > this is a quote
+      BODY
+
+      expect(test_parse_body(fixture_file("emails/valid_new_issue_with_quote.eml"), { append_reply: true }))
+        .to contain_exactly(body, reply)
+    end
   end
 end