Merge branch 'pipeline-emails' into 'master'

Add a new pipeline email service

## What does this MR do?

Add a new pipeline email service

## What are the relevant issue numbers?

Closes #3976 

## Remaining tasks

* [x] Preserve `·` and ` `
* [x] Use XHTML 1.0
* [ ] Use the same layout (`app/views/layouts/notify.html.haml`)
* [ ] Digest or not (assets or public)
* [x] A similar email for succeeded pipeline
* [x] Plain text versions for both emails

## Screenshots (if relevant)

https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/6019#note_16594345

## Does this MR meet the acceptance criteria?

- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [ ] API support added
- Tests
  - [x] `PipelinesEmailService`
  - [x] `SendPipelineNotificationService`

See merge request !6019

CHANGELOG.md

@@ -250,6 +250,7 @@ Please view this file on the master branch, on stable branches it's out of date.
   - Changed MR widget build status to pipeline status !6335
   - Add `web_url` field to issue, merge request, and snippet API objects (Ben Boeckel)
   - Enable pipeline events by default !6278
+  - Add pipeline email service !6019
   - Move parsing of sidekiq ps into helper !6245 (pascalbetz)
   - Added go to issue boards keyboard shortcut
   - Expose `sha` and `merge_commit_sha` in merge request API (Ben Boeckel)

app/controllers/admin/services_controller.rb

@@ -26,14 +26,10 @@ class Admin::ServicesController < Admin::ApplicationController
   private
 
   def services_templates
-    templates = []
-
-    Service.available_services_names.each do |service_name|
+    Service.available_services_names.map do |service_name|
       service_template = service_name.concat("_service").camelize.constantize
-      templates << service_template.where(template: true).first_or_create
+      service_template.where(template: true).first_or_create
     end
-
-    templates
   end
 
   def service

app/controllers/projects/builds_controller.rb

@@ -47,7 +47,9 @@ class Projects::BuildsController < Projects::ApplicationController
   def trace
     respond_to do |format|
       format.json do
-        render json: @build.trace_with_state(params[:state].presence).merge!(id: @build.id, status: @build.status)
+        state = params[:state].presence
+        render json: @build.trace_with_state(state: state).
+          merge!(id: @build.id, status: @build.status)
       end
     end
   end

app/helpers/gitlab_routing_helper.rb

@@ -94,6 +94,22 @@ module GitlabRoutingHelper
     namespace_project_merge_request_url(entity.project.namespace, entity.project, entity, *args)
   end
 
+  def pipeline_url(pipeline, *args)
+    namespace_project_pipeline_url(pipeline.project.namespace, pipeline.project, pipeline.id, *args)
+  end
+
+  def pipeline_build_url(pipeline, build, *args)
+    namespace_project_build_url(pipeline.project.namespace, pipeline.project, build.id, *args)
+  end
+
+  def commits_url(entity, *args)
+    namespace_project_commits_url(entity.project.namespace, entity.project, entity.ref, *args)
+  end
+
+  def commit_url(entity, *args)
+    namespace_project_commit_url(entity.project.namespace, entity.project, entity.sha, *args)
+  end
+
   def project_snippet_url(entity, *args)
     namespace_project_snippet_url(entity.project.namespace, entity.project, entity, *args)
   end

app/mailers/emails/pipelines.rb

+module Emails
+  module Pipelines
+    def pipeline_success_email(pipeline, to)
+      pipeline_mail(pipeline, to, 'succeeded')
+    end
+
+    def pipeline_failed_email(pipeline, to)
+      pipeline_mail(pipeline, to, 'failed')
+    end
+
+    private
+
+    def pipeline_mail(pipeline, to, status)
+      @project = pipeline.project
+      @pipeline = pipeline
+      @merge_request = pipeline.merge_requests.first
+      add_headers
+
+      mail(to: to, subject: pipeline_subject(status), skip_premailer: true) do |format|
+        format.html { render layout: false }
+        format.text
+      end
+    end
+
+    def add_headers
+      add_project_headers
+      add_pipeline_headers
+    end
+
+    def add_pipeline_headers
+      headers['X-GitLab-Pipeline-Id'] = @pipeline.id
+      headers['X-GitLab-Pipeline-Ref'] = @pipeline.ref
+      headers['X-GitLab-Pipeline-Status'] = @pipeline.status
+    end
+
+    def pipeline_subject(status)
+      commit = @pipeline.short_sha
+      commit << " in #{@merge_request.to_reference}" if @merge_request
+
+      subject("Pipeline ##{@pipeline.id} has #{status} for #{@pipeline.ref}", commit)
+    end
+  end
+end

app/mailers/notify.rb

@@ -7,6 +7,7 @@ class Notify < BaseMailer
   include Emails::Projects
   include Emails::Profile
   include Emails::Builds
+  include Emails::Pipelines
   include Emails::Members
 
   add_template_helper MergeRequestsHelper

app/models/ci/build.rb

@@ -133,13 +133,17 @@ module Ci
       latest_builds.where('stage_idx < ?', stage_idx)
     end
 
-    def trace_html
-      trace_with_state[:html] || ''
+    def trace_html(**args)
+      trace_with_state(**args)[:html] || ''
     end
 
-    def trace_with_state(state = nil)
-      trace_with_state = Ci::Ansi2html::convert(trace, state) if trace.present?
-      trace_with_state || {}
+    def trace_with_state(state: nil, last_lines: nil)
+      trace_ansi = trace(last_lines: last_lines)
+      if trace_ansi.present?
+        Ci::Ansi2html.convert(trace_ansi, state)
+      else
+        {}
+      end
     end
 
     def timeout
@@ -222,9 +226,10 @@ module Ci
       raw_trace.present?
     end
 
-    def raw_trace
+    def raw_trace(last_lines: nil)
       if File.exist?(trace_file_path)
-        File.read(trace_file_path)
+        Gitlab::Ci::TraceReader.new(trace_file_path).
+          read(last_lines: last_lines)
       else
         # backward compatibility
         read_attribute :trace
@@ -239,8 +244,8 @@ module Ci
       project.ci_id && File.exist?(old_path_to_trace)
     end
 
-    def trace
-      hide_secrets(raw_trace)
+    def trace(last_lines: nil)
+      hide_secrets(raw_trace(last_lines: last_lines))
     end
 
     def trace_length

app/models/project.rb

@@ -76,6 +76,7 @@ class Project < ActiveRecord::Base
   has_one :drone_ci_service, dependent: :destroy
   has_one :emails_on_push_service, dependent: :destroy
   has_one :builds_email_service, dependent: :destroy
+  has_one :pipelines_email_service, dependent: :destroy
   has_one :irker_service, dependent: :destroy
   has_one :pivotaltracker_service, dependent: :destroy
   has_one :hipchat_service, dependent: :destroy
@@ -718,7 +719,7 @@ class Project < ActiveRecord::Base
 
         if template.nil?
           # If no template, we should create an instance. Ex `create_gitlab_ci_service`
-          self.send :"create_#{service_name}_service"
+          public_send("create_#{service_name}_service")
         else
           Service.create_from_template(self.id, template)
         end

app/models/project_services/builds_email_service.rb

@@ -43,7 +43,7 @@ class BuildsEmailService < Service
   end
 
   def can_test?
-    project.builds.count > 0
+    project.builds.any?
   end
 
   def disabled_title

app/models/project_services/pipelines_email_service.rb

+class PipelinesEmailService < Service
+  prop_accessor :recipients
+  boolean_accessor :add_pusher
+  boolean_accessor :notify_only_broken_pipelines
+  validates :recipients,
+    presence: true,
+    if: ->(s) { s.activated? && !s.add_pusher? }
+
+  def initialize_properties
+    self.properties ||= { notify_only_broken_pipelines: true }
+  end
+
+  def title
+    'Pipelines emails'
+  end
+
+  def description
+    'Email the pipelines status to a list of recipients.'
+  end
+
+  def to_param
+    'pipelines_email'
+  end
+
+  def supported_events
+    %w[pipeline]
+  end
+
+  def execute(data, force: false)
+    return unless supported_events.include?(data[:object_kind])
+    return unless force || should_pipeline_be_notified?(data)
+
+    all_recipients = retrieve_recipients(data)
+
+    return unless all_recipients.any?
+
+    pipeline = Ci::Pipeline.find(data[:object_attributes][:id])
+    Ci::SendPipelineNotificationService.new(pipeline).execute(all_recipients)
+  end
+
+  def can_test?
+    project.pipelines.any?
+  end
+
+  def disabled_title
+    'Please setup a pipeline on your repository.'
+  end
+
+  def test_data(project, user)
+    data = Gitlab::DataBuilder::Pipeline.build(project.pipelines.last)
+    data[:user] = user.hook_attrs
+    data
+  end
+
+  def fields
+    [
+      { type: 'textarea',
+        name: 'recipients',
+        placeholder: 'Emails separated by comma' },
+      { type: 'checkbox',
+        name: 'add_pusher',
+        label: 'Add pusher to recipients list' },
+      { type: 'checkbox',
+        name: 'notify_only_broken_pipelines' },
+    ]
+  end
+
+  def test(data)
+    result = execute(data, force: true)
+
+    { success: true, result: result }
+  rescue StandardError => error
+    { success: false, result: error }
+  end
+
+  def should_pipeline_be_notified?(data)
+    case data[:object_attributes][:status]
+    when 'success'
+      !notify_only_broken_pipelines?
+    when 'failed'
+      true
+    else
+      false
+    end
+  end
+
+  def retrieve_recipients(data)
+    all_recipients = recipients.to_s.split(',').reject(&:blank?)
+
+    if add_pusher? && data[:user].try(:[], :email)
+      all_recipients << data[:user][:email]
+    end
+
+    all_recipients
+  end
+end

app/models/service.rb

@@ -196,12 +196,13 @@ class Service < ActiveRecord::Base
   end
 
   def self.available_services_names
-    %w(
+    %w[
       asana
       assembla
       bamboo
       buildkite
       builds_email
+      pipelines_email
       bugzilla
       campfire
       custom_issue_tracker
@@ -218,7 +219,7 @@ class Service < ActiveRecord::Base
       redmine
       slack
       teamcity
-    )
+    ]
   end
 
   def self.create_from_template(project_id, template)

app/services/ci/send_pipeline_notification_service.rb

+module Ci
+  class SendPipelineNotificationService
+    attr_reader :pipeline
+
+    def initialize(new_pipeline)
+      @pipeline = new_pipeline
+    end
+
+    def execute(recipients)
+      email_template = "pipeline_#{pipeline.status}_email"
+
+      return unless Notify.respond_to?(email_template)
+
+      recipients.each do |to|
+        Notify.public_send(email_template, pipeline, to).deliver_later
+      end
+    end
+  end
+end

app/views/notify/pipeline_failed_email.text.erb

+Your pipeline has failed.
+
+Project: <%= @project.name %> ( <%= project_url(@project) %> )
+Branch: <%= @pipeline.ref %> ( <%= commits_url(@pipeline) %> )
+<% if @merge_request -%>
+Merge Request: <%= @merge_request.to_reference %> ( <%= merge_request_url(@merge_request) %> )
+<% end -%>
+
+Commit: <%= @pipeline.short_sha %> ( <%= commit_url(@pipeline) %> )
+Commit Message: <%= @pipeline.git_commit_message.truncate(50) %>
+<% commit = @pipeline.commit -%>
+<% if commit.author -%>
+Commit Author: <%= commit.author.name %> ( <%= user_url(commit.author) %> )
+<% else -%>
+Commit Author: <%= commit.author_name %>
+<% end -%>
+
+<% failed = @pipeline.statuses.latest.failed -%>
+Pipeline #<%= @pipeline.id %> ( <%= pipeline_url(@pipeline) %> ) had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
+
+<% failed.each do |build| -%>
+Build #<%= build.id %> ( <%= pipeline_build_url(@pipeline, build) %> )
+Stage: <%= build.stage %>
+Name: <%= build.name %>
+Trace: <%= build.trace_with_state(last_lines: 10)[:text] %>
+
+<% end -%>
+
+You're receiving this email because of your account on <%= Gitlab.config.gitlab.host %>.
+Manage all notifications: <%= profile_notifications_url %>
+Help: <%= help_url %>

app/views/notify/pipeline_success_email.text.erb

+Your pipeline has passed.
+
+Project: <%= @project.name %> ( <%= project_url(@project) %> )
+Branch: <%= @pipeline.ref %> ( <%= commits_url(@pipeline) %> )
+<% if @merge_request -%>
+Merge Request: <%= @merge_request.to_reference %> ( <%= merge_request_url(@merge_request) %> )
+<% end -%>
+
+Commit: <%= @pipeline.short_sha %> ( <%= commit_url(@pipeline) %> )
+Commit Message: <%= @pipeline.git_commit_message.truncate(50) %>
+<% commit = @pipeline.commit -%>
+<% if commit.author -%>
+Commit Author: <%= commit.author.name %> ( <%= user_url(commit.author) %> )
+<% else -%>
+Commit Author: <%= commit.author_name %>
+<% end -%>
+
+<% build_count = @pipeline.statuses.latest.size -%>
+<% stage_count = @pipeline.stages.size -%>
+Pipeline #<%= @pipeline.id %> ( <%= pipeline_url(@pipeline) %> ) successfully completed <%= build_count %> <%= 'build'.pluralize(build_count) %> in <%= stage_count %> <%= 'stage'.pluralize(stage_count) %>.
+
+You're receiving this email because of your account on <%= Gitlab.config.gitlab.host %>.
+Manage all notifications: <%= profile_notifications_url %>
+Help: <%= help_url %>

lib/gitlab/ci/trace_reader.rb

+module Gitlab
+  module Ci
+    # This was inspired from: http://stackoverflow.com/a/10219411/1520132
+    class TraceReader
+      BUFFER_SIZE = 4096
+
+      attr_accessor :path, :buffer_size
+
+      def initialize(new_path, buffer_size: BUFFER_SIZE)
+        self.path = new_path
+        self.buffer_size = Integer(buffer_size)
+      end
+
+      def read(last_lines: nil)
+        if last_lines
+          read_last_lines(last_lines)
+        else
+          File.read(path)
+        end
+      end
+
+      def read_last_lines(max_lines)
+        File.open(path) do |file|
+          chunks = []
+          pos = lines = 0
+          max = file.size
+
+          # We want an extra line to make sure fist line has full contents
+          while lines <= max_lines && pos < max
+            pos += buffer_size
+
+            buf = if pos <= max
+                    file.seek(-pos, IO::SEEK_END)
+                    file.read(buffer_size)
+                  else # Reached the head, read only left
+                    file.seek(0)
+                    file.read(buffer_size - (pos - max))
+                  end
+
+            lines += buf.count("\n")
+            chunks.unshift(buf)
+          end
+
+          chunks.join.lines.last(max_lines).join
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/trace_reader_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::TraceReader do
+  let(:path) { __FILE__ }
+  let(:lines) { File.readlines(path) }
+  let(:bytesize) { lines.sum(&:bytesize) }
+
+  it 'returns last few lines' do
+    10.times do
+      subject = build_subject
+      last_lines = random_lines
+
+      expected = lines.last(last_lines).join
+
+      expect(subject.read(last_lines: last_lines)).to eq(expected)
+    end
+  end
+
+  it 'returns everything if trying to get too many lines' do
+    expect(build_subject.read(last_lines: lines.size * 2)).to eq(lines.join)
+  end
+
+  it 'raises an error if not passing an integer for last_lines' do
+    expect do
+      build_subject.read(last_lines: lines)
+    end.to raise_error(ArgumentError)
+  end
+
+  def random_lines
+    Random.rand(lines.size) + 1
+  end
+
+  def random_buffer
+    Random.rand(bytesize) + 1
+  end
+
+  def build_subject
+    described_class.new(__FILE__, buffer_size: random_buffer)
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -125,6 +125,7 @@ project:
 - drone_ci_service
 - emails_on_push_service
 - builds_email_service
+- pipelines_email_service
 - irker_service
 - pivotaltracker_service
 - hipchat_service
@@ -184,4 +185,4 @@ project:
 - project_feature
 award_emoji:
 - awardable
-- user
\ No newline at end of file
+- user

spec/models/merge_request_spec.rb

@@ -334,7 +334,7 @@ describe MergeRequest, models: true do
       wip_title = "WIP: #{subject.title}"
 
       expect(subject.wip_title).to eq wip_title
-    end 
+    end
 
     it "does not add the WIP: prefix multiple times" do
       wip_title = "WIP: #{subject.title}"

spec/models/project_services/pipeline_email_service_spec.rb

+require 'spec_helper'
+
+describe PipelinesEmailService do
+  let(:pipeline) do
+    create(:ci_pipeline, project: project, sha: project.commit('master').sha)
+  end
+
+  let(:project) { create(:project) }
+  let(:recipient) { 'test@gitlab.com' }
+
+  let(:data) do
+    Gitlab::DataBuilder::Pipeline.build(pipeline)
+  end
+
+  before do
+    ActionMailer::Base.deliveries.clear
+  end
+
+  describe 'Validations' do
+    context 'when service is active' do
+      before do
+        subject.active = true
+      end
+
+      it { is_expected.to validate_presence_of(:recipients) }
+
+      context 'when pusher is added' do
+        before do
+          subject.add_pusher = true
+        end
+
+        it { is_expected.not_to validate_presence_of(:recipients) }
+      end
+    end
+
+    context 'when service is inactive' do
+      before do
+        subject.active = false
+      end
+
+      it { is_expected.not_to validate_presence_of(:recipients) }
+    end
+  end
+
+  describe '#test_data' do
+    let(:build)   { create(:ci_build) }
+    let(:project) { build.project }
+    let(:user)    { create(:user) }
+
+    before do
+      project.team << [user, :developer]
+    end
+
+    it 'builds test data' do
+      data = subject.test_data(project, user)
+
+      expect(data[:object_kind]).to eq('pipeline')
+    end
+  end
+
+  shared_examples 'sending email' do
+    before do
+      perform_enqueued_jobs do
+        run
+      end
+    end
+
+    it 'sends email' do
+      sent_to = ActionMailer::Base.deliveries.flat_map(&:to)
+      expect(sent_to).to contain_exactly(recipient)
+    end
+  end
+
+  shared_examples 'not sending email' do
+    before do
+      perform_enqueued_jobs do
+        run
+      end
+    end
+
+    it 'does not send email' do
+      expect(ActionMailer::Base.deliveries).to be_empty
+    end
+  end
+
+  describe '#test' do
+    def run
+      subject.test(data)
+    end
+
+    before do
+      subject.recipients = recipient
+    end
+
+    context 'when pipeline is failed' do
+      before do
+        data[:object_attributes][:status] = 'failed'
+        pipeline.update(status: 'failed')
+      end
+
+      it_behaves_like 'sending email'
+    end
+
+    context 'when pipeline is succeeded' do
+      before do
+        data[:object_attributes][:status] = 'success'
+        pipeline.update(status: 'success')
+      end
+
+      it_behaves_like 'sending email'
+    end
+  end
+
+  describe '#execute' do
+    def run
+      subject.execute(data)
+    end
+
+    context 'with recipients' do
+      before do
+        subject.recipients = recipient
+      end
+
+      context 'with failed pipeline' do
+        before do
+          data[:object_attributes][:status] = 'failed'
+          pipeline.update(status: 'failed')
+        end
+
+        it_behaves_like 'sending email'
+      end
+
+      context 'with succeeded pipeline' do
+        before do
+          data[:object_attributes][:status] = 'success'
+          pipeline.update(status: 'success')
+        end
+
+        it_behaves_like 'not sending email'
+      end
+
+      context 'with notify_only_broken_pipelines on' do
+        before do
+          subject.notify_only_broken_pipelines = true
+        end
+
+        context 'with failed pipeline' do
+          before do
+            data[:object_attributes][:status] = 'failed'
+            pipeline.update(status: 'failed')
+          end
+
+          it_behaves_like 'sending email'
+        end
+
+        context 'with succeeded pipeline' do
+          before do
+            data[:object_attributes][:status] = 'success'
+            pipeline.update(status: 'success')
+          end
+
+          it_behaves_like 'not sending email'
+        end
+      end
+    end
+
+    context 'with empty recipients list' do
+      before do
+        subject.recipients = ' ,, '
+      end
+
+      context 'with failed pipeline' do
+        before do
+          data[:object_attributes][:status] = 'failed'
+          pipeline.update(status: 'failed')
+        end
+
+        it_behaves_like 'not sending email'
+      end
+    end
+  end
+end

spec/services/ci/send_pipeline_notification_service_spec.rb

+require 'spec_helper'
+
+describe Ci::SendPipelineNotificationService, services: true do
+  let(:pipeline) do
+    create(:ci_pipeline,
+           project: project,
+           sha: project.commit('master').sha,
+           user: user,
+           status: status)
+  end
+
+  let(:project) { create(:project) }
+  let(:user) { create(:user) }
+
+  subject{ described_class.new(pipeline) }
+
+  describe '#execute' do
+    before do
+      reset_delivered_emails!
+    end
+
+    shared_examples 'sending emails' do
+      it 'sends an email to pipeline user' do
+        perform_enqueued_jobs do
+          subject.execute([user.email])
+        end
+
+        email = ActionMailer::Base.deliveries.last
+        expect(email.subject).to include(email_subject)
+        expect(email.to).to eq([user.email])
+      end
+    end
+
+    context 'with success pipeline' do
+      let(:status) { 'success' }
+      let(:email_subject) { "Pipeline ##{pipeline.id} has succeeded" }
+
+      it_behaves_like 'sending emails'
+    end
+
+    context 'with failed pipeline' do
+      let(:status) { 'failed' }
+      let(:email_subject) { "Pipeline ##{pipeline.id} has failed" }
+
+      it_behaves_like 'sending emails'
+    end
+  end
+end