Merge remote-tracking branch 'upstream/master' into 36089-handle-ref-failure-better

* upstream/master: (47 commits)
  Update CHANGELOG.md for 9.4.5
  Update charlock_holmes
  add a changelog entry
  switch to multi-line before block
  restructure the #new_key notification spec
  don't send devise notifications to the ghost user
  reset_delivered_emails before testing #new_key
  skip the :read_project check for new_project_member
  move the member spec to be with the other ones
  add a spec for new_group_member
  add a spec for never emailing the ghost user
  rubocop fix
  a membership with no user is always notifiable
  check notifiability for more emails
  add Member#notifiable?(type, opts)
  make NotificationRecipient a little more customizable
  Add notes about database performance for MySQL
  fix confidential border issue as well as confidential styles leaking on new MR
  Migrate force push check to Gitaly
  Add option to disable project export on instance
  ...

.rubocop.yml

@@ -1045,7 +1045,7 @@ RSpec/BeforeAfterAll:
 RSpec/DescribeClass:
   Enabled: false
 
-# Use `described_class` for tested class / module.
+# Checks that the second argument to `describe` specifies a method.
 RSpec/DescribeMethod:
   Enabled: false
 
@@ -1053,8 +1053,7 @@ RSpec/DescribeMethod:
 RSpec/DescribeSymbol:
   Enabled: true
 
-# Checks that the second argument to top level describe is the tested method
-# name.
+# Checks that tests use `described_class`.
 RSpec/DescribedClass:
   Enabled: true
 

CHANGELOG.md

@@ -2,6 +2,27 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 9.4.5 (2017-08-14)
+
+- Fix deletion of deploy keys linked to other projects. !13162
+- Allow any logged in users to read_users_list even if it's restricted. !13201
+- Make Delete Merged Branches handle wildcard protected branches correctly. !13251
+- Fix an order of operations for CI connection error message in merge request widget. !13252
+- Fix pipeline_schedules pages when active schedule has an abnormal state. !13286
+- Add missing validation error for username change with container registry tags. !13356
+- Fix destroy of case-insensitive conflicting redirects. !13357
+- Project pending delete no longer return 500 error in admins projects view. !13389
+- Fix search box losing focus when typing.
+- Use jQuery to control scroll behavior in job log for cross browser consistency.
+- Use project_ref_path to create the link to a branch to fix links that 404.
+- improve file upload/replace experience.
+- fix jump to next discussion button.
+- Fixes new issue button for failed job returning 404.
+- Fix links to group milestones from issue and merge request sidebar.
+- Fixed sign-in restrictions buttons not toggling active state.
+- Fix Mattermost integration.
+- Change project FK migration to skip existing FKs.
+
 ## 9.4.4 (2017-08-09)
 
 - Remove hidden symlinks from project import files.

GITALY_SERVER_VERSION

-0.29.0
+0.30.0

Gemfile

@@ -402,7 +402,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly', '~> 0.26.0'
+gem 'gitaly', '~> 0.27.0'
 
 gem 'toml-rb', '~> 0.3.15', require: false
 

Gemfile.lock

@@ -113,7 +113,7 @@ GEM
       activesupport (>= 4.0.0)
       mime-types (>= 1.16)
     cause (0.1)
-    charlock_holmes (0.7.3)
+    charlock_holmes (0.7.4)
     chronic (0.10.2)
     chronic_duration (0.10.6)
       numerizer (~> 0.1.1)
@@ -270,7 +270,7 @@ GEM
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
     gherkin-ruby (0.3.2)
-    gitaly (0.26.0)
+    gitaly (0.27.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.0)
     github-linguist (4.7.6)
@@ -984,7 +984,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.2.0)
-  gitaly (~> 0.26.0)
+  gitaly (~> 0.27.0)
   github-linguist (~> 4.7.0)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-markup (~> 1.5.1)

app/assets/javascripts/dispatcher.js

@@ -347,6 +347,9 @@ import initChangesDropdown from './init_changes_dropdown';
           if ($('#tree-slider').length) new TreeView();
           if ($('.blob-viewer').length) new BlobViewer();
           if ($('.project-show-activity').length) new gl.Activities();
+          $('#tree-slider').waitForImages(function() {
+            gl.utils.ajaxGet(document.querySelector('.js-tree-content').dataset.logsPath);
+          });
           break;
         case 'projects:edit':
           setupProjectEdit();

app/assets/javascripts/projects/project_new.js

@@ -36,7 +36,7 @@ const bindEvents = () => {
 
   $('.how_to_import_link').on('click', (e) => {
     e.preventDefault();
-    $('.how_to_import_link').next('.modal').show();
+    $(e.currentTarget).next('.modal').show();
   });
 
   $('.modal-header .close').on('click', () => {

app/assets/stylesheets/framework/dropdowns.scss

@@ -725,9 +725,9 @@
 }
 
 // TODO: change global style and remove mixin
-@mixin new-style-dropdown {
-  .dropdown-menu,
-  .dropdown-menu-nav {
+@mixin new-style-dropdown($selector: '') {
+  #{$selector}.dropdown-menu,
+  #{$selector}.dropdown-menu-nav {
     .divider {
       margin: 6px 0;
     }
@@ -773,7 +773,7 @@
     }
   }
 
-  .dropdown-menu-align-right {
+  #{$selector}.dropdown-menu-align-right {
     margin-top: 2px;
   }
 }

app/assets/stylesheets/framework/typography.scss

@@ -339,6 +339,8 @@ a > code {
   @extend .ref-name;
 }
 
+@include new-style-dropdown('.git-revision-dropdown');
+
 /**
  * Apply Markdown typography
  *

app/assets/stylesheets/pages/note_form.scss

@@ -108,6 +108,7 @@
   background-color: $orange-50;
   border-radius: $border-radius-default $border-radius-default 0 0;
   border: 1px solid $border-gray-normal;
+  border-bottom: none;
   padding: 3px 12px;
   margin: auto;
   align-items: center;
@@ -132,22 +133,9 @@
   }
 }
 
-.not-confidential {
-  padding: 0;
-  border-top: none;
-}
-
-.right-sidebar-expanded {
-  .md-area {
-    border-radius: 0;
-    border-top: none;
-  }
-}
-
-.right-sidebar-collapsed {
-  .confidential-issue-warning {
-    border-bottom: none;
-  }
+.confidential-issue-warning + .md-area {
+  border-top-left-radius: 0;
+  border-top-right-radius: 0;
 }
 
 .discussion-form {

app/assets/stylesheets/pages/pipelines.scss

@@ -824,6 +824,7 @@ button.mini-pipeline-graph-dropdown-toggle {
  * Top arrow in the dropdown in the mini pipeline graph
  */
 .mini-pipeline-graph-dropdown-menu {
+  z-index: 200;
 
   &::before,
   &::after {

app/controllers/admin/appearances_controller.rb

@@ -45,7 +45,7 @@ class Admin::AppearancesController < Admin::ApplicationController
 
   # Use callbacks to share common setup or constraints between actions.
   def set_appearance
-    @appearance = Appearance.last || Appearance.new
+    @appearance = Appearance.current || Appearance.new
   end
 
   # Only allow a trusted parameter "white list" through.

app/controllers/concerns/authenticates_with_two_factor.rb

@@ -69,7 +69,7 @@ module AuthenticatesWithTwoFactor
     if U2fRegistration.authenticate(user, u2f_app_id, user_params[:device_response], session[:challenge])
       # Remove any lingering user data from login
       session.delete(:otp_user_id)
-      session.delete(:challenges)
+      session.delete(:challenge)
 
       remember_me(user) if user_params[:remember_me] == '1'
       sign_in(user)

app/controllers/dashboard/projects_controller.rb

@@ -52,8 +52,10 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController
   end
 
   def load_events
-    @events = Event.in_projects(load_projects(params.merge(non_public: true)))
-    @events = event_filter.apply_filter(@events).with_associations
-    @events = @events.limit(20).offset(params[:offset] || 0)
+    projects = load_projects(params.merge(non_public: true))
+
+    @events = EventCollection
+      .new(projects, offset: params[:offset].to_i, filter: event_filter)
+      .to_a
   end
 end

app/controllers/dashboard_controller.rb

@@ -29,9 +29,9 @@ class DashboardController < Dashboard::ApplicationController
         current_user.authorized_projects
       end
 
-    @events = Event.in_projects(projects)
-    @events = @event_filter.apply_filter(@events).with_associations
-    @events = @events.limit(20).offset(params[:offset] || 0)
+    @events = EventCollection
+      .new(projects, offset: params[:offset].to_i, filter: @event_filter)
+      .to_a
   end
 
   def set_show_full_reference

app/controllers/groups_controller.rb

@@ -160,9 +160,9 @@ class GroupsController < Groups::ApplicationController
   end
 
   def load_events
-    @events = Event.in_projects(@projects)
-    @events = event_filter.apply_filter(@events).with_associations
-    @events = @events.limit(20).offset(params[:offset] || 0)
+    @events = EventCollection
+      .new(@projects, offset: params[:offset].to_i, filter: event_filter)
+      .to_a
   end
 
   def user_actions

app/controllers/projects_controller.rb

@@ -7,6 +7,7 @@ class ProjectsController < Projects::ApplicationController
   before_action :repository, except: [:index, :new, :create]
   before_action :assign_ref_vars, only: [:show], if: :repo_exists?
   before_action :tree, only: [:show], if: [:repo_exists?, :project_view_files?]
+  before_action :project_export_enabled, only: [:export, :download_export, :remove_export, :generate_new_export]
 
   # Authorize
   before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping, :download_export, :export, :remove_export, :generate_new_export]
@@ -301,10 +302,11 @@ class ProjectsController < Projects::ApplicationController
   end
 
   def load_events
-    @events = @project.events.recent
-    @events = event_filter.apply_filter(@events).with_associations
-    limit = (params[:limit] || 20).to_i
-    @events = @events.limit(limit).offset(params[:offset] || 0)
+    projects = Project.where(id: @project.id)
+
+    @events = EventCollection
+      .new(projects, offset: params[:offset].to_i, filter: event_filter)
+      .to_a
   end
 
   def project_params
@@ -389,4 +391,8 @@ class ProjectsController < Projects::ApplicationController
 
     url_for(params)
   end
+
+  def project_export_enabled
+    render_404 unless current_application_settings.project_export_enabled?
+  end
 end

app/controllers/unicorn_test_controller.rb

+# :nocov:
 if Rails.env.test?
   class UnicornTestController < ActionController::Base
     def pid
       render plain: Process.pid.to_s
     end
-  
+
     def kill
       Process.kill(params[:signal], Process.pid)
       render plain: 'Bye!'
     end
   end
 end
+# :nocov:

app/finders/admin/projects_finder.rb

@@ -18,7 +18,7 @@ class Admin::ProjectsFinder
   end
 
   def execute
-    items = Project.with_statistics
+    items = Project.without_deleted.with_statistics
     items = items.in_namespace(namespace_id) if namespace_id.present?
     items = items.where(visibility_level: visibility_level) if visibility_level.present?
     items = items.with_push if with_push.present?

app/helpers/appearances_helper.rb

@@ -20,7 +20,7 @@ module AppearancesHelper
   end
 
   def brand_item
-    @appearance ||= Appearance.first
+    @appearance ||= Appearance.current
   end
 
   def brand_header_logo

app/helpers/application_settings_helper.rb

@@ -146,6 +146,7 @@ module ApplicationSettingsHelper
       :plantuml_enabled,
       :plantuml_url,
       :polling_interval_multiplier,
+      :project_export_enabled,
       :prometheus_metrics_enabled,
       :recaptcha_enabled,
       :recaptcha_private_key,

app/models/appearance.rb

@@ -8,7 +8,27 @@ class Appearance < ActiveRecord::Base
   validates :logo,        file_size: { maximum: 1.megabyte }
   validates :header_logo, file_size: { maximum: 1.megabyte }
 
+  validate :single_appearance_row, on: :create
+
   mount_uploader :logo,         AttachmentUploader
   mount_uploader :header_logo,  AttachmentUploader
   has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+
+  CACHE_KEY = 'current_appearance'.freeze
+
+  after_commit :flush_redis_cache
+
+  def self.current
+    Rails.cache.fetch(CACHE_KEY) { first }
+  end
+
+  def flush_redis_cache
+    Rails.cache.delete(CACHE_KEY)
+  end
+
+  def single_appearance_row
+    if self.class.any?
+      errors.add(:single_appearance_row, 'Only 1 appearances row can exist')
+    end
+  end
 end

app/models/application_setting.rb

@@ -241,6 +241,7 @@ class ApplicationSetting < ActiveRecord::Base
       performance_bar_allowed_group_id: nil,
       plantuml_enabled: false,
       plantuml_url: nil,
+      project_export_enabled: true,
       recaptcha_enabled: false,
       repository_checks_enabled: true,
       repository_storages: ['default'],

app/models/broadcast_message.rb

@@ -14,9 +14,15 @@ class BroadcastMessage < ActiveRecord::Base
   default_value_for :color, '#E75E40'
   default_value_for :font,  '#FFFFFF'
 
+  CACHE_KEY = 'broadcast_message_current'.freeze
+
+  after_commit :flush_redis_cache
+
   def self.current
-    Rails.cache.fetch("broadcast_message_current", expires_in: 1.minute) do
-      where('ends_at > :now AND starts_at <= :now', now: Time.zone.now).order([:created_at, :id]).to_a
+    Rails.cache.fetch(CACHE_KEY) do
+      where('ends_at > :now AND starts_at <= :now', now: Time.zone.now)
+        .reorder(id: :asc)
+        .to_a
     end
   end
 
@@ -31,4 +37,8 @@ class BroadcastMessage < ActiveRecord::Base
   def ended?
     ends_at < Time.zone.now
   end
+
+  def flush_redis_cache
+    Rails.cache.delete(CACHE_KEY)
+  end
 end

app/models/event.rb

@@ -48,6 +48,7 @@ class Event < ActiveRecord::Base
   belongs_to :author, class_name: "User"
   belongs_to :project
   belongs_to :target, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
+  has_one :push_event_payload, foreign_key: :event_id
 
   # For Hash only
   serialize :data # rubocop:disable Cop/ActiveRecordSerialize
@@ -55,19 +56,51 @@ class Event < ActiveRecord::Base
   # Callbacks
   after_create :reset_project_activity
   after_create :set_last_repository_updated_at, if: :push?
+  after_create :replicate_event_for_push_events_migration
 
   # Scopes
   scope :recent, -> { reorder(id: :desc) }
   scope :code_push, -> { where(action: PUSHED) }
 
-  scope :in_projects, ->(projects) do
-    where(project_id: projects.pluck(:id)).recent
+  scope :in_projects, -> (projects) do
+    sub_query = projects
+      .except(:order)
+      .select(1)
+      .where('projects.id = events.project_id')
+
+    where('EXISTS (?)', sub_query).recent
+  end
+
+  scope :with_associations, -> do
+    # We're using preload for "push_event_payload" as otherwise the association
+    # is not always available (depending on the query being built).
+    includes(:author, :project, project: :namespace)
+      .preload(:target, :push_event_payload)
   end
 
-  scope :with_associations, -> { includes(:author, :project, project: :namespace).preload(:target) }
   scope :for_milestone_id, ->(milestone_id) { where(target_type: "Milestone", target_id: milestone_id) }
 
+  self.inheritance_column = 'action'
+
   class << self
+    def find_sti_class(action)
+      if action.to_i == PUSHED
+        PushEvent
+      else
+        Event
+      end
+    end
+
+    def subclass_from_attributes(attrs)
+      # Without this Rails will keep calling this method on the returned class,
+      # resulting in an infinite loop.
+      return unless self == Event
+
+      action = attrs.with_indifferent_access[inheritance_column].to_i
+
+      PushEvent if action == PUSHED
+    end
+
     # Update Gitlab::ContributionsCalendar#activity_dates if this changes
     def contributions
       where("action = ? OR (target_type IN (?) AND action IN (?)) OR (target_type = ? AND action = ?)",
@@ -290,6 +323,16 @@ class Event < ActiveRecord::Base
     @commits ||= (data[:commits] || []).reverse
   end
 
+  def commit_title
+    commit = commits.last
+
+    commit[:message] if commit
+  end
+
+  def commit_id
+    commit_to || commit_from
+  end
+
   def commits_count
     data[:total_commits_count] || commits.count || 0
   end
@@ -385,6 +428,16 @@ class Event < ActiveRecord::Base
     user ? author_id == user.id : false
   end
 
+  # We're manually replicating data into the new table since database triggers
+  # are not dumped to db/schema.rb. This could mean that a new installation
+  # would not have the triggers in place, thus losing events data in GitLab
+  # 10.0.
+  def replicate_event_for_push_events_migration
+    new_attributes = attributes.with_indifferent_access.except(:title, :data)
+
+    EventForMigration.create!(new_attributes)
+  end
+
   private
 
   def recent_update?

app/models/event_collection.rb

+# A collection of events to display in an event list.
+#
+# An EventCollection is meant to be used for displaying events to a user (e.g.
+# in a controller), it's not suitable for building queries that are used for
+# building other queries.
+class EventCollection
+  # To prevent users from putting too much pressure on the database by cycling
+  # through thousands of events we put a limit on the number of pages.
+  MAX_PAGE = 10
+
+  # projects - An ActiveRecord::Relation object that returns the projects for
+  #            which to retrieve events.
+  # filter - An EventFilter instance to use for filtering events.
+  def initialize(projects, limit: 20, offset: 0, filter: nil)
+    @projects = projects
+    @limit = limit
+    @offset = offset
+    @filter = filter
+  end
+
+  # Returns an Array containing the events.
+  def to_a
+    return [] if current_page > MAX_PAGE
+
+    relation = if Gitlab::Database.join_lateral_supported?
+                 relation_with_join_lateral
+               else
+                 relation_without_join_lateral
+               end
+
+    relation.with_associations.to_a
+  end
+
+  private
+
+  # Returns the events relation to use when JOIN LATERAL is not supported.
+  #
+  # This relation simply gets all the events for all authorized projects, then
+  # limits that set.
+  def relation_without_join_lateral
+    events = filtered_events.in_projects(projects)
+
+    paginate_events(events)
+  end
+
+  # Returns the events relation to use when JOIN LATERAL is supported.
+  #
+  # This relation is built using JOIN LATERAL, producing faster queries than a
+  # regular LIMIT + OFFSET approach.
+  def relation_with_join_lateral
+    projects_for_lateral = projects.select(:id).to_sql
+
+    lateral = filtered_events
+      .limit(limit_for_join_lateral)
+      .where('events.project_id = projects_for_lateral.id')
+      .to_sql
+
+    # The outer query does not need to re-apply the filters since the JOIN
+    # LATERAL body already takes care of this.
+    outer = base_relation
+      .from("(#{projects_for_lateral}) projects_for_lateral")
+      .joins("JOIN LATERAL (#{lateral}) AS #{Event.table_name} ON true")
+
+    paginate_events(outer)
+  end
+
+  def filtered_events
+    @filter ? @filter.apply_filter(base_relation) : base_relation
+  end
+
+  def paginate_events(events)
+    events.limit(@limit).offset(@offset)
+  end
+
+  def base_relation
+    # We want to have absolute control over the event queries being built, thus
+    # we're explicitly opting out of any default scopes that may be set.
+    Event.unscoped.recent
+  end
+
+  def limit_for_join_lateral
+    # Applying the OFFSET on the inside of a JOIN LATERAL leads to incorrect
+    # results. To work around this we need to increase the inner limit for every
+    # page.
+    #
+    # This means that on page 1 we use LIMIT 20, and an outer OFFSET of 0. On
+    # page 2 we use LIMIT 40 and an outer OFFSET of 20.
+    @limit + @offset
+  end
+
+  def current_page
+    (@offset / @limit) + 1
+  end
+
+  def projects
+    @projects.except(:order)
+  end
+end

app/models/event_for_migration.rb

+# This model is used to replicate events between the old "events" table and the
+# new "events_for_migration" table that will replace "events" in GitLab 10.0.
+class EventForMigration < ActiveRecord::Base
+  self.table_name = 'events_for_migration'
+end

app/models/member.rb

@@ -276,6 +276,13 @@ class Member < ActiveRecord::Base
     @notification_setting ||= user.notification_settings_for(source)
   end
 
+  def notifiable?(type, opts = {})
+    # always notify when there isn't a user yet
+    return true if user.blank?
+
+    NotificationRecipientService.notifiable?(user, type, notifiable_options.merge(opts))
+  end
+
   private
 
   def send_invite
@@ -332,4 +339,8 @@ class Member < ActiveRecord::Base
   def notification_service
     NotificationService.new
   end
+
+  def notifiable_options
+    {}
+  end
 end

app/models/members/group_member.rb

@@ -30,6 +30,10 @@ class GroupMember < Member
     'Group'
   end
 
+  def notifiable_options
+    { group: group }
+  end
+
   private
 
   def send_invite

app/models/members/project_member.rb

@@ -87,6 +87,10 @@ class ProjectMember < Member
     project.owner == user
   end
 
+  def notifiable_options
+    { project: project }
+  end
+
   private
 
   def delete_member_todos

app/models/notification_recipient.rb

@@ -5,14 +5,22 @@ class NotificationRecipient
     custom_action: nil,
     target: nil,
     acting_user: nil,
-    project: nil
+    project: nil,
+    group: nil,
+    skip_read_ability: false
   )
+    unless NotificationSetting.levels.key?(type) || type == :subscription
+      raise ArgumentError, "invalid type: #{type.inspect}"
+    end
+
     @custom_action = custom_action
     @acting_user = acting_user
     @target = target
-    @project = project || @target&.project
+    @project = project || default_project
+    @group = group || @project&.group
     @user = user
     @type = type
+    @skip_read_ability = skip_read_ability
   end
 
   def notification_setting
@@ -77,6 +85,8 @@ class NotificationRecipient
   def has_access?
     DeclarativePolicy.subject_scope do
       return false unless user.can?(:receive_notifications)
+      return true if @skip_read_ability
+
       return false if @project && !user.can?(:read_project, @project)
 
       return true unless read_ability
@@ -96,6 +106,7 @@ class NotificationRecipient
   private
 
   def read_ability
+    return nil if @skip_read_ability
     return @read_ability if instance_variable_defined?(:@read_ability)
 
     @read_ability =
@@ -111,12 +122,18 @@ class NotificationRecipient
       end
   end
 
+  def default_project
+    return nil if @target.nil?
+    return @target if @target.is_a?(Project)
+    return @target.project if @target.respond_to?(:project)
+  end
+
   def find_notification_setting
     project_setting = @project && user.notification_settings_for(@project)
 
     return project_setting unless project_setting.nil? || project_setting.global?
 
-    group_setting = @project&.group && user.notification_settings_for(@project.group)
+    group_setting = @group && user.notification_settings_for(@group)
 
     return group_setting unless group_setting.nil? || group_setting.global?
 

app/models/push_event.rb

+class PushEvent < Event
+  # This validation exists so we can't accidentally use PushEvent with a
+  # different "action" value.
+  validate :validate_push_action
+
+  # Authors are required as they're used to display who pushed data.
+  #
+  # We're just validating the presence of the ID here as foreign key constraints
+  # should ensure the ID points to a valid user.
+  validates :author_id, presence: true
+
+  # The project is required to build links to commits, commit ranges, etc.
+  #
+  # We're just validating the presence of the ID here as foreign key constraints
+  # should ensure the ID points to a valid project.
+  validates :project_id, presence: true
+
+  # The "data" field must not be set for push events since it's not used and a
+  # waste of space.
+  validates :data, absence: true
+
+  # These fields are also not used for push events, thus storing them would be a
+  # waste.
+  validates :target_id, absence: true
+  validates :target_type, absence: true
+
+  def self.sti_name
+    PUSHED
+  end
+
+  def push?
+    true
+  end
+
+  def push_with_commits?
+    !!(commit_from && commit_to)
+  end
+
+  def tag?
+    return super unless push_event_payload
+
+    push_event_payload.tag?
+  end
+
+  def branch?
+    return super unless push_event_payload
+
+    push_event_payload.branch?
+  end
+
+  def valid_push?
+    return super unless push_event_payload
+
+    push_event_payload.ref.present?
+  end
+
+  def new_ref?
+    return super unless push_event_payload
+
+    push_event_payload.created?
+  end
+
+  def rm_ref?
+    return super unless push_event_payload
+
+    push_event_payload.removed?
+  end
+
+  def commit_from
+    return super unless push_event_payload
+
+    push_event_payload.commit_from
+  end
+
+  def commit_to
+    return super unless push_event_payload
+
+    push_event_payload.commit_to
+  end
+
+  def ref_name
+    return super unless push_event_payload
+
+    push_event_payload.ref
+  end
+
+  def ref_type
+    return super unless push_event_payload
+
+    push_event_payload.ref_type
+  end
+
+  def branch_name
+    return super unless push_event_payload
+
+    ref_name
+  end
+
+  def tag_name
+    return super unless push_event_payload
+
+    ref_name
+  end
+
+  def commit_title
+    return super unless push_event_payload
+
+    push_event_payload.commit_title
+  end
+
+  def commit_id
+    commit_to || commit_from
+  end
+
+  def commits_count
+    return super unless push_event_payload
+
+    push_event_payload.commit_count
+  end
+
+  def validate_push_action
+    return if action == PUSHED
+
+    errors.add(:action, "the action #{action.inspect} is not valid")
+  end
+end

app/models/push_event_payload.rb

+class PushEventPayload < ActiveRecord::Base
+  include ShaAttribute
+
+  belongs_to :event, inverse_of: :push_event_payload
+
+  validates :event_id, :commit_count, :action, :ref_type, presence: true
+  validates :commit_title, length: { maximum: 70 }
+
+  sha_attribute :commit_from
+  sha_attribute :commit_to
+
+  enum action: {
+    created: 0,
+    removed: 1,
+    pushed: 2
+  }
+
+  enum ref_type: {
+    branch: 0,
+    tag: 1
+  }
+end

app/models/user.rb

@@ -1069,6 +1069,7 @@ class User < ActiveRecord::Base
 
   # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration
   def send_devise_notification(notification, *args)
+    return true unless can?(:receive_notifications)
     devise_mailer.send(notification, self, *args).deliver_later
   end
 

app/services/event_create_service.rb

@@ -71,7 +71,14 @@ class EventCreateService
   end
 
   def push(project, current_user, push_data)
-    create_event(project, current_user, Event::PUSHED, data: push_data)
+    # We're using an explicit transaction here so that any errors that may occur
+    # when creating push payload data will result in the event creation being
+    # rolled back as well.
+    Event.transaction do
+      event = create_event(project, current_user, Event::PUSHED)
+
+      PushEventPayloadService.new(event, push_data).execute
+    end
 
     Users::ActivityService.new(current_user, 'push').execute
   end

app/services/notification_service.rb

@@ -10,9 +10,11 @@ class NotificationService
   # only if ssh key is not deploy key
   #
   # This is security email so it will be sent
-  # even if user disabled notifications
+  # even if user disabled notifications. However,
+  # it won't be sent to internal users like the
+  # ghost user or the EE support bot.
   def new_key(key)
-    if key.user
+    if key.user&.can?(:receive_notifications)
       mailer.new_ssh_key_email(key.id).deliver_later
     end
   end
@@ -22,14 +24,14 @@ class NotificationService
   # This is a security email so it will be sent even if the user user disabled
   # notifications
   def new_gpg_key(gpg_key)
-    if gpg_key.user
+    if gpg_key.user&.can?(:receive_notifications)
       mailer.new_gpg_key_email(gpg_key.id).deliver_later
     end
   end
 
   # Always notify user about email added to profile
   def new_email(email)
-    if email.user
+    if email.user&.can?(:receive_notifications)
       mailer.new_email_email(email.id).deliver_later
     end
   end
@@ -185,6 +187,8 @@ class NotificationService
 
   # Notify new user with email after creation
   def new_user(user, token = nil)
+    return true unless notifiable?(user, :mention)
+
     # Don't email omniauth created users
     mailer.new_user_email(user.id, token).deliver_later unless user.identities.any?
   end
@@ -206,19 +210,27 @@ class NotificationService
 
   # Members
   def new_access_request(member)
+    return true unless member.notifiable?(:subscription)
+
     mailer.member_access_requested_email(member.real_source_type, member.id).deliver_later
   end
 
   def decline_access_request(member)
+    return true unless member.notifiable?(:subscription)
+
     mailer.member_access_denied_email(member.real_source_type, member.source_id, member.user_id).deliver_later
   end
 
   # Project invite
   def invite_project_member(project_member, token)
+    return true unless project_member.notifiable?(:subscription)
+
     mailer.member_invited_email(project_member.real_source_type, project_member.id, token).deliver_later
   end
 
   def accept_project_invite(project_member)
+    return true unless project_member.notifiable?(:subscription)
+
     mailer.member_invite_accepted_email(project_member.real_source_type, project_member.id).deliver_later
   end
 
@@ -232,10 +244,14 @@ class NotificationService
   end
 
   def new_project_member(project_member)
+    return true unless project_member.notifiable?(:mention, skip_read_ability: true)
+
     mailer.member_access_granted_email(project_member.real_source_type, project_member.id).deliver_later
   end
 
   def update_project_member(project_member)
+    return true unless project_member.notifiable?(:mention)
+
     mailer.member_access_granted_email(project_member.real_source_type, project_member.id).deliver_later
   end
 
@@ -249,6 +265,9 @@ class NotificationService
   end
 
   def decline_group_invite(group_member)
+    # always send this one, since it's a response to the user's own
+    # action
+
     mailer.member_invite_declined_email(
       group_member.real_source_type,
       group_member.group.id,
@@ -258,15 +277,19 @@ class NotificationService
   end
 
   def new_group_member(group_member)
+    return true unless group_member.notifiable?(:mention)
+
     mailer.member_access_granted_email(group_member.real_source_type, group_member.id).deliver_later
   end
 
   def update_group_member(group_member)
+    return true unless group_member.notifiable?(:mention)
+
     mailer.member_access_granted_email(group_member.real_source_type, group_member.id).deliver_later
   end
 
   def project_was_moved(project, old_path_with_namespace)
-    recipients = NotificationRecipientService.notifiable_users(project.team.members, :mention, project: project)
+    recipients = notifiable_users(project.team.members, :mention, project: project)
 
     recipients.each do |recipient|
       mailer.project_was_moved_email(
@@ -288,10 +311,14 @@ class NotificationService
   end
 
   def project_exported(project, current_user)
+    return true unless notifiable?(current_user, :mention, project: project)
+
     mailer.project_was_exported_email(current_user, project).deliver_later
   end
 
   def project_not_exported(project, current_user, errors)
+    return true unless notifiable?(current_user, :mention, project: project)
+
     mailer.project_was_not_exported_email(current_user, project, errors).deliver_later
   end
 
@@ -300,7 +327,7 @@ class NotificationService
 
     return unless mailer.respond_to?(email_template)
 
-    recipients ||= NotificationRecipientService.notifiable_users(
+    recipients ||= notifiable_users(
       [pipeline.user], :watch,
       custom_action: :"#{pipeline.status}_pipeline",
       target: pipeline
@@ -369,7 +396,7 @@ class NotificationService
 
   def relabeled_resource_email(target, labels, current_user, method)
     recipients = labels.flat_map { |l| l.subscribers(target.project) }
-    recipients = NotificationRecipientService.notifiable_users(
+    recipients = notifiable_users(
       recipients, :subscription,
       target: target,
       acting_user: current_user
@@ -401,4 +428,14 @@ class NotificationService
       object.previous_changes[attribute].first
     end
   end
+
+  private
+
+  def notifiable?(*args)
+    NotificationRecipientService.notifiable?(*args)
+  end
+
+  def notifiable_users(*args)
+    NotificationRecipientService.notifiable_users(*args)
+  end
 end

app/services/push_event_payload_service.rb

+# Service class for creating push event payloads as stored in the
+# "push_event_payloads" table.
+#
+# Example:
+#
+#     data = Gitlab::DataBuilder::Push.build(...)
+#     event = Event.create(...)
+#
+#     PushEventPayloadService.new(event, data).execute
+class PushEventPayloadService
+  # event - The event this push payload belongs to.
+  # push_data - A Hash produced by `Gitlab::DataBuilder::Push.build` to use for
+  #             building the push payload.
+  def initialize(event, push_data)
+    @event = event
+    @push_data = push_data
+  end
+
+  # Creates and returns a new PushEventPayload row.
+  #
+  # This method will raise upon encountering validation errors.
+  #
+  # Returns an instance of PushEventPayload.
+  def execute
+    @event.build_push_event_payload(
+      commit_count: commit_count,
+      action: action,
+      ref_type: ref_type,
+      commit_from: commit_from_id,
+      commit_to: commit_to_id,
+      ref: trimmed_ref,
+      commit_title: commit_title,
+      event_id: @event.id
+    )
+
+    @event.push_event_payload.save!
+    @event.push_event_payload
+  end
+
+  # Returns the commit title to use.
+  #
+  # The commit title is limited to the first line and a maximum of 70
+  # characters.
+  def commit_title
+    commit = @push_data.fetch(:commits).last
+
+    return nil unless commit && commit[:message]
+
+    raw_msg = commit[:message]
+
+    # Find where the first line ends, without turning the entire message into an
+    # Array of lines (this is a waste of memory for large commit messages).
+    index = raw_msg.index("\n")
+    message = index ? raw_msg[0..index] : raw_msg
+
+    message.strip.truncate(70)
+  end
+
+  def commit_from_id
+    if create?
+      nil
+    else
+      revision_before
+    end
+  end
+
+  def commit_to_id
+    if remove?
+      nil
+    else
+      revision_after
+    end
+  end
+
+  def commit_count
+    @push_data.fetch(:total_commits_count)
+  end
+
+  def ref
+    @push_data.fetch(:ref)
+  end
+
+  def revision_before
+    @push_data.fetch(:before)
+  end
+
+  def revision_after
+    @push_data.fetch(:after)
+  end
+
+  def trimmed_ref
+    Gitlab::Git.ref_name(ref)
+  end
+
+  def create?
+    Gitlab::Git.blank_ref?(revision_before)
+  end
+
+  def remove?
+    Gitlab::Git.blank_ref?(revision_after)
+  end
+
+  def action
+    if create?
+      :created
+    elsif remove?
+      :removed
+    else
+      :pushed
+    end
+  end
+
+  def ref_type
+    if Gitlab::Git.tag_ref?(ref)
+      :tag
+    else
+      :branch
+    end
+  end
+end

app/uploaders/personal_file_uploader.rb

@@ -4,7 +4,7 @@ class PersonalFileUploader < FileUploader
   end
 
   def self.base_dir
-    File.join(root_dir, 'system')
+    File.join(root_dir, '-', 'system')
   end
 
   private

app/views/admin/application_settings/_form.html.haml

@@ -48,6 +48,12 @@
         = select(:application_setting, :enabled_git_access_protocol, [['Both SSH and HTTP(S)', nil], ['Only SSH', 'ssh'], ['Only HTTP(S)', 'http']], {}, class: 'form-control')
         %span.help-block#clone-protocol-help
           Allow only the selected protocols to be used for Git access.
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :project_export_enabled do
+            = f.check_box :project_export_enabled
+            Project export enabled
 
   %fieldset
     %legend Account and Limit Settings

app/views/events/_commit.html.haml

 %li.commit
   .commit-row-title
-    = link_to truncate_sha(commit[:id]), project_commit_path(project, commit[:id]), class: "commit-sha", alt: '', title: truncate_sha(commit[:id])
+    = link_to truncate_sha(event.commit_id), project_commit_path(project, event.commit_id), class: "commit-sha", alt: '', title: truncate_sha(event.commit_id)
     ·
-    = markdown event_commit_title(commit[:message]), project: project, pipeline: :single_line, author: event.author
+    = markdown event_commit_title(event.commit_title), project: project, pipeline: :single_line, author: event.author

app/views/events/_event_push.atom.haml

 %div{ xmlns: "http://www.w3.org/1999/xhtml" }
-  - event.commits.first(15).each do |commit|
-    %p
-      %strong= commit[:author][:name]
-      = link_to "(##{truncate_sha(commit[:id])})", project_commit_path(event.project, id: commit[:id])
-      %i
-        at
-        = commit[:timestamp].to_time.to_s(:short)
-    %blockquote= markdown(escape_once(commit[:message]), pipeline: :atom, project: event.project, author: event.author)
-  - if event.commits_count > 15
+  %p
+    %strong= event.author_name
+    = link_to "(#{truncate_sha(event.commit_id)})", project_commit_path(event.project, event.commit_id)
+    %i
+      at
+      = event.created_at.to_s(:short)
+  %blockquote= markdown(escape_once(event.commit_title), pipeline: :atom, project: event.project, author: event.author)
+  - if event.commits_count > 1
     %p
       %i
         \... and
-        = pluralize(event.commits_count - 15, "more commit")
+        = pluralize(event.commits_count - 1, "more commit")

app/views/events/event/_push.html.haml

@@ -14,9 +14,7 @@
 - if event.push_with_commits?
   .event-body
     %ul.well-list.event_commits
-      - few_commits = event.commits[0...2]
-      - few_commits.each do |commit|
-        = render "events/commit", commit: commit, project: project, event: event
+      = render "events/commit", project: project, event: event
 
       - create_mr = event.new_ref? && create_mr_button?(project.default_branch, event.ref_name, project) && event.authored_by?(current_user)
       - if event.commits_count > 1
@@ -44,9 +42,6 @@
           = link_to create_mr_path(project.default_branch, event.ref_name, project) do
             Create Merge Request
 - elsif event.rm_ref?
-  - repository = project.repository
-  - last_commit = repository.commit(event.commit_from)
-  - if last_commit
-    .event-body
-      %ul.well-list.event_commits
-        = render "events/commit", commit: last_commit, project: project, event: event
+  .event-body
+    %ul.well-list.event_commits
+      = render "events/commit", project: project, event: event

app/views/projects/_export.html.haml

+- return unless current_application_settings.project_export_enabled?
+
+- project = local_assigns.fetch(:project)
+- expanded = Rails.env.test?
+
+%section.settings
+  .settings-header
+    %h4
+      Export project
+    %button.btn.js-settings-toggle
+      = expanded ? 'Collapse' : 'Expand'
+    %p
+      Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page.
+  .settings-content.no-animate{ class: ('expanded' if expanded) }
+    .bs-callout.bs-callout-info
+      %p.append-bottom-0
+        %p
+          The following items will be exported:
+        %ul
+          %li Project and wiki repositories
+          %li Project uploads
+          %li Project configuration including web hooks and services
+          %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
+        %p
+          The following items will NOT be exported:
+        %ul
+          %li Job traces and artifacts
+          %li LFS objects
+          %li Container registry images
+          %li CI variables
+          %li Any encrypted tokens
+    %p
+      Once the exported file is ready, you will receive a notification email with a download link.
+    - if project.export_project_path
+      = link_to 'Download export',  download_export_project_path(project),
+              rel: 'nofollow', download: '', method: :get, class: "btn btn-default"
+      = link_to 'Generate new export',  generate_new_export_project_path(project),
+              method: :post, class: "btn btn-default"
+    - else
+      = link_to 'Export project', export_project_path(project),
+              method: :post, class: "btn btn-default"

app/views/projects/_md_preview.html.haml

 - referenced_users = local_assigns.fetch(:referenced_users, nil)
 
 - if defined?(@issue) && @issue.confidential?
-  %li.confidential-issue-warning
+  .confidential-issue-warning
     = confidential_icon(@issue)
     %span This is a confidential issue. Your comment will not be visible to the public.
-- else
-  %li.confidential-issue-warning.not-confidential
 
 .md-area
   .md-header

app/views/projects/edit.html.haml

@@ -161,42 +161,7 @@
         = render 'merge_request_settings', form: f
         = f.submit 'Save changes', class: "btn btn-save"
 
-  %section.settings
-    .settings-header
-      %h4
-        Export project
-      %button.btn.js-settings-toggle
-        = expanded ? 'Collapse' : 'Expand'
-      %p
-        Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page.
-    .settings-content.no-animate{ class: ('expanded' if expanded) }
-      .bs-callout.bs-callout-info
-        %p.append-bottom-0
-          %p
-            The following items will be exported:
-          %ul
-            %li Project and wiki repositories
-            %li Project uploads
-            %li Project configuration including web hooks and services
-            %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
-          %p
-            The following items will NOT be exported:
-          %ul
-            %li Job traces and artifacts
-            %li LFS objects
-            %li Container registry images
-            %li CI variables
-            %li Any encrypted tokens
-      %p
-        Once the exported file is ready, you will receive a notification email with a download link.
-      - if @project.export_project_path
-        = link_to 'Download export',  download_export_project_path(@project),
-                rel: 'nofollow', download: '', method: :get, class: "btn btn-default"
-        = link_to 'Generate new export',  generate_new_export_project_path(@project),
-                method: :post, class: "btn btn-default"
-      - else
-        = link_to 'Export project', export_project_path(@project),
-                method: :post, class: "btn btn-default"
+  = render 'export', project: @project
 
   %section.settings.advanced-settings
     .settings-header

changelogs/unreleased/13325-bugfix-silence-on-disabled-notifications.yml

+---
+title: disabling notifications globally now properly turns off group/project added
+  emails
+merge_request: 13325
+author: @jneen
+type: fixed

changelogs/unreleased/34492-firefox-job.yml

----
-title: Use jQuery to control scroll behavior in job log for cross browser consistency
-merge_request:
-author:

changelogs/unreleased/35232-next-unresolved.yml

----
-title: fix jump to next discussion button
-merge_request:
-author:

changelogs/unreleased/35697-allow-logged-in-user-to-read-user-list.yml

----
-title: Allow any logged in users to read_users_list even if it's restricted
-merge_request: 13201
-author:

changelogs/unreleased/36158-new-issue-button.yml

----
-title: Fixes new issue button for failed job returning 404
-merge_request:
-author:

changelogs/unreleased/36213-return-is_admin-in-users-api-when-current_user-is-admin.yml

+---
+title: Include the `is_admin` field in the `GET /users/:id` API when current user
+  is an admin
+merge_request:
+author:
+type: fixed

changelogs/unreleased/mattermost_fixes.yml → changelogs/unreleased/appearances-caching-and-schema.yml

 ---
-title: Fix Mattermost integration
+title: Cache Appearance instances in Redis
 merge_request:
 author:

changelogs/unreleased/35052-please-select-a-file-when-attempting-to-upload-or-replace-from-the-ui.yml → changelogs/unreleased/broadcast-messages-cache.yml

 ---
-title: improve file upload/replace experience
+title: Better caching and indexing of broadcast messages
 merge_request:
 author:

changelogs/unreleased/bvl-rollback-renamed-system-namespace.yml

+---
+title: Don't rename namespace called system when upgrading from 9.1.x to 9.5
+merge_request: 13228
+author:

changelogs/unreleased/disable-project-export.yml

+---
+title: Add option to disable project export on instance
+merge_request: 13211
+author: Robin Bobbitt

changelogs/unreleased/fix-group-milestone-link-in-issuable-sidebar.yml

----
-title: Fix links to group milestones from issue and merge request sidebar
-merge_request:
-author:

changelogs/unreleased/fix-oauth-checkboxes.yml

----
-title: Fixed sign-in restrictions buttons not toggling active state
-merge_request:
-author:

changelogs/unreleased/fix-sm-34547-cannot-connect-to-ci-server-error-messages.yml

----
-title: Fix an order of operations for CI connection error message in merge request
-  widget
-merge_request: 13252
-author:

changelogs/unreleased/fix-sm-35931-active-ci-pipelineschedule-have-nullified-next_run_at.yml

----
-title: Fix pipeline_schedules pages when active schedule has an abnormal state
-merge_request: 13286
-author:

changelogs/unreleased/migrate-events-into-a-new-format.yml

+---
+title: Migrate events into a new format to reduce the storage necessary and improve performance
+merge_request:
+author:

changelogs/unreleased/mk-fix-case-insensitive-redirect-matching.yml

----
-title: Fix destroy of case-insensitive conflicting redirects
-merge_request: 13357
-author:

changelogs/unreleased/mk-fix-deploy-key-deletion.yml

----
-title: Fix deletion of deploy keys linked to other projects
-merge_request: 13162
-author:

changelogs/unreleased/mk-validate-username-change-with-container-registry-tags.yml

----
-title: Add missing validation error for username change with container registry tags
-merge_request: 13356
-author:

changelogs/unreleased/project-foreign-keys-without-errors.yml

----
-title: Change project FK migration to skip existing FKs
-merge_request:
-author:

changelogs/unreleased/search-flickering.yml

----
-title: Fix search box losing focus when typing
-merge_request:
-author:

changelogs/unreleased/tc-fix-wildcard-protected-delete-merged.yml

----
-title: Make Delete Merged Branches handle wildcard protected branches correctly
-merge_request: 13251
-author:

changelogs/unreleased/use-a-specialized-class-for-querying-events.yml

+---
+title: Use a specialized class for querying events to improve performance
+merge_request:
+author:

changelogs/unreleased/zj-ref-path-monospace.yml

----
-title: Use project_ref_path to create the link to a branch to fix links that 404
-merge_request:
-author:

config/routes/uploads.rb

@@ -5,12 +5,12 @@ scope path: :uploads do
       constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /[^\/]+/ }
 
   # show uploads for models, snippets (notes) available for now
-  get 'system/:model/:id/:secret/:filename',
+  get '-/system/:model/:id/:secret/:filename',
     to: 'uploads#show',
     constraints: { model: /personal_snippet/, id: /\d+/, filename: /[^\/]+/ }
 
   # show temporary uploads
-  get 'system/temp/:secret/:filename',
+  get '-/system/temp/:secret/:filename',
     to: 'uploads#show',
     constraints: { filename: /[^\/]+/ }
 

db/migrate/20170316163800_rename_system_namespaces.rb

-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-class RenameSystemNamespaces < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-  include Gitlab::ShellAdapter
-  disable_ddl_transaction!
-
-  class User < ActiveRecord::Base
-    self.table_name = 'users'
-  end
-
-  class Namespace < ActiveRecord::Base
-    self.table_name = 'namespaces'
-    belongs_to :parent, class_name: 'RenameSystemNamespaces::Namespace'
-    has_one :route, as: :source
-    has_many :children, class_name: 'RenameSystemNamespaces::Namespace', foreign_key: :parent_id
-    belongs_to :owner, class_name: 'RenameSystemNamespaces::User'
-
-    # Overridden to have the correct `source_type` for the `route` relation
-    def self.name
-      'Namespace'
-    end
-
-    def full_path
-      if route && route.path.present?
-        @full_path ||= route.path
-      else
-        update_route if persisted?
-
-        build_full_path
-      end
-    end
-
-    def build_full_path
-      if parent && path
-        parent.full_path + '/' + path
-      else
-        path
-      end
-    end
-
-    def update_route
-      prepare_route
-      route.save
-    end
-
-    def prepare_route
-      route || build_route(source: self)
-      route.path = build_full_path
-      route.name = build_full_name
-      @full_path = nil
-      @full_name = nil
-    end
-
-    def build_full_name
-      if parent && name
-        parent.human_name + ' / ' + name
-      else
-        name
-      end
-    end
-
-    def human_name
-      owner&.name
-    end
-  end
-
-  class Route < ActiveRecord::Base
-    self.table_name = 'routes'
-    belongs_to :source, polymorphic: true
-  end
-
-  class Project < ActiveRecord::Base
-    self.table_name = 'projects'
-
-    def repository_storage_path
-      Gitlab.config.repositories.storages[repository_storage]['path']
-    end
-  end
-
-  DOWNTIME = false
-
-  def up
-    return unless system_namespace
-
-    old_path = system_namespace.path
-    old_full_path = system_namespace.full_path
-    # Only remove the last occurrence of the path name to get the parent namespace path
-    namespace_path = remove_last_occurrence(old_full_path, old_path)
-    new_path = rename_path(namespace_path, old_path)
-    new_full_path = join_namespace_path(namespace_path, new_path)
-
-    Namespace.where(id: system_namespace).update_all(path: new_path) # skips callbacks & validations
-
-    replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path)
-    route_matches = [old_full_path, "#{old_full_path}/%"]
-
-    update_column_in_batches(:routes, :path, replace_statement) do |table, query|
-      query.where(Route.arel_table[:path].matches_any(route_matches))
-    end
-
-    clear_cache_for_namespace(system_namespace)
-
-    # tasks here are based on `Namespace#move_dir`
-    move_repositories(system_namespace, old_full_path, new_full_path)
-    move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage?
-    move_namespace_folders(pages_dir, old_full_path, new_full_path)
-  end
-
-  def down
-    # nothing to do
-  end
-
-  def remove_last_occurrence(string, pattern)
-    string.reverse.sub(pattern.reverse, "").reverse
-  end
-
-  def move_namespace_folders(directory, old_relative_path, new_relative_path)
-    old_path = File.join(directory, old_relative_path)
-    return unless File.directory?(old_path)
-
-    new_path = File.join(directory, new_relative_path)
-    FileUtils.mv(old_path, new_path)
-  end
-
-  def move_repositories(namespace, old_full_path, new_full_path)
-    repo_paths_for_namespace(namespace).each do |repository_storage_path|
-      # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, old_full_path)
-
-      unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-        say "Exception moving path #{repository_storage_path} from #{old_full_path} to #{new_full_path}"
-      end
-    end
-  end
-
-  def rename_path(namespace_path, path_was)
-    counter = 0
-    path = "#{path_was}#{counter}"
-
-    while route_exists?(join_namespace_path(namespace_path, path))
-      counter += 1
-      path = "#{path_was}#{counter}"
-    end
-
-    path
-  end
-
-  def route_exists?(full_path)
-    Route.where(Route.arel_table[:path].matches(full_path)).any?
-  end
-
-  def join_namespace_path(namespace_path, path)
-    if namespace_path.present?
-      File.join(namespace_path, path)
-    else
-      path
-    end
-  end
-
-  def system_namespace
-    @system_namespace ||= Namespace.where(parent_id: nil)
-                            .where(arel_table[:path].matches(system_namespace_path))
-                            .first
-  end
-
-  def system_namespace_path
-    "system"
-  end
-
-  def clear_cache_for_namespace(namespace)
-    project_ids = projects_for_namespace(namespace).pluck(:id)
-
-    update_column_in_batches(:projects, :description_html, nil) do |table, query|
-      query.where(table[:id].in(project_ids))
-    end
-
-    update_column_in_batches(:issues, :description_html, nil) do |table, query|
-      query.where(table[:project_id].in(project_ids))
-    end
-
-    update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
-      query.where(table[:target_project_id].in(project_ids))
-    end
-
-    update_column_in_batches(:notes, :note_html, nil) do |table, query|
-      query.where(table[:project_id].in(project_ids))
-    end
-
-    update_column_in_batches(:milestones, :description_html, nil) do |table, query|
-      query.where(table[:project_id].in(project_ids))
-    end
-  end
-
-  def projects_for_namespace(namespace)
-    namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
-    namespace_or_children = Project.arel_table[:namespace_id].in(namespace_ids)
-    Project.unscoped.where(namespace_or_children)
-  end
-
-  # This won't scale to huge trees, but it should do for a handful of namespaces
-  # called `system`.
-  def child_ids_for_parent(namespace, ids: [])
-    namespace.children.each do |child|
-      ids << child.id
-      child_ids_for_parent(child, ids: ids) if child.children.any?
-    end
-    ids
-  end
-
-  def repo_paths_for_namespace(namespace)
-    projects_for_namespace(namespace).distinct
-      .select(:repository_storage).map(&:repository_storage_path)
-  end
-
-  def uploads_dir
-    File.join(Rails.root, "public", "uploads")
-  end
-
-  def pages_dir
-    Settings.pages.path
-  end
-
-  def file_storage?
-    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
-  end
-
-  def arel_table
-    Namespace.arel_table
-  end
-end

db/migrate/20170316163845_move_uploads_to_system_dir.rb

@@ -54,6 +54,6 @@ class MoveUploadsToSystemDir < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "public", "uploads", "system")
+    File.join(base_directory, "public", "uploads", "-", "system")
   end
 end

db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PrepareEventsTableForPushEventsMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # The order of these columns is deliberate and results in the following
+    # columns and sizes:
+    #
+    # * id (4 bytes)
+    # * project_id (4 bytes)
+    # * author_id (4 bytes)
+    # * target_id (4 bytes)
+    # * created_at (8 bytes)
+    # * updated_at (8 bytes)
+    # * action (2 bytes)
+    # * target_type (variable)
+    #
+    # Unfortunately we can't make the "id" column a bigint/bigserial as Rails 4
+    # does not support this properly.
+    create_table :events_for_migration do |t|
+      t.references :project,
+                   index: true,
+                   foreign_key: { on_delete: :cascade }
+
+      t.integer :author_id, index: true, null: false
+      t.integer :target_id
+
+      t.timestamps_with_timezone null: false
+
+      t.integer :action, null: false, limit: 2, index: true
+      t.string :target_type
+
+      t.index %i[target_type target_id]
+    end
+
+    # t.references doesn't like it when the column name doesn't make the table
+    # name so we have to add the foreign key separately.
+    add_concurrent_foreign_key(:events_for_migration, :users, column: :author_id)
+  end
+
+  def down
+    drop_table :events_for_migration
+  end
+end

db/migrate/20170608152748_create_push_event_payloads_tables.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreatePushEventPayloadsTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :push_event_payloads, id: false do |t|
+      t.bigint :commit_count, null: false
+
+      t.integer :event_id, null: false
+      t.integer :action, null: false, limit: 2
+      t.integer :ref_type, null: false, limit: 2
+
+      t.binary :commit_from
+      t.binary :commit_to
+
+      t.text :ref
+      t.string :commit_title, limit: 70
+
+      t.index :event_id, unique: true
+    end
+
+    # We're adding a foreign key to the _shadow_ table, and this is deliberate.
+    # By using the shadow table we don't have to recreate/revalidate this
+    # foreign key after swapping the "events_for_migration" and "events" tables.
+    #
+    # The "events_for_migration" table has a foreign key to "projects.id"
+    # ensuring that project removals also remove events from the shadow table
+    # (and thus also from this table).
+    add_concurrent_foreign_key(
+      :push_event_payloads,
+      :events_for_migration,
+      column: :event_id
+    )
+  end
+
+  def down
+    drop_table :push_event_payloads
+  end
+end

db/migrate/20170717074009_move_system_upload_folder.rb

@@ -15,6 +15,11 @@ class MoveSystemUploadFolder < ActiveRecord::Migration
       return
     end
 
+    if File.directory?(new_directory)
+      say "#{new_directory} already exists. No need to redo the move."
+      return
+    end
+
     FileUtils.mkdir_p(File.join(base_directory, '-'))
 
     say "Moving #{old_directory} -> #{new_directory}"
@@ -33,6 +38,11 @@ class MoveSystemUploadFolder < ActiveRecord::Migration
       return
     end
 
+    if !File.symlink?(old_directory) && File.directory?(old_directory)
+      say "#{old_directory} already exists and is not a symlink, no need to revert."
+      return
+    end
+
     if File.symlink?(old_directory)
       say "Removing #{old_directory} -> #{new_directory} symlink"
       FileUtils.rm(old_directory)

db/migrate/20170727123534_add_index_on_events_project_id_id.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexOnEventsProjectIdId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  COLUMNS = %i[project_id id].freeze
+  TABLES = %i[events events_for_migration].freeze
+
+  disable_ddl_transaction!
+
+  def up
+    TABLES.each do |table|
+      add_concurrent_index(table, COLUMNS) unless index_exists?(table, COLUMNS)
+
+      # We remove the index _after_ adding the new one since MySQL doesn't let
+      # you remove an index when a foreign key exists for the same column.
+      if index_exists?(table, :project_id)
+        remove_concurrent_index(table, :project_id)
+      end
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      unless index_exists?(table, :project_id)
+        add_concurrent_index(table, :project_id)
+      end
+
+      unless index_exists?(table, COLUMNS)
+        remove_concurrent_index(table, COLUMNS)
+      end
+    end
+  end
+end

db/migrate/20170809133343_add_broadcast_messages_index.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddBroadcastMessagesIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  COLUMNS = %i[starts_at ends_at id].freeze
+
+  def up
+    add_concurrent_index :broadcast_messages, COLUMNS
+  end
+
+  def down
+    remove_concurrent_index :broadcast_messages, COLUMNS
+  end
+end

db/migrate/20170809134534_add_broadcast_message_not_null_constraints.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddBroadcastMessageNotNullConstraints < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  COLUMNS = %i[starts_at ends_at created_at updated_at message_html]
+
+  def change
+    COLUMNS.each do |column|
+      change_column_null :broadcast_messages, column, false
+    end
+  end
+end

db/migrate/20170809142252_cleanup_appearances_schema.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CleanupAppearancesSchema < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  NOT_NULL_COLUMNS = %i[title description description_html created_at updated_at]
+
+  TIME_COLUMNS = %i[created_at updated_at]
+
+  def up
+    NOT_NULL_COLUMNS.each do |column|
+      change_column_null :appearances, column, false
+    end
+
+    TIME_COLUMNS.each do |column|
+      change_column :appearances, column, :datetime_with_timezone
+    end
+  end
+
+  def down
+    NOT_NULL_COLUMNS.each do |column|
+      change_column_null :appearances, column, true
+    end
+
+    TIME_COLUMNS.each do |column|
+      change_column :appearances, column, :datetime # rubocop: disable Migration/Datetime
+    end
+  end
+end

db/migrate/20170809161910_add_project_export_enabled_to_application_settings.rb

+class AddProjectExportEnabledToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    add_column_with_default(:application_settings, :project_export_enabled, :boolean, default: true)
+  end
+
+  def down
+    remove_column(:application_settings, :project_export_enabled)
+  end
+end

db/post_migrate/20170317162059_update_upload_paths_to_system.rb

@@ -48,7 +48,7 @@ class UpdateUploadPathsToSystem < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "system")
+    File.join(base_directory, "-",  "system")
   end
 
   def arel_table

db/post_migrate/20170406111121_clean_upload_symlinks.rb

@@ -47,6 +47,6 @@ class CleanUploadSymlinks < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "public", "uploads", "system")
+    File.join(base_directory, "public", "uploads", "-", "system")
   end
 end

db/post_migrate/20170606202615_move_appearance_to_system_dir.rb

@@ -52,6 +52,6 @@ class MoveAppearanceToSystemDir < ActiveRecord::Migration
   end
 
   def new_upload_dir
-    File.join(base_directory, "public", "uploads", "system")
+    File.join(base_directory, "public", "uploads", "-", "system")
   end
 end

db/post_migrate/20170612071012_move_personal_snippets_files.rb

@@ -10,7 +10,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration
     return unless file_storage?
 
     @source_relative_location = File.join('/uploads', 'personal_snippet')
-    @destination_relative_location = File.join('/uploads', 'system', 'personal_snippet')
+    @destination_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet')
 
     move_personal_snippet_files
   end
@@ -18,7 +18,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration
   def down
     return unless file_storage?
 
-    @source_relative_location = File.join('/uploads', 'system', 'personal_snippet')
+    @source_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet')
     @destination_relative_location = File.join('/uploads', 'personal_snippet')
 
     move_personal_snippet_files

db/post_migrate/20170627101016_schedule_event_migrations.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ScheduleEventMigrations < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BUFFER_SIZE = 1000
+
+  disable_ddl_transaction!
+
+  class Event < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'events'
+  end
+
+  def up
+    jobs = []
+
+    Event.each_batch(of: 1000) do |relation|
+      min, max = relation.pluck('MIN(id), MAX(id)').first
+
+      if jobs.length == BUFFER_SIZE
+        # We push multiple jobs at a time to reduce the time spent in
+        # Sidekiq/Redis operations. We're using this buffer based approach so we
+        # don't need to run additional queries for every range.
+        BackgroundMigrationWorker.perform_bulk(jobs)
+        jobs.clear
+      end
+
+      jobs << ['MigrateEventsToPushEventPayloads', [min, max]]
+    end
+
+    BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+  end
+
+  def down
+  end
+end

db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MovePersonalSnippetFilesIntoCorrectFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  NEW_DIRECTORY = File.join('/uploads', '-', 'system', 'personal_snippet')
+  OLD_DIRECTORY = File.join('/uploads', 'system', 'personal_snippet')
+
+  def up
+    return unless file_storage?
+
+    BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles',
+                                            [OLD_DIRECTORY, NEW_DIRECTORY])
+  end
+
+  def down
+    return unless file_storage?
+
+    BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles',
+                                            [NEW_DIRECTORY, OLD_DIRECTORY])
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170807160457) do
+ActiveRecord::Schema.define(version: 20170809161910) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -28,13 +28,13 @@ ActiveRecord::Schema.define(version: 20170807160457) do
   end
 
   create_table "appearances", force: :cascade do |t|
-    t.string "title"
-    t.text "description"
+    t.string "title", null: false
+    t.text "description", null: false
     t.string "header_logo"
     t.string "logo"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.text "description_html"
+    t.text "description_html", null: false
     t.integer "cached_markdown_version"
   end
 
@@ -127,6 +127,7 @@ ActiveRecord::Schema.define(version: 20170807160457) do
     t.string "help_page_support_url"
     t.integer "performance_bar_allowed_group_id"
     t.boolean "password_authentication_enabled"
+    t.boolean "project_export_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -163,16 +164,18 @@ ActiveRecord::Schema.define(version: 20170807160457) do
 
   create_table "broadcast_messages", force: :cascade do |t|
     t.text "message", null: false
-    t.datetime "starts_at"
-    t.datetime "ends_at"
-    t.datetime "created_at"
-    t.datetime "updated_at"
+    t.datetime "starts_at", null: false
+    t.datetime "ends_at", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
     t.string "color"
     t.string "font"
-    t.text "message_html"
+    t.text "message_html", null: false
     t.integer "cached_markdown_version"
   end
 
+  add_index "broadcast_messages", ["starts_at", "ends_at", "id"], name: "index_broadcast_messages_on_starts_at_and_ends_at_and_id", using: :btree
+
   create_table "chat_names", force: :cascade do |t|
     t.integer "user_id", null: false
     t.integer "service_id", null: false
@@ -530,10 +533,25 @@ ActiveRecord::Schema.define(version: 20170807160457) do
   add_index "events", ["action"], name: "index_events_on_action", using: :btree
   add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree
   add_index "events", ["created_at"], name: "index_events_on_created_at", using: :btree
-  add_index "events", ["project_id"], name: "index_events_on_project_id", using: :btree
+  add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree
   add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
   add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
 
+  create_table "events_for_migration", force: :cascade do |t|
+    t.integer "project_id"
+    t.integer "author_id", null: false
+    t.integer "target_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "action", limit: 2, null: false
+    t.string "target_type"
+  end
+
+  add_index "events_for_migration", ["action"], name: "index_events_for_migration_on_action", using: :btree
+  add_index "events_for_migration", ["author_id"], name: "index_events_for_migration_on_author_id", using: :btree
+  add_index "events_for_migration", ["project_id", "id"], name: "index_events_for_migration_on_project_id_and_id", using: :btree
+  add_index "events_for_migration", ["target_type", "target_id"], name: "index_events_for_migration_on_target_type_and_target_id", using: :btree
+
   create_table "feature_gates", force: :cascade do |t|
     t.string "feature_key", null: false
     t.string "key", null: false
@@ -1254,6 +1272,19 @@ ActiveRecord::Schema.define(version: 20170807160457) do
 
   add_index "protected_tags", ["project_id"], name: "index_protected_tags_on_project_id", using: :btree
 
+  create_table "push_event_payloads", id: false, force: :cascade do |t|
+    t.integer "commit_count", limit: 8, null: false
+    t.integer "event_id", null: false
+    t.integer "action", limit: 2, null: false
+    t.integer "ref_type", limit: 2, null: false
+    t.binary "commit_from"
+    t.binary "commit_to"
+    t.text "ref"
+    t.string "commit_title", limit: 70
+  end
+
+  add_index "push_event_payloads", ["event_id"], name: "index_push_event_payloads_on_event_id", unique: true, using: :btree
+
   create_table "redirect_routes", force: :cascade do |t|
     t.integer "source_id", null: false
     t.string "source_type", null: false
@@ -1654,6 +1685,8 @@ ActiveRecord::Schema.define(version: 20170807160457) do
   add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade
   add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
   add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
+  add_foreign_key "events_for_migration", "projects", on_delete: :cascade
+  add_foreign_key "events_for_migration", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
   add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
   add_foreign_key "gpg_keys", "users", on_delete: :cascade
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
@@ -1696,6 +1729,7 @@ ActiveRecord::Schema.define(version: 20170807160457) do
   add_foreign_key "protected_tag_create_access_levels", "protected_tags"
   add_foreign_key "protected_tag_create_access_levels", "users"
   add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
+  add_foreign_key "push_event_payloads", "events_for_migration", column: "event_id", name: "fk_36c74129da", on_delete: :cascade
   add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
   add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade

doc/api/events.md

@@ -79,7 +79,6 @@ Example response:
     "target_id":160,
     "target_type":"Issue",
     "author_id":25,
-    "data":null,
     "target_title":"Qui natus eos odio tempore et quaerat consequuntur ducimus cupiditate quis.",
     "created_at":"2017-02-09T10:43:19.667Z",
     "author":{
@@ -99,7 +98,6 @@ Example response:
     "target_id":159,
     "target_type":"Issue",
     "author_id":21,
-    "data":null,
     "target_title":"Nostrum enim non et sed optio illo deleniti non.",
     "created_at":"2017-02-09T10:43:19.426Z",
     "author":{
@@ -151,7 +149,6 @@ Example response:
     "target_id": 830,
     "target_type": "Issue",
     "author_id": 1,
-    "data": null,
     "target_title": "Public project search field",
     "author": {
       "name": "Dmitriy Zaporozhets",
@@ -166,7 +163,7 @@ Example response:
   {
     "title": null,
     "project_id": 15,
-    "action_name": "opened",
+    "action_name": "pushed",
     "target_id": null,
     "target_type": null,
     "author_id": 1,
@@ -179,31 +176,14 @@ Example response:
       "web_url": "http://localhost:3000/root"
     },
     "author_username": "john",
-    "data": {
-      "before": "50d4420237a9de7be1304607147aec22e4a14af7",
-      "after": "c5feabde2d8cd023215af4d2ceeb7a64839fc428",
-      "ref": "refs/heads/master",
-      "user_id": 1,
-      "user_name": "Dmitriy Zaporozhets",
-      "repository": {
-        "name": "gitlabhq",
-        "url": "git@dev.gitlab.org:gitlab/gitlabhq.git",
-        "description": "GitLab: self hosted Git management software. \r\nDistributed under the MIT License.",
-        "homepage": "https://dev.gitlab.org/gitlab/gitlabhq"
-      },
-      "commits": [
-        {
-          "id": "c5feabde2d8cd023215af4d2ceeb7a64839fc428",
-          "message": "Add simple search to projects in public area",
-          "timestamp": "2013-05-13T18:18:08+00:00",
-          "url": "https://dev.gitlab.org/gitlab/gitlabhq/commit/c5feabde2d8cd023215af4d2ceeb7a64839fc428",
-          "author": {
-            "name": "Dmitriy Zaporozhets",
-            "email": "dmitriy.zaporozhets@gmail.com"
-          }
-        }
-      ],
-      "total_commits_count": 1
+    "push_data": {
+      "commit_count": 1,
+      "action": "pushed",
+      "ref_type": "branch",
+      "commit_from": "50d4420237a9de7be1304607147aec22e4a14af7",
+      "commit_to": "c5feabde2d8cd023215af4d2ceeb7a64839fc428",
+      "ref": "master",
+      "commit_title": "Add simple search to projects in public area"
     },
     "target_title": null
   },
@@ -214,7 +194,6 @@ Example response:
     "target_id": 840,
     "target_type": "Issue",
     "author_id": 1,
-    "data": null,
     "target_title": "Finish & merge Code search PR",
     "author": {
       "name": "Dmitriy Zaporozhets",
@@ -233,7 +212,6 @@ Example response:
     "target_id": 1312,
     "target_type": "Note",
     "author_id": 1,
-    "data": null,
     "target_title": null,
     "created_at": "2015-12-04T10:33:58.089Z",
     "note": {
@@ -305,7 +283,6 @@ Example response:
     "target_iid":160,
     "target_type":"Issue",
     "author_id":25,
-    "data":null,
     "target_title":"Qui natus eos odio tempore et quaerat consequuntur ducimus cupiditate quis.",
     "created_at":"2017-02-09T10:43:19.667Z",
     "author":{
@@ -326,7 +303,6 @@ Example response:
     "target_iid":159,
     "target_type":"Issue",
     "author_id":21,
-    "data":null,
     "target_title":"Nostrum enim non et sed optio illo deleniti non.",
     "created_at":"2017-02-09T10:43:19.426Z",
     "author":{

doc/development/testing.md

@@ -157,8 +157,9 @@ trade-off:
 
 - Unit tests are usually cheap, and you should consider them like the basement
   of your house: you need them to be confident that your code is behaving
-  correctly. However if you run only unit tests without integration / system tests, you might [miss] the [big] [picture]!
-- Integration tests are a bit more expensive, but don't abuse them. A feature test
+  correctly. However if you run only unit tests without integration / system
+  tests, you might [miss] the [big] [picture]!
+- Integration tests are a bit more expensive, but don't abuse them. A system test
   is often better than an integration test that is stubbing a lot of internals.
 - System tests are expensive (compared to unit tests), even more if they require
   a JavaScript driver. Make sure to follow the guidelines in the [Speed](#test-speed)
@@ -195,11 +196,27 @@ Please consult the [dedicated "Frontend testing" guide](./fe_guide/testing.md).
 - Try to match the ordering of tests to the ordering within the class.
 - Try to follow the [Four-Phase Test][four-phase-test] pattern, using newlines
   to separate phases.
-- Try to use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'`
+- Use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'`
+- Don't assert against the absolute value of a sequence-generated attribute (see
+  [Gotchas](gotchas.md#dont-assert-against-the-absolute-value-of-a-sequence-generated-attribute)).
+- Don't supply the `:each` argument to hooks since it's the default.
 - On `before` and `after` hooks, prefer it scoped to `:context` over `:all`
 
 [four-phase-test]: https://robots.thoughtbot.com/four-phase-test
 
+### Automatic retries and flaky tests detection
+
+On our CI, we use [rspec-retry] to automatically retry a failing example a few
+times (see [`spec/spec_helper.rb`] for the precise retries count).
+
+We also use a home-made `RspecFlaky::Listener` listener which records flaky
+examples in a JSON report file on `master` (`retrieve-tests-metadata` and `update-tests-metadata` jobs), and warns when a new flaky example
+is detected in any other branch (`flaky-examples-check` job). In the future, the
+`flaky-examples-check` job will not be allowed to fail.
+
+[rspec-retry]: https://github.com/NoRedInk/rspec-retry
+[`spec/spec_helper.rb`]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/spec_helper.rb
+
 ### `let` variables
 
 GitLab's RSpec suite has made extensive use of `let` variables to reduce
@@ -270,6 +287,15 @@ complexity of RSpec expectations.They should be placed under
 a certain type of specs only (e.g. features, requests etc.) but shouldn't be if
 they apply to multiple type of specs.
 
+#### have_gitlab_http_status
+
+Prefer `have_gitlab_http_status` over `have_http_status` because the former
+could also show the response body whenever the status mismatched. This would
+be very useful whenever some tests start breaking and we would love to know
+why without editing the source and rerun the tests.
+
+This is especially useful whenever it's showing 500 internal server error.
+
 ### Shared contexts
 
 All shared contexts should be be placed under `spec/support/shared_contexts/`.

doc/install/requirements.md

@@ -104,6 +104,10 @@ features of GitLab work with MySQL/MariaDB:
    See [issue #30472][30472] for more information.
 1. GitLab Geo does [not support MySQL](https://docs.gitlab.com/ee/gitlab-geo/database.html#mysql-replication).
 1. [Zero downtime migrations][zero] do not work with MySQL
+1. GitLab [optimizes the loading of dashboard events](https://gitlab.com/gitlab-org/gitlab-ce/issues/31806) using [PostgreSQL LATERAL JOINs](https://blog.heapanalytics.com/postgresqls-powerful-new-join-type-lateral/).
+1. In general, SQL optimized for PostgreSQL may run much slower in MySQL due to
+   differences in query planners. For example, subqueries that work well in PostgreSQL
+   may not be [performant in MySQL](https://dev.mysql.com/doc/refman/5.7/en/optimizing-subqueries.html)
 1. We expect this list to grow over time.
 
 Existing users using GitLab with MySQL/MariaDB are advised to

doc/update/patch_versions.md

@@ -56,21 +56,17 @@ sudo -u git -H bundle clean
 # Run database migrations
 sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production
 
-### 4. Compile GetText PO files
-
-Internationalization was added in `v9.2.0` so these commands are only
-required for versions equal or major to it.
-
-```bash
+# Compile GetText PO files
+# Internationalization was added in `v9.2.0` so these commands are only
+# required for versions equal or major to it.
 sudo -u git -H bundle exec rake gettext:pack RAILS_ENV=production
 sudo -u git -H bundle exec rake gettext:po_to_json RAILS_ENV=production
-```
 
 # Clean up assets and cache
 sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile cache:clear RAILS_ENV=production NODE_ENV=production
 ```
 
-### 5. Update gitlab-workhorse to the corresponding version
+### 4. Update gitlab-workhorse to the corresponding version
 
 ```bash
 cd /home/git/gitlab
@@ -78,7 +74,7 @@ cd /home/git/gitlab
 sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]" RAILS_ENV=production
 ```
 
-### 6. Update gitlab-shell to the corresponding version
+### 5. Update gitlab-shell to the corresponding version
 
 ```bash
 cd /home/git/gitlab-shell
@@ -88,14 +84,14 @@ sudo -u git -H git checkout v`cat /home/git/gitlab/GITLAB_SHELL_VERSION` -b v`ca
 sudo -u git -H sh -c 'if [ -x bin/compile ]; then bin/compile; fi'
 ```
 
-### 7. Start application
+### 6. Start application
 
 ```bash
 sudo service gitlab start
 sudo service nginx restart
 ```
 
-### 8. Check application status
+### 7. Check application status
 
 Check if GitLab and its environment are configured correctly:
 

doc/user/project/settings/import_export.md

@@ -9,6 +9,9 @@
 >    application settings (`/admin/application_settings`) under 'Import sources'.
 >    Ask your administrator if you don't see the **GitLab export** button when
 >    creating a new project.
+>  - Starting with GitLab 10.0, administrators can disable the project export option
+>    on the GitLab instance in application settings (`/admin/application_settings`)
+>    under 'Visibility and Access Controls'.
 >  - You can find some useful raketasks if you are an administrator in the
 >    [import_export](../../../administration/raketasks/project_import_export.md)
 >    raketask.

features/steps/shared/project.rb

@@ -71,28 +71,14 @@ module SharedProject
 
   step 'project "Shop" has push event' do
     @project = Project.find_by(name: "Shop")
-
-    data = {
-      before: Gitlab::Git::BLANK_SHA,
-      after: "6d394385cf567f80a8fd85055db1ab4c5295806f",
-      ref: "refs/heads/fix",
-      user_id: @user.id,
-      user_name: @user.name,
-      repository: {
-        name: @project.name,
-        url: "localhost/rubinius",
-        description: "",
-        homepage: "localhost/rubinius",
-        private: true
-      }
-    }
-
-    @event = Event.create(
-      project: @project,
-      action: Event::PUSHED,
-      data: data,
-      author_id: @user.id
-    )
+    @event = create(:push_event, project: @project, author: @user)
+
+    create(:push_event_payload,
+           event: @event,
+           action: :created,
+           commit_to: '6d394385cf567f80a8fd85055db1ab4c5295806f',
+           ref: 'fix',
+           commit_count: 1)
   end
 
   step 'I should see project "Shop" activity feed' do

lib/api/entities.rb

@@ -497,14 +497,24 @@ module API
       expose :author, using: Entities::UserBasic
     end
 
+    class PushEventPayload < Grape::Entity
+      expose :commit_count, :action, :ref_type, :commit_from, :commit_to
+      expose :ref, :commit_title
+    end
+
     class Event < Grape::Entity
       expose :title, :project_id, :action_name
       expose :target_id, :target_iid, :target_type, :author_id
-      expose :data, :target_title
+      expose :target_title
       expose :created_at
       expose :note, using: Entities::Note, if: ->(event, options) { event.note? }
       expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author }
 
+      expose :push_event_payload,
+        as: :push_data,
+        using: PushEventPayload,
+        if: -> (event, _) { event.push? }
+
       expose :author_username do |event, options|
         event.author&.username
       end

lib/api/settings.rb

@@ -29,6 +29,7 @@ module API
                                 desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
       optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources'
       optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.'
+      optional :project_export_enabled, type: Boolean, desc: 'Enable project export'
       optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
       optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
       optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'

lib/api/users.rb

@@ -79,22 +79,17 @@ module API
       end
 
       desc 'Get a single user' do
-        success Entities::UserBasic
+        success Entities::User
       end
       params do
         requires :id, type: Integer, desc: 'The ID of the user'
       end
       get ":id" do
         user = User.find_by(id: params[:id])
-        not_found!('User') unless user
+        not_found!('User') unless user && can?(current_user, :read_user, user)
 
-        if current_user && current_user.admin?
-          present user, with: Entities::UserPublic
-        elsif can?(current_user, :read_user, user)
-          present user, with: Entities::User
-        else
-          render_api_error!("User not found.", 404)
-        end
+        opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {}
+        present user, opts
       end
 
       desc 'Create a user. Available only for admins.' do

lib/api/v3/entities.rb

@@ -25,14 +25,24 @@ module API
         expose(:downvote?)  { |note| false }
       end
 
+      class PushEventPayload < Grape::Entity
+        expose :commit_count, :action, :ref_type, :commit_from, :commit_to
+        expose :ref, :commit_title
+      end
+
       class Event < Grape::Entity
         expose :title, :project_id, :action_name
         expose :target_id, :target_type, :author_id
-        expose :data, :target_title
+        expose :target_title
         expose :created_at
         expose :note, using: Entities::Note, if: ->(event, options) { event.note? }
         expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author }
 
+        expose :push_event_payload,
+          as: :push_data,
+          using: PushEventPayload,
+          if: -> (event, _) { event.push? }
+
         expose :author_username do |event, options|
           event.author&.username
         end

lib/file_streamer.rb

-class FileStreamer #:nodoc:
-  attr_reader :to_path
-
-  def initialize(path)
-    @to_path = path
-  end
-
-  # Stream the file's contents if Rack::Sendfile isn't present.
-  def each
-    File.open(to_path, 'rb') do |file|
-      while chunk = file.read(16384)
-        yield chunk
-      end
-    end
-  end
-end

lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb

+module Gitlab
+  module BackgroundMigration
+    # Class that migrates events for the new push event payloads setup. All
+    # events are copied to a shadow table, and push events will also have a row
+    # created in the push_event_payloads table.
+    class MigrateEventsToPushEventPayloads
+      class Event < ActiveRecord::Base
+        self.table_name = 'events'
+
+        serialize :data
+
+        BLANK_REF = ('0' * 40).freeze
+        TAG_REF_PREFIX = 'refs/tags/'.freeze
+        MAX_INDEX = 69
+        PUSHED = 5
+
+        def push_event?
+          action == PUSHED && data.present?
+        end
+
+        def commit_title
+          commit = commits.last
+
+          return nil unless commit && commit[:message]
+
+          index = commit[:message].index("\n")
+          message = index ? commit[:message][0..index] : commit[:message]
+
+          message.strip.truncate(70)
+        end
+
+        def commit_from_sha
+          if create?
+            nil
+          else
+            data[:before]
+          end
+        end
+
+        def commit_to_sha
+          if remove?
+            nil
+          else
+            data[:after]
+          end
+        end
+
+        def data
+          super || {}
+        end
+
+        def commits
+          data[:commits] || []
+        end
+
+        def commit_count
+          data[:total_commits_count] || 0
+        end
+
+        def ref
+          data[:ref]
+        end
+
+        def trimmed_ref_name
+          if ref_type == :tag
+            ref[10..-1]
+          else
+            ref[11..-1]
+          end
+        end
+
+        def create?
+          data[:before] == BLANK_REF
+        end
+
+        def remove?
+          data[:after] == BLANK_REF
+        end
+
+        def push_action
+          if create?
+            :created
+          elsif remove?
+            :removed
+          else
+            :pushed
+          end
+        end
+
+        def ref_type
+          if ref.start_with?(TAG_REF_PREFIX)
+            :tag
+          else
+            :branch
+          end
+        end
+      end
+
+      class EventForMigration < ActiveRecord::Base
+        self.table_name = 'events_for_migration'
+      end
+
+      class PushEventPayload < ActiveRecord::Base
+        self.table_name = 'push_event_payloads'
+
+        enum action: {
+          created: 0,
+          removed: 1,
+          pushed: 2
+        }
+
+        enum ref_type: {
+          branch: 0,
+          tag: 1
+        }
+      end
+
+      # start_id - The start ID of the range of events to process
+      # end_id - The end ID of the range to process.
+      def perform(start_id, end_id)
+        return unless migrate?
+
+        find_events(start_id, end_id).each { |event| process_event(event) }
+      end
+
+      def process_event(event)
+        replicate_event(event)
+        create_push_event_payload(event) if event.push_event?
+      end
+
+      def replicate_event(event)
+        new_attributes = event.attributes
+          .with_indifferent_access.except(:title, :data)
+
+        EventForMigration.create!(new_attributes)
+      rescue ActiveRecord::InvalidForeignKey
+        # A foreign key error means the associated event was removed. In this
+        # case we'll just skip migrating the event.
+      end
+
+      def create_push_event_payload(event)
+        commit_from = pack(event.commit_from_sha)
+        commit_to = pack(event.commit_to_sha)
+
+        PushEventPayload.create!(
+          event_id: event.id,
+          commit_count: event.commit_count,
+          ref_type: event.ref_type,
+          action: event.push_action,
+          commit_from: commit_from,
+          commit_to: commit_to,
+          ref: event.trimmed_ref_name,
+          commit_title: event.commit_title
+        )
+      rescue ActiveRecord::InvalidForeignKey
+        # A foreign key error means the associated event was removed. In this
+        # case we'll just skip migrating the event.
+      end
+
+      def find_events(start_id, end_id)
+        Event
+          .where('NOT EXISTS (SELECT true FROM events_for_migration WHERE events_for_migration.id = events.id)')
+          .where(id: start_id..end_id)
+      end
+
+      def migrate?
+        Event.table_exists? && PushEventPayload.table_exists? &&
+          EventForMigration.table_exists?
+      end
+
+      def pack(value)
+        value ? [value].pack('H*') : nil
+      end
+    end
+  end
+end

lib/gitlab/background_migration/move_personal_snippet_files.rb

+module Gitlab
+  module BackgroundMigration
+    class MovePersonalSnippetFiles
+      delegate :select_all, :execute, :quote_string, to: :connection
+
+      def perform(relative_source, relative_destination)
+        @source_relative_location = relative_source
+        @destination_relative_location = relative_destination
+
+        move_personal_snippet_files
+      end
+
+      def move_personal_snippet_files
+        query = "SELECT uploads.path, uploads.model_id FROM uploads "\
+                "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'"
+        select_all(query).each do |upload|
+          secret = upload['path'].split('/')[0]
+          file_name = upload['path'].split('/')[1]
+
+          move_file(upload['model_id'], secret, file_name)
+          update_markdown(upload['model_id'], secret, file_name)
+        end
+      end
+
+      def move_file(snippet_id, secret, file_name)
+        source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret)
+        destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret)
+
+        source_file_path = File.join(source_dir, file_name)
+        destination_file_path = File.join(destination_dir, file_name)
+
+        unless File.exist?(source_file_path)
+          say "Source file `#{source_file_path}` doesn't exist. Skipping."
+          return
+        end
+
+        say "Moving file #{source_file_path} -> #{destination_file_path}"
+
+        FileUtils.mkdir_p(destination_dir)
+        FileUtils.move(source_file_path, destination_file_path)
+      end
+
+      def update_markdown(snippet_id, secret, file_name)
+        source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name)
+        destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name)
+
+        source_markdown = "](#{source_markdown_path})"
+        destination_markdown = "](#{destination_markdown_path})"
+        quoted_source = quote_string(source_markdown)
+        quoted_destination = quote_string(destination_markdown)
+
+        execute("UPDATE snippets "\
+                "SET description = replace(snippets.description, '#{quoted_source}', '#{quoted_destination}'), description_html = NULL "\
+                "WHERE id = #{snippet_id}")
+
+        query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\
+                "AND noteable_type = 'Snippet' AND note IS NOT NULL"
+        select_all(query).each do |note|
+          text = note['note'].gsub(source_markdown, destination_markdown)
+          quoted_text = quote_string(text)
+
+          execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}")
+        end
+      end
+
+      def base_directory
+        File.join(Rails.root, 'public')
+      end
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def say(message)
+        Rails.logger.debug(message)
+      end
+    end
+  end
+end