Merge branch 'master' into es_docs

CHANGELOG-EE

+Please view this file on the master branch, on stable branches it's out of date.
+
 v 8.5.0 (unreleased)
+  - Show warning when mirror repository default branch could not be updated because it has diverged from upstream.
+
+v 8.4.2
+  - Elasticsearch indexer performance improvements
+  - Don't redirect away from Mirror Repository settings when repo is empty
+  - Fix updating of branches in mirrored repository
+  - Fix a 500 error preventing LDAP users with 2FA enabled from logging in
+  - Rake task gitlab:elastic:index_repositories handles errors and shows progress
+  - Partial indexing of repo on push (indexing changes only)
 
 v 8.4.1
   - No EE-specific changes

Gemfile

@@ -94,7 +94,7 @@ gem "seed-fu", '~> 2.3.5'
 # Search
 gem 'elasticsearch-model'
 gem 'elasticsearch-rails'
-gem 'gitlab-elasticsearch-git', require: "elasticsearch/git"
+gem 'gitlab-elasticsearch-git', '~> 0.0.9', require: "elasticsearch/git"
 
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'

Gemfile.lock

@@ -358,13 +358,13 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.3.3)
-    gitlab-elasticsearch-git (0.0.7)
-      activemodel (~> 4.2.0)
-      activesupport (~> 4.2.0)
-      charlock_holmes (~> 0.7.3)
-      elasticsearch-api (~> 1.0.15)
-      elasticsearch-model
-      github-linguist (~> 4.7.0)
+    gitlab-elasticsearch-git (0.0.9)
+      activemodel (~> 4.2)
+      activesupport (~> 4.2)
+      charlock_holmes (~> 0.7)
+      elasticsearch-api (~> 1.0)
+      elasticsearch-model (~> 0.1.8)
+      github-linguist (~> 4.7)
       rugged (~> 0.23.3)
     gitlab-flowdock-git-hook (1.0.1)
       flowdock (~> 0.7)
@@ -958,7 +958,7 @@ DEPENDENCIES
   gemnasium-gitlab-service (~> 0.2)
   github-linguist (~> 4.7.0)
   github-markup (~> 1.3.1)
-  gitlab-elasticsearch-git
+  gitlab-elasticsearch-git (~> 0.0.9)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 0.0.4)
   gitlab_emoji (~> 0.2.0)

app/controllers/omniauth_callbacks_controller.rb

@@ -22,18 +22,19 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController
   # We only find ourselves here
   # if the authentication to LDAP was successful.
   def ldap
-    @user = Gitlab::LDAP::User.new(oauth)
-    @user.save if @user.changed? # will also save new users
-    gl_user = @user.gl_user
-    gl_user.remember_me = params[:remember_me] if @user.persisted?
+    ldap_user = Gitlab::LDAP::User.new(oauth)
+    ldap_user.save if ldap_user.changed? # will also save new users
+
+    @user = ldap_user.gl_user
+    @user.remember_me = params[:remember_me] if ldap_user.persisted?
 
     # Do additional LDAP checks for the user filter and EE features
-    if @user.allowed?
+    if ldap_user.allowed?
       if @user.otp_required_for_login?
-        prompt_for_two_factor(gl_user)
+        prompt_for_two_factor(@user)
       else
-        log_audit_event(gl_user, with: :ldap)
-        sign_in_and_redirect(gl_user)
+        log_audit_event(@user, with: :ldap)
+        sign_in_and_redirect(@user)
       end
     else
       flash[:alert] = "Access denied for your LDAP account."

app/controllers/projects/mirrors_controller.rb

 class Projects::MirrorsController < Projects::ApplicationController
   # Authorize
-  before_action :require_non_empty_project
   before_action :authorize_admin_project!, except: [:update_now]
   before_action :authorize_push_code!, only: [:update_now]
 

app/helpers/application_helper.rb

@@ -142,7 +142,7 @@ module ApplicationHelper
     return false unless project.repository.branch_names.include?(event.branch_name)
 
     # Skip if this was a mirror update
-    return false if project.mirror? && project.up_to_date_with_upstream?(event.branch_name)
+    return false if project.mirror? && project.repository.up_to_date_with_upstream?(event.branch_name)
 
     true
   end

app/services/git_push_service.rb

@@ -59,15 +59,15 @@ class GitPushService
     @push_data = build_push_data(oldrev, newrev, ref)
 
     branch_name = Gitlab::Git.ref_name(ref)
-    mirror_update = project.mirror? && project.up_to_date_with_upstream?(branch_name)
+    mirror_update = project.mirror? && project.repository.up_to_date_with_upstream?(branch_name)
 
     EventCreateService.new.push(project, user, @push_data)
     project.execute_hooks(@push_data.dup, :push_hooks)
     project.execute_services(@push_data.dup, :push_hooks)
 
     if Gitlab.config.elasticsearch.enabled
-      project.repository.index_commits
-      project.repository.index_blobs
+      project.repository.index_commits(from_rev: oldrev, to_rev: newrev)
+      project.repository.index_blobs(from_rev: oldrev, to_rev: oldrev)
     end
 
     CreateCommitBuildsService.new.execute(project, @user, @push_data, mirror_update: mirror_update)

app/services/projects/update_mirror_service.rb

@@ -26,6 +26,8 @@ module Projects
     def update_branches
       local_branches = repository.branches.each_with_object({}) { |branch, branches| branches[branch.name] = branch }
 
+      errors = []
+
       repository.upstream_branches.each do |upstream_branch|
         name = upstream_branch.name
 
@@ -34,20 +36,27 @@ module Projects
         if local_branch.nil?
           result = CreateBranchService.new(project, current_user).execute(name, upstream_branch.target)
           if result[:status] == :error
-            raise UpdateError, result[:message]
+            errors << result[:message]
           end
         elsif local_branch.target == upstream_branch.target
           # Already up to date
         elsif repository.diverged_from_upstream?(name)
           # Cannot be updated
+          if name == project.default_branch
+            errors << "The default branch (#{project.default_branch}) has diverged from its upstream counterpart and could not be updated automatically."
+          end
         else
           begin
             repository.ff_merge(current_user, upstream_branch.target, name)
           rescue GitHooksService::PreReceiveError, Repository::CommitError => e
-            raise UpdateError, e.message
+            errors << e.message
           end
         end
       end
+
+      unless errors.empty?
+        raise UpdateError, errors.join("\n\n")
+      end
     end
 
     def update_tags(&block)

app/views/projects/branches/_branch.html.haml

@@ -20,9 +20,7 @@
         protected
 
     - if @project.mirror_ever_updated_successfully? && @repository.diverged_from_upstream?(branch.name)
-      - tooltip_message = "The branch could not be updated automatically because it has diverged from its upstream counterpart."
-      - tooltip_message << "<br>To discard the local changes and overwrite the branch with the upstream version, delete it here and choose 'Update Now' above." if can?(current_user, :push_code, @project)
-      %span.label.label-danger.has_tooltip{data: { html: "true", title: tooltip_message }}
+      %span.label.label-danger.has_tooltip{data: { html: "true", title: branch_diverged_tooltip_message }}
         = icon('exclamation-triangle')
         diverged from upstream
 

app/views/projects/commits/_mirror_status.html.haml

@@ -8,3 +8,8 @@
       This project is mirrored from #{link_to import_url, import_url}.
 
       = render "shared/mirror_status"
+
+      - if @ref.present? && @project.mirror_ever_updated_successfully? && @repository.diverged_from_upstream?(@ref)
+        %span.has_tooltip{data: { html: "true", title: branch_diverged_tooltip_message }}
+          = icon('exclamation-triangle')
+          This branch has diverged from upstream.

doc/integration/elasticsearch.md

@@ -60,13 +60,9 @@ If you have used one of the [Omnibus packages][pkg] to install GitLab, all
 you have to do is edit `/etc/gitlab/gitlab.rb` and add the following lines:
 
 ```ruby
-gitlab_rails['elasticsearch'] = [
-  {
-    "enabled" => "true",
-    "host" => "localhost",
-    "port" => 9200
-  }
-]
+gitlab_rails['elasticsearch_enabled'] = true
+gitlab_rails['elasticsearch_host'] = "localhost"
+gitlab_rails['elasticsearch_port'] = 9200
 ```
 
 Replace the values as you see fit according to the

lib/gitlab/ldap/user.rb

@@ -24,9 +24,6 @@ module Gitlab
         update_user_attributes
       end
 
-      delegate :otp_required_for_login?, :otp_backup_codes, :otp_attempt,
-        to: :gl_user
-
       def gl_user
         @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user
       end

lib/tasks/gitlab/elastic.rake

@@ -2,7 +2,21 @@ namespace :gitlab do
   namespace :elastic do
     desc "Indexing repositories"
     task index_repositories: :environment  do
-      Repository.import
+      Repository.__elasticsearch__.create_index!
+
+      Project.find_each do |project|
+        if project.repository.exists? && !project.repository.empty?
+          puts "Indexing #{project.name_with_namespace}..."
+
+          begin
+            project.repository.index_commits
+            project.repository.index_blobs
+            puts "Done!".green
+          rescue StandardError => e
+            puts "#{e.message}, trace - #{e.backtrace}"
+          end
+        end
+      end
     end
 
     desc "Indexing all wikis"