Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ee into ce_upstream

CHANGELOG

@@ -5,6 +5,7 @@ v 8.8.0 (unreleased)
 v 8.7.0 (unreleased)
   - The number of InfluxDB points stored per UDP packet can now be configured
   - Fix error when cross-project label reference used with non-existent project
+v 8.7.0
   - Transactions for /internal/allowed now have an "action" tag set
   - Method instrumentation now uses Module#prepend instead of aliasing methods
   - Repository.clean_old_archives is now instrumented

CHANGELOG-EE

 Please view this file on the master branch, on stable branches it's out of date.
 
-v 8.7.0 (unreleased)
+v 8.7.0
   - Update GitLab Pages to 0.2.1: support user-defined 404 pages
   - Refactor group sync to pull access level logic to its own class. !306
   - [Elastic] Stabilize database indexer if database is inconsistent
   - Add ability to sync to remote mirrors. !249
+  - GitLab Geo: Many replication improvements and fixes !354
 
 v 8.6.7
   - No EE-specific changes

app/controllers/admin/geo_nodes_controller.rb

 class Admin::GeoNodesController < Admin::ApplicationController
+  before_action :check_license
+
   def index
     @nodes = GeoNode.all
     @node = GeoNode.new
@@ -41,4 +43,11 @@ class Admin::GeoNodesController < Admin::ApplicationController
   def geo_node_params
     params.require(:geo_node).permit(:url, :primary, geo_node_key_attributes: [:key])
   end
+
+  def check_license
+    unless Gitlab::Geo.license_allows?
+      flash[:alert] = 'You need a diferent license to enable Geo replication'
+      redirect_to admin_license_path
+    end
+  end
 end

app/services/geo/move_repository_service.rb

+module Geo
+  class MoveRepositoryService
+    include Gitlab::ShellAdapter
+
+    attr_reader :id, :name, :old_path_with_namespace, :new_path_with_namespace
+
+    def initialize(id, name, old_path_with_namespace, new_path_with_namespace)
+      @id = id
+      @name = name
+      @old_path_with_namespace = old_path_with_namespace
+      @new_path_with_namespace = new_path_with_namespace
+    end
+
+    def execute
+      project = Project.find(id)
+      project.expire_caches_before_rename(old_path_with_namespace)
+
+      # Make sure target directory exists (used when transfering repositories)
+      project.namespace.ensure_dir_exist
+
+      if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
+        # If repository moved successfully we need to send update instructions to users.
+        # However we cannot allow rollback since we moved repository
+        # So we basically we mute exceptions in next actions
+        begin
+          gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
+        rescue
+          # Returning false does not rollback after_* transaction but gives
+          # us information about failing some of tasks
+          false
+        end
+      else
+        # if we cannot move namespace directory we should rollback
+        # db changes in order to prevent out of sync between db and fs
+        raise Exception.new('repository cannot be renamed')
+      end
+    end
+  end
+end

app/services/geo/schedule_repo_destroy_service.rb

+module Geo
+  class ScheduleRepoDestroyService
+    attr_reader :id, :name, :path_with_namespace
+
+    def initialize(params)
+      @id = params['project_id']
+      @name = params['name']
+      @path_with_namespace = params['path_with_namespace']
+    end
+
+    def execute
+      GeoRepositoryDestroyWorker.perform_async(id, name, path_with_namespace)
+    end
+  end
+end

app/services/geo/schedule_repo_move_service.rb

+module Geo
+  class ScheduleRepoRenameService
+    attr_reader :id, :name, :old_path_with_namespace, :path_with_namespace
+
+    def initialize(params)
+      @id = params['project_id']
+      @name = params['name']
+      @old_path_with_namespace = params['old_path_with_namespace']
+      @path_with_namespace = params['path_with_namespace']
+    end
+
+    def execute
+      GeoRepositoryMoveWorker.perform_async(id, name, old_path_with_namespace, path_with_namespace)
+    end
+  end
+end

app/services/projects/destroy_service.rb

@@ -17,9 +17,6 @@ module Projects
 
       project.team.truncate
 
-      repo_path = project.path_with_namespace
-      wiki_path = repo_path + '.wiki'
-
       # Flush the cache for both repositories. This has to be done _before_
       # removing the physical repositories as some expiration code depends on
       # Git data (e.g. a list of branch names).
@@ -27,14 +24,7 @@ module Projects
 
       Project.transaction do
         project.destroy!
-
-        unless remove_repository(repo_path)
-          raise_error('Failed to remove project repository. Please try again or contact administrator')
-        end
-
-        unless remove_repository(wiki_path)
-          raise_error('Failed to remove wiki repository. Please try again or contact administrator')
-        end
+        trash_repositories!
       end
 
       log_info("Project \"#{project.name}\" was removed")
@@ -42,8 +32,39 @@ module Projects
       true
     end
 
+    # Removes physical repository in a Geo replicated secondary node
+    # There is no need to do any database operation as it will be
+    # replicated by itself.
+    def geo_replicate
+      # Flush the cache for both repositories. This has to be done _before_
+      # removing the physical repositories as some expiration code depends on
+      # Git data (e.g. a list of branch names).
+      flush_caches(project, wiki_path)
+
+      trash_repositories!
+      log_info("Project \"#{project.name}\" was removed")
+    end
+
     private
 
+    def repo_path
+      project.path_with_namespace
+    end
+
+    def wiki_path
+      repo_path + '.wiki'
+    end
+
+    def trash_repositories!
+      unless remove_repository(repo_path)
+        raise_error('Failed to remove project repository. Please try again or contact administrator')
+      end
+
+      unless remove_repository(wiki_path)
+        raise_error('Failed to remove wiki repository. Please try again or contact administrator')
+      end
+    end
+
     def remove_repository(path)
       # Skip repository removal. We use this flag when remove user or group
       return true if params[:skip_repo] == true

app/workers/geo_repository_destroy_worker.rb

+class GeoRepositoryDestroyWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :default
+
+  def perform(id, name, path_with_namespace)
+    # We don't have access to the original model anymore, so we are
+    # rebuilding only what our service class requires
+    project = FakeProject.new(id, name, path_with_namespace)
+
+    ::Projects::DestroyService.new(project, nil).geo_replicate
+  end
+
+  FakeProject = Struct.new(:id, :name, :path_with_namespace) do
+    def repository
+      @repository ||= Repository.new(path_with_namespace, self)
+    end
+  end
+end

app/workers/geo_repository_move_worker.rb

+class GeoRepositoryMoveWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :default
+
+  def perform(id, name, old_path_with_namespace, new_path_with_namespace)
+    Geo::MoveRepositoryService.new(id, name, old_path_with_namespace, new_path_with_namespace).execute
+  end
+end

app/workers/historical_data_worker.rb

@@ -2,6 +2,7 @@ class HistoricalDataWorker
   include Sidekiq::Worker
 
   def perform
+    return if Gitlab::Geo.secondary?
     HistoricalData.track!
   end
 end

app/workers/stuck_ci_builds_worker.rb

@@ -4,6 +4,7 @@ class StuckCiBuildsWorker
   BUILD_STUCK_TIMEOUT = 1.day
 
   def perform
+    return if Gitlab::Geo.secondary?
     Rails.logger.info 'Cleaning stuck builds'
 
     builds = Ci::Build.running_or_pending.where('updated_at < ?', BUILD_STUCK_TIMEOUT.ago)

doc/gitlab-geo/README.md

@@ -54,7 +54,7 @@ Geo instances. Follow the steps below in the order that they appear:
 
 1. Install GitLab Enterprise Edition on the server that will serve as the
    secondary Geo node
-1. [Setup a database replication](./database.md) in `master <-> slave` topology
+1. [Setup a database replication](./database.md) in `primary <-> secondary (read-only)` topology
 1. [Configure GitLab](configuration.md) and set the primary and secondary nodes
 After you set up the database replication and configure the GitLab Geo nodes,
 there are a few things to consider:

doc/gitlab-geo/configuration.md

@@ -14,7 +14,8 @@ complete the process.
 - [Create SSH key pairs for Geo nodes](#create-ssh-key-pairs-for-geo-nodes)
 - [Primary Node GitLab setup](#primary-node-gitlab-setup)
 - [Secondary Node GitLab setup](#secondary-node-gitlab-setup)
-    - [Authorized keys regeneration](#authorized-keys-regeneration)
+  - [Database Encryptation Key](#database-encryptation-key)
+  - [Authorized keys regeneration](#authorized-keys-regeneration)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -81,8 +82,8 @@ Host example.com                    # The FQDN of the primary Geo node
 ## Primary Node GitLab setup
 
 >**Note:**
-You will need to setup your database into a **Master <-> Slave** replication
-topology, and your Primary node should always point to a database's Master
+You will need to setup your database into a **Primary <-> Secondary (read-only)** replication
+topology, and your Primary node should always point to a database's Primary
 instance. If you haven't done that already, read [database replication](./database.md).
 
 Go to the server that you chose to be your primary, and visit
@@ -119,6 +120,18 @@ Edition installation, with some extra requirements:
 - Your secondary node should be allowed to communicate via HTTP/HTTPS and
   SSH with your primary node (make sure your firewall is not blocking that).
 
+### Database Encryption Key
+
+GitLab stores a unique encryption key in disk that we use to safely store sensitive
+data in the database.
+
+Any secondary node must have the exact same value for `db_key_base` as defined in the primary one.
+
+For Omnibus installations it is stored at `/etc/gitlab/gitlab-secrets.json`.
+
+For Source installations it is stored at `/home/git/gitlab/config/secrets.yml`.
+
+
 ### Authorized keys regeneration
 
 The final step will be to regenerate the keys for `.ssh/authorized_keys` using

doc/gitlab-geo/database.md

@@ -4,12 +4,13 @@ This document describes the minimal steps you have to take in order to
 replicate your GitLab database into another server. You may have to change
 some values according to your database setup, how big it is, etc.
 
-The GitLab primary node where the write operations happen will act as `master`,
-and the secondary ones which are read-only will act as `slaves`.
+The GitLab primary node where the write operations happen will connect to
+`primary` database server, and the secondary ones which are read-only will
+connect to `secondary` database servers (which are read-only too).
 
 >**Note:**
-To be on par with GitLab's notation, we will use `primary` to denote the `master`
-server, and `secondary` for the `slave`.
+In many databases documentation you will see `primary` being references as `master`
+and `secondary` as either `slave` or `standby` server (read-only).
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->

doc/update/README.md

-Depending on the installation method and your GitLab version, there are multiple update guides. Choose one that fits your needs.
+# Updating GitLab
 
-## CE to EE
+Depending on the installation method and your GitLab version, there are multiple
+update guides.
 
-- [The CE to EE update guides](https://gitlab.com/gitlab-org/gitlab-ee/tree/master/doc/update) the steps are very similar to a version upgrade: stop the server, get the code, update config files for the new functionality, install libs and do migrations, update the init script, start the application and check the application status.
+There are currently 3 official ways to install GitLab:
 
-## EE to CE
+- Omnibus packages
+- Source installation
+- Docker installation
 
-- If you need to downgrade your EE installation back to CE, you can follow [this guide](../downgrade_ee_to_ce/README.md) to make
-the process as smooth as possible.
+Based on your installation, choose a section below that fits your needs.
+
+---
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [Omnibus Packages](#omnibus-packages)
+- [Installation from source](#installation-from-source)
+- [Installation using Docker](#installation-using-docker)
+- [Upgrading between editions](#upgrading-between-editions)
+    - [Community to Enterprise Edition](#community-to-enterprise-edition)
+    - [Enterprise to Community Edition](#enterprise-to-community-edition)
+- [Miscellaneous](#miscellaneous)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
 ## Omnibus Packages
 
-- [Omnibus update guide](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/update.md) contains the steps needed to update a GitLab [package](https://about.gitlab.com/downloads/).
+- The [Omnibus update guide](http://doc.gitlab.com/omnibus/update/README.html)
+  contains the steps needed to update an Omnibus GitLab package.
 
 ## Installation from source
 
-- [The individual upgrade guides](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/update) are for those who have installed GitLab from source.
-- [The CE to EE update guides](https://gitlab.com/gitlab-org/gitlab-ee/tree/master/doc/update). The steps are very similar to a version upgrade: stop the server, get the code, update config files for the new functionality, install libs and do migrations, update the init script, start the application and check the application status.
-- [Upgrader](upgrader.md) is an automatic ruby script that performs the update for installations from source.
-- [Patch versions](patch_versions.md) guide includes the steps needed for a patch version, eg. 6.2.0 to 6.2.1.
+- [Upgrading Community Edition from source][source-ce] - The individual
+  upgrade guides are for those who have installed GitLab CE from source.
+- [Upgrading Enterprise Edition from source][source-ee] - The individual
+  upgrade guides are for those who have installed GitLab EE from source.
+- [Patch versions](patch_versions.md) guide includes the steps needed for a
+  patch version, eg. 6.2.0 to 6.2.1, and apply to both Community and Enterprise
+  Editions.
+
+## Installation using Docker
+
+GitLab provides official Docker images for both Community and Enterprise
+editions. They are based on the Omnibus package and instructions on how to
+update them are in [a separate document][omnidocker].
+
+## Upgrading between editions
+
+GitLab comes in two flavors: [Community Edition][ce] which is MIT licensed,
+and [Enterprise Edition][ee] which builds on top of the Community Edition and
+includes extra features mainly aimed at organizations with more than 100 users.
+
+Below you can find some guides to help you change editions easily.
+
+### Community to Enterprise Edition
+
+>**Note:**
+The following guides are for subscribers of the Enterprise Edition only.
+
+If you wish to upgrade your GitLab installation from Community to Enterprise
+Edition, follow the guides below based on the installation method:
+
+- [Source CE to EE update guides][source-ee] - Find your version, and follow the
+  `-ce-to-ee.md` guide. The steps are very similar to a version upgrade: stop
+  the server, get the code, update config files for the new functionality,
+  install libraries and do migrations, update the init script, start the
+  application and check its status.
+- [Omnibus CE to EE][omni-ce-ee] - Follow this guide to update your Omnibus
+  GitLab Community Edition to the Enterprise Edition.
+
+### Enterprise to Community Edition
+
+If you need to downgrade your Enterprise Edition installation back to Community
+Edition, you can follow [this guide][ee-ce] to make the process as smooth as
+possible.
 
 ## Miscellaneous
 
-- [MySQL to PostgreSQL](mysql_to_postgresql.md) guides you through migrating your database from MySQL to PostgreSQL.
-- [MySQL installation guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/database_mysql.md) contains additional information about configuring GitLab to work with a MySQL database.
+- [MySQL to PostgreSQL](mysql_to_postgresql.md) guides you through migrating
+  your database from MySQL to PostgreSQL.
+- [MySQL installation guide](../install/database_mysql.md) contains additional
+  information about configuring GitLab to work with a MySQL database.
 - [Restoring from backup after a failed upgrade](restore_after_failure.md)
+
+[omnidocker]: http://doc.gitlab.com/omnibus/docker/README.html
+[source-ee]: https://gitlab.com/gitlab-org/gitlab-ee/tree/master/doc/update
+[source-ce]: https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/update
+[ee-ce]: ../downgrade_ee_to_ce/README.md
+[ce]: https://about.gitlab.com/features/#community
+[ee]: https://about.gitlab.com/features/#enterprise
+[omni-ce-ee]: http://doc.gitlab.com/omnibus/update/README.html#from-community-edition-to-enterprise-edition

lib/api/geo.rb

@@ -30,6 +30,15 @@ module API
         when 'tag_push'
           required_attributes! %w(event_name project_id project)
           ::Geo::ScheduleWikiRepoUpdateService.new(params).execute
+        when 'project_destroy'
+          required_attributes! %w(event_name project_id path_with_namespace)
+          ::Geo::ScheduleRepoDestroyService.new(params).execute
+        when 'project_rename'
+          required_attributes! %w(event_name project_id path_with_namespace old_path_with_namespace)
+          ::Geo::ScheduleRepoRenameService.new(params).execute
+        when 'project_transfer'
+          required_attributes! %w(event_name project_id path_with_namespace old_path_with_namespace)
+          ::Geo::ScheduleRepoRenameService.new(params).execute
         end
       end
     end

lib/gitlab/geo.rb

@@ -22,6 +22,10 @@ module Gitlab
       RequestStore.store[:geo_node_enabled] ||= GeoNode.exists?
     end
 
+    def self.license_allows?
+      ::License.current && ::License.current.add_on?('GitLab_Geo')
+    end
+
     def self.primary?
       RequestStore.store[:geo_node_primary?] ||= self.enabled? && self.current_node && self.current_node.primary?
     end

lib/gitlab/git_access.rb

@@ -70,6 +70,10 @@ module Gitlab
         return build_status_object(false, 'The project you were looking for could not be found.')
       end
 
+      if Gitlab::Geo.secondary? && !Gitlab::Geo.license_allows?
+        return build_status_object(false, 'Your current license does not have GitLab Geo add-on enabled.')
+      end
+
       case cmd
       when *DOWNLOAD_COMMANDS
         download_access_check
@@ -94,7 +98,7 @@ module Gitlab
 
     def push_access_check(changes)
 
-      if Gitlab::Geo.enabled? && Gitlab::Geo.secondary?
+      if Gitlab::Geo.secondary?
         return build_status_object(false, "You can't push code on a secondary GitLab Geo node.")
       end
 

lib/gitlab/middleware/readonly_geo.rb

@@ -13,8 +13,8 @@ module Gitlab
         @env = env
 
         if disallowed_request? && Gitlab::Geo.secondary?
-          Rails.logger.debug('Gitlab Geo: preventing possible non readonly operation')
-          error_message = 'You cannot do writing operations on a secondary Gitlab Geo instance'
+          Rails.logger.debug('GitLab Geo: preventing possible non readonly operation')
+          error_message = 'You cannot do writing operations on a secondary GitLab Geo instance'
 
           if json_request?
             return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]]
@@ -60,12 +60,20 @@ module Gitlab
       end
 
       def whitelisted_routes
-        logout_route || WHITELISTED.any? { |path| @request.path.include?(path) }
+        logout_route || grack_route || WHITELISTED.any? { |path| @request.path.include?(path) } || sidekiq_route
       end
 
       def logout_route
         route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy'
       end
+
+      def sidekiq_route
+        @request.path.start_with?('/admin/sidekiq')
+      end
+
+      def grack_route
+        @request.path.end_with?('.git/git-upload-pack')
+      end
     end
   end
 end

spec/lib/gitlab/geo_spec.rb

@@ -68,4 +68,21 @@ describe Gitlab::Geo, lib: true do
       expect(described_class.geo_node?(host: 'inexistent', port: 1234)).to be_falsey
     end
   end
+
+  describe 'license_allows?' do
+    it 'returns true if license has Geo addon' do
+      allow_any_instance_of(License).to receive(:add_on?).with('GitLab_Geo') { true }
+      expect(described_class.license_allows?).to be_truthy
+    end
+
+    it 'returns false if license doesnt have Geo addon' do
+      allow_any_instance_of(License).to receive(:add_on?).with('GitLab_Geo') { false }
+      expect(described_class.license_allows?).to be_falsey
+    end
+
+    it 'returns false if no license is present' do
+      allow(License).to receive(:current) { nil }
+      expect(described_class.license_allows?).to be_falsey
+    end
+  end
 end