Merge master into ce-to-ee

CHANGELOG-EE.md

@@ -15,7 +15,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 
 - No changes.
 
-## 8.16.0 (2017-02-22)
+## 8.16.0 (2017-01-22)
 
 - Allow to limit shared runners minutes quota for group. !965
 - About GitLab link in sidebar that links to help page. !1008

app/models/concerns/repository_mirroring.rb

+module RepositoryMirroring
+  def storage_path
+    @project.repository_storage_path
+  end
+
+  def push_remote_branches(remote, branches)
+    gitlab_shell.push_remote_branches(storage_path, path_with_namespace, remote, branches)
+  end
+
+  def delete_remote_branches(remote, branches)
+    gitlab_shell.delete_remote_branches(storage_path, path_with_namespace, remote, branches)
+  end
+
+  def add_remote(name, url)
+    raw_repository.remote_add(name, url)
+  rescue Rugged::ConfigError
+    raw_repository.remote_update(name, url: url)
+  end
+
+  def remove_remote(name)
+    raw_repository.remote_delete(name)
+    true
+  rescue Rugged::ConfigError
+    false
+  end
+
+  def set_remote_as_mirror(name)
+    config = raw_repository.rugged.config
+
+    # This is used by Gitlab Geo to define repository as equivalent as "git clone --mirror"
+    config["remote.#{name}.fetch"] = 'refs/*:refs/*'
+    config["remote.#{name}.mirror"] = true
+    config["remote.#{name}.prune"] = true
+  end
+
+  def fetch_remote(remote, forced: false, no_tags: false)
+    gitlab_shell.fetch_remote(storage_path, path_with_namespace, remote, forced: forced, no_tags: no_tags)
+  end
+
+  def remote_tags(remote)
+    gitlab_shell.list_remote_tags(storage_path, path_with_namespace, remote).map do |name, target|
+      Gitlab::Git::Tag.new(raw_repository, name, target)
+    end
+  end
+
+  def remote_branches(remote_name)
+    branches = []
+
+    rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
+      name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
+
+      begin
+        branches << Gitlab::Git::Branch.new(raw_repository, name, ref.target)
+      rescue Rugged::ReferenceError
+        # Omit invalid branch
+      end
+    end
+
+    branches
+  end
+end

app/models/project.rb

@@ -1548,22 +1548,19 @@ class Project < ActiveRecord::Base
        size_in_bytes + repository_and_lfs_size > actual_size_limit)
   end
 
-  def environments_for(ref, commit: nil, with_tags: false)
-    deployments_query = with_tags ? 'ref = ? OR tag IS TRUE' : 'ref = ?'
-
-    environment_ids = deployments
-      .where(deployments_query, ref.to_s)
-      .group(:environment_id)
-      .select(:environment_id)
-
-    environments_found = environments.available
-      .where(id: environment_ids).to_a
-
-    return environments_found unless commit
+  def route_map_for(commit_sha)
+    @route_maps_by_commit ||= Hash.new do |h, sha|
+      h[sha] = begin
+        data = repository.route_map_for(sha)
+        next unless data
 
-    environments_found.select do |environment|
-      environment.includes_commit?(commit)
+        Gitlab::RouteMap.new(data)
+      rescue Gitlab::RouteMap::FormatError
+        nil
+      end
     end
+
+    @route_maps_by_commit[commit_sha]
   end
 
   def route_map_for(commit_sha)

app/models/repository.rb

@@ -5,6 +5,7 @@ require 'forwardable'
 class Repository
   include Gitlab::ShellAdapter
   include Elastic::RepositoriesSearch
+  include RepositoryMirroring
 
   attr_accessor :path_with_namespace, :project
 
@@ -70,10 +71,6 @@ class Repository
     @raw_repository ||= Gitlab::Git::Repository.new(path_to_repo)
   end
 
-  def storage_path
-    @project.repository_storage_path
-  end
-
   # Return absolute path to repository
   def path_to_repo
     @path_to_repo ||= File.expand_path(
@@ -185,10 +182,6 @@ class Repository
     find_branch(branch_name)
   end
 
-  def push_remote_branches(remote, branches)
-    gitlab_shell.push_remote_branches(storage_path, path_with_namespace, remote, branches)
-  end
-
   def add_tag(user, tag_name, target, message = nil)
     newrev = commit(target).try(:id)
     options = { message: message, tagger: user_to_committer(user) } if message
@@ -210,10 +203,6 @@ class Repository
     true
   end
 
-  def delete_remote_branches(remote, branches)
-    gitlab_shell.delete_remote_branches(storage_path, path_with_namespace, remote, branches)
-  end
-
   def rm_tag(user, tag_name)
     before_remove_tag
     tag = find_tag(tag_name)
@@ -224,40 +213,6 @@ class Repository
     true
   end
 
-  def config
-    raw_repository.rugged.config
-  end
-
-  def add_remote(name, url)
-    raw_repository.remote_add(name, url)
-  rescue Rugged::ConfigError
-    raw_repository.remote_update(name, url: url)
-  end
-
-  def remove_remote(name)
-    raw_repository.remote_delete(name)
-    true
-  rescue Rugged::ConfigError
-    false
-  end
-
-  def set_remote_as_mirror(name)
-    # This is used by Gitlab Geo to define repository as equivalent as "git clone --mirror"
-    config["remote.#{name}.fetch"] = 'refs/*:refs/*'
-    config["remote.#{name}.mirror"] = true
-    config["remote.#{name}.prune"] = true
-  end
-
-  def fetch_remote(remote, forced: false, no_tags: false)
-    gitlab_shell.fetch_remote(storage_path, path_with_namespace, remote, forced: forced, no_tags: no_tags)
-  end
-
-  def remote_tags(remote)
-    gitlab_shell.list_remote_tags(storage_path, path_with_namespace, remote).map do |name, target|
-      Gitlab::Git::Tag.new(raw_repository, name, target)
-    end
-  end
-
   def ref_names
     branch_names + tag_names
   end
@@ -807,22 +762,6 @@ class Repository
 
   alias_method :branches, :local_branches
 
-  def remote_branches(remote_name)
-    branches = []
-
-    rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
-      name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
-
-      begin
-        branches << Gitlab::Git::Branch.new(raw_repository, name, ref.target)
-      rescue Rugged::ReferenceError
-        # Omit invalid branch
-      end
-    end
-
-    branches
-  end
-
   def tags
     @tags ||= raw_repository.tags
   end

db/migrate/20170207150212_add_indexes_to_mirrors.rb

+class AddIndexesToMirrors < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    add_concurrent_index :projects, [:sync_time]
+    add_concurrent_index :remote_mirrors, [:sync_time]
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170206101030) do
+ActiveRecord::Schema.define(version: 20170207150212) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1130,6 +1130,7 @@ ActiveRecord::Schema.define(version: 20170206101030) do
   add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree
   add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree
   add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree
+  add_index "projects", ["sync_time"], name: "index_projects_on_sync_time", using: :btree
   add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree
 
   create_table "protected_branch_merge_access_levels", force: :cascade do |t|
@@ -1212,6 +1213,7 @@ ActiveRecord::Schema.define(version: 20170206101030) do
   end
 
   add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
+  add_index "remote_mirrors", ["sync_time"], name: "index_remote_mirrors_on_sync_time", using: :btree
 
   create_table "routes", force: :cascade do |t|
     t.integer "source_id", null: false

doc/ci/environments.md

@@ -263,7 +263,7 @@ This works just like any other terminal - you'll be in the container created
 by your deployment, so you can run shell commands and get responses in real
 time, check the logs, try out configuration or code tweaks, etc. You can open
 multiple terminals to the same environment - they each get their own shell
-session -  and even a multiplexer like `screen` or `tmux`!  
+session -  and even a multiplexer like `screen` or `tmux`!
 
 >**Note:**
 Container-based deployments often lack basic tools (like an editor), and may

doc/gitlab-geo/configuration.md

@@ -139,10 +139,10 @@ sensitive data in the database. Any secondary node must have the
 
      ```
      # Omnibus GitLab installations
-     cat /etc/gitlab/gitlab-secrets.json
+     cat /etc/gitlab/gitlab-secrets.json | grep db_key_base
 
      # Installations from source
-     cat /home/git/gitlab/config/secrets.yml
+     cat /home/git/gitlab/config/secrets.yml | grep db_key_base
      ```
 
 1. SSH into the **secondary** node and login as root:

doc/gitlab-geo/database.md

@@ -76,8 +76,33 @@ The following guide assumes that:
     ```
 
     Where `1.2.3.4` is the public IP address of the primary server, and `5.6.7.8`
-    the public IP address of the secondary one. If you want to add another
-    secondary, the relevant setting would look like:
+    the public IP address of the secondary one.
+
+    For security reasons, PostgreSQL by default only listens on the local
+    interface (e.g. 127.0.0.1). However, GitLab Geo needs to communicate
+    between the primary and secondary nodes over a common network, such as a
+    corporate LAN or the public Internet. For this reason, we need to
+    configure PostgreSQL to listen on more interfaces.
+
+    The `listen_address` option opens PostgreSQL up to external connections
+    with the interface corresponding to the given IP. See [the PostgreSQL
+    documentation](https://www.postgresql.org/docs/9.6/static/runtime-config-connection.html)
+    for more details.
+
+    Note that if you are running GitLab Geo with a cloud provider (e.g. Amazon
+    Web Services), the internal interface IP (as provided by `ifconfig`) may
+    be different from the public IP address. For example, suppose you have a
+    nodes with the following configuration:
+
+    |Node Type|Internal IP|External IP|
+    |---------|-----------|-----------|
+    |Primary|10.1.5.3|54.193.124.100|
+    |Secondary|10.1.10.5|54.193.100.155|
+
+    In this case, for `1.2.3.4` use the internal IP of the primary node: 10.1.5.3.
+    For `5.6.7.8`, use the external of the secondary node: 54.193.100.155.
+
+    If you want to add another secondary, the relevant setting would look like:
 
     ```ruby
     postgresql['md5_auth_cidr_addresses'] = ['5.6.7.8/32','11.22.33.44/32']
@@ -85,6 +110,8 @@ The following guide assumes that:
 
     Edit the `wal` values as you see fit.
 
+1. Check to make sure your firewall rules are set so that the secondary nodes
+   can access port 5432 on the primary node.
 1. Save the file and [reconfigure GitLab][] for the changes to take effect.
 1. Now that the PostgreSQL server is set up to accept remote connections, run
    `netstat -plnt` to make sure that PostgreSQL is listening to the server's
@@ -119,6 +146,8 @@ The following guide assumes that:
     hot_standby = on
     ```
 
+    See the Omnibus notes above for more details of `listen_address`.
+
     Edit the `wal` values as you see fit.
 
 1. Set the access control on the primary to allow TCP connections using the

spec/models/remote_mirror_spec.rb

@@ -50,7 +50,8 @@ describe RemoteMirror do
 
         mirror.update_attribute(:url, 'http://foo:baz@test.com')
 
-        expect(repo.config["remote.#{mirror.ref_name}.url"]).to eq('http://foo:baz@test.com')
+        config = repo.raw_repository.rugged.config
+        expect(config["remote.#{mirror.ref_name}.url"]).to eq('http://foo:baz@test.com')
       end
     end
   end