Update Migration and Secrets steps in Reference Architecture docs

doc/administration/reference_architectures/2k_users.md

@@ -295,6 +295,9 @@ further configuration steps.
    gitlab_rails['auto_migrate'] = false
    ```
 
+1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace
+   the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step.
+
 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
 1. Note the PostgreSQL node's IP address or hostname, port, and
    plain text password. These will be necessary when configuring the [GitLab
@@ -347,18 +350,17 @@ Omnibus:
    ## Enable Redis
    redis['enable'] = true
 
-   ## Disable all other services
+   # Avoid running unnecessary services on the Redis server
+   gitaly['enable'] = false
+   postgresql['enable'] = false
+   puma['enable'] = false
    sidekiq['enable'] = false
    gitlab_workhorse['enable'] = false
-   puma['enable'] = false
-   postgresql['enable'] = false
-   nginx['enable'] = false
    prometheus['enable'] = false
    alertmanager['enable'] = false
-   pgbouncer_exporter['enable'] = false
-   gitlab_exporter['enable'] = false
-   gitaly['enable'] = false
    grafana['enable'] = false
+   gitlab_exporter['enable'] = false
+   nginx['enable'] = false
 
    redis['bind'] = '0.0.0.0'
    redis['port'] = 6379
@@ -375,7 +377,11 @@ Omnibus:
    }
    ```
 
+1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace
+   the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step.
+
 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
+
 1. Note the Redis node's IP address or hostname, port, and
    Redis password. These will be necessary when [configuring the GitLab
    application servers](#configure-gitlab-rails) later.
@@ -454,15 +460,14 @@ To configure the Gitaly server, on the server node you want to use for Gitaly:
    # Avoid running unnecessary services on the Gitaly server
    postgresql['enable'] = false
    redis['enable'] = false
-   nginx['enable'] = false
    puma['enable'] = false
    sidekiq['enable'] = false
    gitlab_workhorse['enable'] = false
-   grafana['enable'] = false
-
-   # If you run a separate monitoring node you can disable these services
-   alertmanager['enable'] = false
    prometheus['enable'] = false
+   alertmanager['enable'] = false
+   grafana['enable'] = false
+   gitlab_exporter['enable'] = false
+   nginx['enable'] = false
 
    # Prevent database migrations from running on upgrade automatically
    gitlab_rails['auto_migrate'] = false
@@ -470,9 +475,11 @@ To configure the Gitaly server, on the server node you want to use for Gitaly:
    # Configure the gitlab-shell API callback URL. Without this, `git push` will
    # fail. This can be your 'front door' GitLab URL or an internal load
    # balancer.
-   # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server.
    gitlab_rails['internal_api_url'] = 'https://gitlab.example.com'
 
+   # Gitaly
+   gitaly['enable'] = true
+
    # Make Gitaly accept connections on all network interfaces. You must use
    # firewalls to restrict access to this address/port.
    # Comment out following line if you only want to support TLS connections
@@ -492,7 +499,11 @@ To configure the Gitaly server, on the server node you want to use for Gitaly:
    })
    ```
 
-1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
+1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace
+   the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step.
+
+1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
+
 1. Confirm that Gitaly can perform callbacks to the internal API:
 
    ```shell
@@ -656,10 +667,7 @@ On each node perform the following:
    gitlab_rails['monitoring_whitelist'] = ['<MONITOR NODE IP>/32', '127.0.0.0/8']
    nginx['status']['options']['allow'] = ['<MONITOR NODE IP>/32', '127.0.0.0/8']
 
-   #############################
-   ###     Object storage    ###
-   #############################
-
+   # Object Storage
    # This is an example for configuring Object Storage on GCP
    # Replace this config with your chosen Object Storage provider as desired
    gitlab_rails['object_store']['connection'] = {
@@ -675,6 +683,13 @@ On each node perform the following:
    gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = "<gcp-dependency-proxy-bucket-name>"
    gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = "<gcp-terraform-state-bucket-name>"
 
+   gitlab_rails['backup_upload_connection'] = {
+     'provider' => 'Google',
+     'google_project' => '<gcp-project-name>',
+     'google_json_key_location' => '<path-to-gcp-service-account-key>'
+   }
+   gitlab_rails['backup_upload_remote_directory'] = "<gcp-backups-state-bucket-name>"
+
    ## Uncomment and edit the following options if you have set up NFS
    ##
    ## Prevent GitLab from starting if NFS data mounts are not available
@@ -708,7 +723,20 @@ On each node perform the following:
       sudo cp cert.pem /etc/gitlab/trusted-certs/
       ```
 
-1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
+1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace
+   the file of the same name on this server. If this is the first Omnibus node you are configuring then you can skip this step.
+
+1. To ensure database migrations are only run during reconfigure and not automatically on upgrade, run:
+
+   ```shell
+   sudo touch /etc/gitlab/skip-auto-reconfigure
+   ```
+
+   Only a single designated node should handle migrations as detailed in the
+   [GitLab Rails post-configuration](#gitlab-rails-post-configuration) section.
+
+1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
+
 1. Run `sudo gitlab-rake gitlab:gitaly:check` to confirm the node can connect to Gitaly.
 1. Tail the logs to see the requests:
 
@@ -716,11 +744,6 @@ On each node perform the following:
    sudo gitlab-ctl tail gitaly
    ```
 
-1. Save the `/etc/gitlab/gitlab-secrets.json` file from one of the two
-   application nodes and install it on the other application node and the
-   [Gitaly node](#configure-gitaly) and
-   [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
-
 When you specify `https` in the `external_url`, as in the previous example,
 GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the
 certificates aren't present, NGINX will fail to start. For more information, see
@@ -777,19 +800,14 @@ running [Prometheus](../monitoring/prometheus/index.md) and
    grafana['enable'] = true
    grafana['admin_password'] = 'toomanysecrets'
 
-   # Disable all other services
-   alertmanager['enable'] = false
+   # Avoid running unnecessary services on the Prometheus server
    gitaly['enable'] = false
-   gitlab_exporter['enable'] = false
-   gitlab_workhorse['enable'] = false
-   nginx['enable'] = true
-   postgres_exporter['enable'] = false
    postgresql['enable'] = false
    redis['enable'] = false
-   redis_exporter['enable'] = false
-   sidekiq['enable'] = false
    puma['enable'] = false
-   node_exporter['enable'] = false
+   sidekiq['enable'] = false
+   gitlab_workhorse['enable'] = false
+   alertmanager['enable'] = false
    gitlab_exporter['enable'] = false
 
    # Prevent database migrations from running on upgrade automatically