Merge branch '10586-add-flag-to-object-storage' into 'master'

Make Object Storage synchronization in Geo Nodes configurable via Admin UI

See merge request gitlab-org/gitlab-ee!15139

db/migrate/20190715173819_add_object_storage_flag_to_geo_node.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddObjectStorageFlagToGeoNode < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :geo_nodes, :sync_object_storage, :boolean, default: false
+  end
+
+  def down
+    remove_column :geo_nodes, :sync_object_storage
+  end
+end

db/schema.rb

@@ -1456,6 +1456,7 @@ ActiveRecord::Schema.define(version: 2019_08_06_071559) do
     t.integer "container_repositories_max_capacity", default: 10, null: false
     t.datetime_with_timezone "created_at"
     t.datetime_with_timezone "updated_at"
+    t.boolean "sync_object_storage", default: false, null: false
     t.index ["access_key"], name: "index_geo_nodes_on_access_key"
     t.index ["name"], name: "index_geo_nodes_on_name", unique: true
     t.index ["primary"], name: "index_geo_nodes_on_primary"

doc/api/geo_nodes.md

@@ -10,7 +10,7 @@ GET /geo_nodes
 ```
 
 ```bash
-curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/geo_nodes
+curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes
 ```
 
 Example response:
@@ -29,7 +29,13 @@ Example response:
     "repos_max_capacity": 25,
     "container_repositories_max_capacity": 10,
     "verification_max_capacity": 100,
-    "clone_protocol": "http"
+    "clone_protocol": "http",
+    "web_edit_url": "https://primary.example.com/admin/geo/nodes/1/edit",
+    "_links": {
+      "self": "https://primary.example.com/api/v4/geo_nodes/1",
+      "status":"https://primary.example.com/api/v4/geo_nodes/1/status",
+      "repair":"https://primary.example.com/api/v4/geo_nodes/1/repair"
+    }
   },
   {
     "id": 2,
@@ -43,7 +49,15 @@ Example response:
     "repos_max_capacity": 25,
     "container_repositories_max_capacity": 10,
     "verification_max_capacity": 100,
-    "clone_protocol": "http"
+    "sync_object_storage": true,
+    "clone_protocol": "http",
+    "web_edit_url": "https://primary.example.com/admin/geo/nodes/2/edit",
+    "web_geo_projects_url": "https://secondary.example.com/admin/geo/projects",
+    "_links": {
+      "self":"https://primary.example.com/api/v4/geo_nodes/2",
+      "status":"https://primary.example.com/api/v4/geo_nodes/2/status",
+      "repair":"https://primary.example.com/api/v4/geo_nodes/2/repair"
+    }
   }
 ]
 ```
@@ -55,7 +69,7 @@ GET /geo_nodes/:id
 ```
 
 ```bash
-curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/geo_nodes/1
+curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/1
 ```
 
 Example response:
@@ -73,7 +87,13 @@ Example response:
   "repos_max_capacity": 25,
   "container_repositories_max_capacity": 10,
   "verification_max_capacity": 100,
-  "clone_protocol": "http"
+  "clone_protocol": "http",
+  "web_edit_url": "https://primary.example.com/admin/geo/nodes/1/edit",
+  "_links": {
+    "self": "https://primary.example.com/api/v4/geo_nodes/1",
+    "status":"https://primary.example.com/api/v4/geo_nodes/1/status",
+    "repair":"https://primary.example.com/api/v4/geo_nodes/1/repair"
+  }
 }
 ```
 
@@ -88,7 +108,7 @@ PUT /geo_nodes/:id
 ```
 
 | Attribute                   | Type    | Required  | Description                                                               |
-|----------------------|---------|-----------|---------------------------------------------------------------------------|
+|-----------------------------|---------|-----------|---------------------------------------------------------------------------|
 | `id`                        | integer | yes       | The ID of the Geo node.                                                   |
 | `enabled`                   | boolean | no        | Flag indicating if the Geo node is enabled.                               |
 | `name`                      | string  | yes       | The unique identifier for the Geo node. Must match `geo_node_name` if it is set in gitlab.rb, otherwise it must match `external_url`. |
@@ -98,6 +118,7 @@ PUT /geo_nodes/:id
 | `repos_max_capacity`        | integer | no        | Control the maximum concurrency of repository backfill for this secondary node.     |
 | `verification_max_capacity` | integer | no        | Control the maximum concurrency of verification for this node. |
 | `container_repositories_max_capacity` | integer | no | Control the maximum concurrency of container repository sync for this node. |
+| `sync_object_storage`       | boolean | no        | Flag indicating if the secondary Geo node will replicate blobs in Object Storage. |
 
 Example response:
 
@@ -114,7 +135,15 @@ Example response:
   "repos_max_capacity": 25,
   "container_repositories_max_capacity": 10,
   "verification_max_capacity": 100,
-  "clone_protocol": "http"
+  "sync_object_storage": true,
+  "clone_protocol": "http",
+  "web_edit_url": "https://primary.example.com/admin/geo/nodes/2/edit",
+  "web_geo_projects_url": "https://secondary.example.com/admin/geo/projects",
+  "_links": {
+    "self":"https://primary.example.com/api/v4/geo_nodes/2",
+    "status":"https://primary.example.com/api/v4/geo_nodes/2/status",
+    "repair":"https://primary.example.com/api/v4/geo_nodes/2/repair"
+  }
 }
 ```
 
@@ -158,7 +187,13 @@ Example response:
   "repos_max_capacity": 25,
   "container_repositories_max_capacity": 10,
   "verification_max_capacity": 100,
-  "clone_protocol": "http"
+  "clone_protocol": "http",
+  "web_edit_url": "https://primary.example.com/admin/geo/nodes/1/edit",
+  "_links": {
+    "self": "https://primary.example.com/api/v4/geo_nodes/1",
+    "status":"https://primary.example.com/api/v4/geo_nodes/1/status",
+    "repair":"https://primary.example.com/api/v4/geo_nodes/1/repair"
+  }
 }
 ```
 
@@ -169,7 +204,7 @@ GET /geo_nodes/status
 ```
 
 ```bash
-curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/geo_nodes/status
+curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/status
 ```
 
 Example response:
@@ -320,7 +355,7 @@ GET /geo_nodes/:id/status
 ```
 
 ```bash
-curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/geo_nodes/2/status
+curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/2/status
 ```
 
 Example response:
@@ -394,7 +429,7 @@ GET /geo_nodes/current/failures
 This endpoint uses [Pagination](README.md#pagination).
 
 ```bash
-curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/geo_nodes/current/failures
+curl --header "PRIVATE-TOKEN: <your_access_token>" https://primary.example.com/api/v4/geo_nodes/current/failures
 ```
 
 Example response:

ee/app/assets/javascripts/pages/admin/geo/nodes/shared/init_form.js

@@ -22,7 +22,7 @@ export default function geoNodeForm() {
   const $container = $('.js-geo-node-form');
   const $namespaces = $('.js-hide-if-geo-primary', $container);
   const $reverification = $('.js-hide-if-geo-secondary', $container);
-  const $primaryCheckbox = $('input[type="checkbox"]', $container);
+  const $primaryCheckbox = $('input#geo_node_primary', $container);
   const $selectiveSyncTypeSelect = $('.js-geo-node-selective-sync-type', $container);
   const $select2Dropdown = $('.js-geo-node-namespaces', $container);
   const $syncByNamespaces = $('.js-sync-by-namespace', $container);

ee/app/controllers/admin/geo/nodes_controller.rb

@@ -58,6 +58,7 @@ class Admin::Geo::NodesController < Admin::Geo::ApplicationController
       :verification_max_capacity,
       :minimum_reverification_interval,
       :container_repositories_max_capacity,
+      :sync_object_storage,
       selective_sync_shards: []
     )
   end

ee/app/views/admin/geo/nodes/_form.html.haml

@@ -10,12 +10,6 @@
     = form.text_field :url, class: 'form-control qa-node-url-field'
     .form-text.text-muted= _('The user-facing URL of the Geo node')
 
-  .form-group.col-sm-6.js-internal-url{ class: ('hidden' unless geo_node.primary?) }
-    = form.label :internal_url, s_('Geo|Internal URL (optional)'), class: 'font-weight-bold'
-    = form.text_field :internal_url, class: 'form-control'
-    .form-text.text-muted= s_('Geo|The URL defined on the primary node that secondary nodes should use to contact it. Defaults to URL')
-
-
 .form-group.row
   .col-sm-10
     .form-check
@@ -23,6 +17,12 @@
       = form.label :primary, class: 'form-check-label' do
         %span= s_('Geo|This is a primary node')
 
+.form-row.form-group.js-internal-url{ class: ('hidden' unless geo_node.primary?) }
+  .col-sm-6
+    = form.label :internal_url, s_('Geo|Internal URL (optional)'), class: 'font-weight-bold'
+    = form.text_field :internal_url, class: 'form-control'
+    .form-text.text-muted= s_('Geo|The URL defined on the primary node that secondary nodes should use to contact it. Defaults to URL')
+
 .form-row.form-group.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
   .col-sm-4
     = form.label :selective_sync_type, s_('Geo|Selective synchronization'), class: 'font-weight-bold'
@@ -69,3 +69,14 @@
     = form.label :minimum_reverification_interval, s_('Geo|Re-verification interval'), class: 'font-weight-bold'
     = form.number_field :minimum_reverification_interval, class: 'form-control col-sm-2', min: 1
     .form-text.text-muted= s_('Geo|Control the minimum interval in days that a repository should be reverified for this primary node')
+
+- if ::Feature.enabled?(:geo_object_storage_replication)
+  .form-group.row.js-hide-if-geo-primary{ class: ('hidden' unless geo_node.secondary?) }
+    .col-sm-10
+      = form.label :sync_object_storage, _('Object Storage replication'), class: 'label-bold'
+      .form-check
+        = form.check_box :sync_object_storage, class: 'form-check-input'
+        = form.label :sync_object_storage, class: 'form-check-label' do
+          %span= s_('Geo|Allow this secondary node to replicate content on Object Storage')
+        .form-text.text-muted= s_('Geo|If enabled, and if object storage is enabled, GitLab will handle Object Storage replication using Geo')
+

ee/changelogs/unreleased/10586-make-object-storage-replication-configurable.yml

+---
+title: 'Geo: Make Object Storage synchronization in Geo Nodes configurable via Admin UI'
+merge_request: 15000
+author:
+type: added

ee/lib/api/geo_nodes.rb

@@ -147,6 +147,7 @@ module API
           optional :repos_max_capacity, type: Integer, desc: 'Control the maximum concurrency of repository backfill for this secondary node'
           optional :verification_max_capacity, type: Integer, desc: 'Control the maximum concurrency of repository verification for this node'
           optional :container_repositories_max_capacity, type: Integer, desc: 'Control the maximum concurrency of container repository sync for this node'
+          optional :sync_object_storage, type: Boolean, desc: 'Flag indicating if the secondary Geo node will replicate blobs in Object Storage'
         end
         put do
           not_found!('GeoNode') unless geo_node

ee/lib/ee/api/entities.rb

@@ -485,6 +485,7 @@ module EE
         expose :repos_max_capacity
         expose :verification_max_capacity
         expose :container_repositories_max_capacity
+        expose :sync_object_storage, if: ->(geo_node, _) { ::Feature.enabled?(:geo_object_storage_replication) && geo_node.secondary? }
 
         # Retained for backwards compatibility. Remove in API v5
         expose :clone_protocol do |_record, _options|

ee/spec/factories/geo_nodes.rb

@@ -9,10 +9,12 @@ FactoryBot.define do
     end
 
     primary false
+    sync_object_storage true
 
     trait :primary do
       primary true
       minimum_reverification_interval 7
+      sync_object_storage false
     end
   end
 end

ee/spec/features/admin/geo/admin_geo_nodes_spec.rb

@@ -89,16 +89,48 @@ describe 'admin Geo Nodes', :js, :geo do
       end
     end
 
-    it 'changes re-verification interval field visibility based on primary node checkbox' do
-      expect(page).not_to have_field('Re-verification interval')
+    it 'toggles the visibility of secondary only params based on primary node checkbox' do
+      primary_only_fields = [
+        'Internal URL (optional)',
+        'Re-verification interval'
+      ]
+
+      secondary_only_fields = [
+        'Selective synchronization',
+        'Repository sync capacity',
+        'File sync capacity',
+        'Object Storage replication'
+      ]
+
+      expect(page).to have_unchecked_field('This is a primary node')
+
+      primary_only_fields.each do |field|
+        expect(page).to have_field(field, visible: false)
+      end
+
+      secondary_only_fields.each do |field|
+        expect(page).to have_field(field)
+      end
 
       check 'This is a primary node'
 
-      expect(page).to have_field('Re-verification interval')
+      primary_only_fields.each do |field|
+        expect(page).to have_field(field)
+      end
+
+      secondary_only_fields.each do |field|
+        expect(page).to have_field(field, visible: false)
+      end
 
       uncheck 'This is a primary node'
 
-      expect(page).not_to have_field('Re-verification interval')
+      primary_only_fields.each do |field|
+        expect(page).to have_field(field, visible: false)
+      end
+
+      secondary_only_fields.each do |field|
+        expect(page).to have_field(field)
+      end
     end
 
     it 'returns an error message when a duplicate primary is added' do

ee/spec/fixtures/api/schemas/public_api/v4/geo_node.json

@@ -12,6 +12,7 @@
     "verification_max_capacity",
     "container_repositories_max_capacity",
     "clone_protocol",
+    "web_edit_url",
     "_links"
   ],
   "properties" : {
@@ -26,6 +27,7 @@
     "repos_max_capacity": { "type": "integer" },
     "verification_max_capacity": { "type": "integer" },
     "container_repositories_max_capacity": { "type": "integer" },
+    "sync_object_storage" : { "type":  "boolean" },
     "clone_protocol": { "type": "string" },
     "web_edit_url": { "type": "string" },
     "web_geo_projects_url" : { "type": ["string", "null"] },

ee/spec/javascripts/geo_nodes/mock_data.js

@@ -15,12 +15,13 @@ export const mockNodes = [
     files_max_capacity: 10,
     repos_max_capacity: 25,
     container_repositories_max_capacity: 10,
+    verification_max_capacity: 100,
     clone_protocol: 'http',
+    web_edit_url: 'http://127.0.0.1:3001/admin/geo/nodes/1/edit',
     _links: {
       self: 'http://127.0.0.1:3001/api/v4/geo_nodes/1',
       repair: 'http://127.0.0.1:3001/api/v4/geo_nodes/1/repair',
       status: 'http://127.0.0.1:3001/api/v4/geo_nodes/1/status',
-      web_edit: 'http://127.0.0.1:3001/admin/geo/nodes/1/edit',
     },
   },
   {
@@ -32,12 +33,14 @@ export const mockNodes = [
     files_max_capacity: 10,
     repos_max_capacity: 25,
     container_repositories_max_capacity: 10,
+    verification_max_capacity: 100,
+    sync_object_storage: true,
     clone_protocol: 'http',
+    web_edit_url: 'http://127.0.0.1:3001/admin/geo/nodes/1/edit',
     _links: {
       self: 'http://127.0.0.1:3001/api/v4/geo_nodes/2',
       repair: 'http://127.0.0.1:3001/api/v4/geo_nodes/2/repair',
       status: 'http://127.0.0.1:3001/api/v4/geo_nodes/2/status',
-      web_edit: 'http://127.0.0.1:3001/admin/geo/nodes/2/edit',
     },
   },
 ];

ee/spec/models/geo_node_spec.rb

@@ -118,6 +118,7 @@ describe GeoNode, :geo, type: :model do
       :repos_max_capacity                  | 25
       :files_max_capacity                  | 10
       :container_repositories_max_capacity | 10
+      :sync_object_storage                 | false
     end
 
     with_them do

locale/gitlab.pot

@@ -6814,6 +6814,9 @@ msgstr ""
 msgid "Geo|All projects are being scheduled for re-verify"
 msgstr ""
 
+msgid "Geo|Allow this secondary node to replicate content on Object Storage"
+msgstr ""
+
 msgid "Geo|Batch operations"
 msgstr ""
 
@@ -6853,6 +6856,9 @@ msgstr ""
 msgid "Geo|Groups to synchronize"
 msgstr ""
 
+msgid "Geo|If enabled, and if object storage is enabled, GitLab will handle Object Storage replication using Geo"
+msgstr ""
+
 msgid "Geo|In sync"
 msgstr ""
 
@@ -10112,6 +10118,9 @@ msgstr ""
 msgid "OK"
 msgstr ""
 
+msgid "Object Storage replication"
+msgstr ""
+
 msgid "Object does not exist on the server or you don't have permissions to access it"
 msgstr ""