Also remove refresh_wikis

After removing the `refresh_projects` Geo API endpoint, also remove
the `refresh_wikis` endpoint, because refreshing wikis is also done by
the Geo Log Cursor.

app/controllers/projects/wikis_controller.rb

@@ -48,9 +48,6 @@ class Projects::WikisController < Projects::ApplicationController
     @page = WikiPages::UpdateService.new(@project, current_user, wiki_params).execute(@page)
 
     if @page.valid?
-      # Triggers repository update on secondary nodes when Geo is enabled
-      Gitlab::Geo.notify_wiki_update(@project) if Gitlab::Geo.primary?
-
       redirect_to(
         project_wiki_path(@project, @page),
         notice: 'Wiki was successfully updated.'
@@ -67,8 +64,6 @@ class Projects::WikisController < Projects::ApplicationController
     @page = WikiPages::CreateService.new(@project, current_user, wiki_params).execute
 
     if @page.persisted?
-      # Triggers repository update on secondary nodes when Geo is enabled
-      Gitlab::Geo.notify_wiki_update(@project) if Gitlab::Geo.primary?
       redirect_to(
         project_wiki_path(@project, @page),
         notice: 'Wiki was successfully updated.'

app/models/geo_node.rb

@@ -63,14 +63,6 @@ class GeoNode < ActiveRecord::Base
     self.relative_url_root = new_uri.path != '/' ? new_uri.path : ''
   end
 
-  def notify_projects_url
-    geo_api_url('refresh_projects')
-  end
-
-  def notify_wikis_url
-    geo_api_url('refresh_wikis')
-  end
-
   def geo_transfers_url(file_type, file_id)
     geo_api_url("transfers/#{file_type}/#{file_id}")
   end

app/services/geo/enqueue_wiki_update_service.rb

-module Geo
-  class EnqueueWikiUpdateService
-    attr_reader :project
-
-    def initialize(project)
-      @queue = Gitlab::Geo::UpdateQueue.new('updated_wikis')
-      @project = project
-    end
-
-    def execute
-      @queue.store({ id: @project.id, clone_url: @project.wiki.url_to_repo })
-    end
-  end
-end

app/services/geo/notify_nodes_service.rb

-module Geo
-  class NotifyNodesService < BaseNotify
-    def initialize
-      @wiki_queue = Gitlab::Geo::UpdateQueue.new('updated_wikis')
-    end
-
-    def execute
-      process(@wiki_queue, :notify_wikis_url)
-    end
-
-    private
-
-    def process(queue, notify_url_method)
-      return if queue.empty?
-
-      projects = queue.fetch_batched_data
-      content = { projects: projects }.to_json
-
-      ::Gitlab::Geo.secondary_nodes.each do |node|
-        next unless node.enabled?
-
-        notify_url = node.__send__(notify_url_method.to_sym) # rubocop:disable GitlabSecurity/PublicSend
-        success, details = notify(notify_url, content)
-
-        unless success
-          Gitlab::Geo::Logger.error(
-            class: self.class.name,
-            message: "GitLab failed to notify",
-            error: details,
-            node_url: node.url,
-            notify_url: notify_url)
-          queue.store_batched_data(projects)
-        end
-      end
-    end
-  end
-end

app/services/geo/schedule_wiki_repo_update_service.rb

-module Geo
-  class ScheduleWikiRepoUpdateService
-    attr_reader :projects
-
-    def initialize(projects)
-      @projects = projects
-    end
-
-    def execute
-      @projects.each do |project|
-        next unless Gitlab::Geo.current_node&.projects_include?(project['id'].to_i)
-
-        GeoWikiRepositoryUpdateWorker.perform_async(project['id'], project['clone_url'])
-      end
-    end
-  end
-end

app/workers/geo_bulk_notify_worker.rb

-class GeoBulkNotifyWorker
-  include Sidekiq::Worker
-  include CronjobQueue
-
-  def perform
-    Geo::NotifyNodesService.new.execute
-  end
-end

app/workers/geo_repository_move_worker.rb

+class GeoRepositoryMoveWorker
+  include Sidekiq::Worker
+  include GeoQueue
+
+  def perform(id, name, old_path_with_namespace, new_path_with_namespace)
+    Geo::MoveRepositoryService.new(id, name, old_path_with_namespace, new_path_with_namespace).execute
+  end
+end

app/workers/geo_wiki_repository_update_worker.rb

-class GeoWikiRepositoryUpdateWorker
-  include Sidekiq::Worker
-  include Gitlab::ShellAdapter
-  include GeoQueue
-
-  attr_accessor :project
-
-  def perform(project_id, clone_url)
-    @project = Project.find(project_id)
-
-    fetch_repository(clone_url)
-  end
-
-  private
-
-  def fetch_repository(remote_url)
-    # Second .wiki call returns a Gollum::Wiki, and it will always create the physical repository when not found
-    if @project.wiki.wiki.exist?
-      @project.wiki.repository.fetch_geo_mirror(remote_url)
-    end
-  end
-end

config/gitlab.yml.example

@@ -219,11 +219,6 @@ production: &base
     ldap_sync_worker:
       cron: "30 1 * * *"
 
-    # GitLab Geo nodes notification worker
-    # NOTE: This will only take effect if Geo is enabled
-    geo_bulk_notify_worker:
-      cron: "*/10 * * * * *"
-
     # GitLab Geo repository sync worker
     # NOTE: This will only take effect if Geo is enabled
     geo_repository_sync_worker:

config/initializers/1_settings.rb

@@ -425,9 +425,6 @@ Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker'
 Settings.cron_jobs['ldap_group_sync_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['ldap_group_sync_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['ldap_group_sync_worker']['job_class'] = 'LdapAllGroupsSyncWorker'
-Settings.cron_jobs['geo_bulk_notify_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['geo_bulk_notify_worker']['cron'] ||= '*/10 * * * * *'
-Settings.cron_jobs['geo_bulk_notify_worker']['job_class'] ||= 'GeoBulkNotifyWorker'
 Settings.cron_jobs['geo_repository_sync_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['geo_repository_sync_worker']['cron'] ||= '*/5 * * * *'
 Settings.cron_jobs['geo_repository_sync_worker']['job_class'] ||= 'Geo::RepositorySyncWorker'

ee/app/services/ee/wiki_pages/base_service.rb

@@ -18,9 +18,6 @@ module EE
         if ::Gitlab::Geo.primary?
           # Create wiki repository updated event on Geo event log
           ::Geo::RepositoryUpdatedEventStore.new(project, source: Geo::RepositoryUpdatedEvent::WIKI).create
-
-          # Triggers repository update on secondary nodes
-          ::Gitlab::Geo.notify_wiki_update(project)
         end
       end
     end

ee/app/workers/ee/post_receive.rb

@@ -24,9 +24,6 @@ module EE
       if ::Gitlab::Geo.enabled?
         # Create wiki repository updated event on Geo event log
         ::Geo::RepositoryUpdatedEventStore.new(post_received.project, source: Geo::RepositoryUpdatedEvent::WIKI).create
-
-        # Triggers repository update on secondary nodes
-        ::Gitlab::Geo.notify_wiki_update(post_received.project)
       end
     end
 

lib/api/geo.rb

@@ -38,18 +38,6 @@ module API
 
         present GeoNodeStatus.new(id: Gitlab::Geo.current_node.id), with: Entities::GeoNodeStatus
       end
-
-      # Enqueue a batch of IDs of wiki's projects to have their
-      # wiki repositories updated
-      #
-      # Example request:
-      #   POST /geo/refresh_wikis
-      post 'refresh_wikis' do
-        authenticated_as_admin!
-        require_node_to_be_enabled!
-        required_attributes! [:projects]
-        ::Geo::ScheduleWikiRepoUpdateService.new(params[:projects]).execute
-      end
     end
 
     helpers do

lib/gitlab/geo.rb

@@ -11,7 +11,6 @@ module Gitlab
       geo_oauth_application
     ).freeze
 
-    PRIMARY_JOBS = %i(bulk_notify_job).freeze
     SECONDARY_JOBS = %i(repository_sync_job file_download_job).freeze
 
     def self.current_node
@@ -66,14 +65,6 @@ module Gitlab
       GeoNode.where(host: host, port: port).exists?
     end
 
-    def self.notify_wiki_update(project)
-      ::Geo::EnqueueWikiUpdateService.new(project).execute
-    end
-
-    def self.bulk_notify_job
-      Sidekiq::Cron::Job.find('geo_bulk_notify_worker')
-    end
-
     def self.repository_sync_job
       Sidekiq::Cron::Job.find('geo_repository_sync_worker')
     end
@@ -93,7 +84,6 @@ module Gitlab
     end
 
     def self.disable_all_geo_jobs!
-      PRIMARY_JOBS.each { |job| self.__send__(job).try(:disable!) } # rubocop:disable GitlabSecurity/PublicSend
       SECONDARY_JOBS.each { |job| self.__send__(job).try(:disable!) } # rubocop:disable GitlabSecurity/PublicSend
     end
 

lib/gitlab/geo/update_queue.rb

-module Gitlab
-  module Geo
-    class UpdateQueue
-      BATCH_SIZE = 250
-      NAMESPACE = 'geo:gitlab'.freeze
-
-      def initialize(queue)
-        @queue = queue
-      end
-
-      def store(data)
-        redis.rpush(@queue, data.to_json)
-        expire_queue_size!
-      end
-
-      def first
-        data = fetch(0, 0)
-        data.first unless data.empty?
-      end
-
-      def last
-        data = fetch(-1, -1)
-        data.first unless data.empty?
-      end
-
-      def fetch_batched_data
-        projects = []
-        bsize = batch_size
-
-        redis.multi do
-          projects = redis.lrange(@queue, 0, bsize - 1)
-          redis.ltrim(@queue, bsize, -1)
-        end
-
-        expire_queue_size!
-        deserialize(projects.value)
-      end
-
-      def store_batched_data(projects)
-        redis.pipelined do
-          projects.reverse_each do |project|
-            # enqueue again to the head of the queue
-            redis.lpush(@queue, project.to_json)
-          end
-        end
-        expire_queue_size!
-      end
-
-      def batch_size
-        queue_size > BATCH_SIZE ? BATCH_SIZE : queue_size
-      end
-
-      def queue_size
-        @queue_size ||= fetch_queue_size
-      end
-
-      def empty?
-        queue_size == 0
-      end
-
-      def empty!
-        redis.del(@queue)
-      end
-
-      protected
-
-      def fetch(start, stop)
-        deserialize(redis.lrange(@queue, start, stop))
-      end
-
-      def fetch_queue_size
-        redis.llen(@queue)
-      end
-
-      def expire_queue_size!
-        @queue_size = nil
-      end
-
-      def deserialize(data)
-        data.map! { |item| JSON.parse(item) } unless data.empty?
-        data
-      end
-
-      def redis
-        self.class.redis
-      end
-
-      def self.redis_connection
-        ::Redis::Namespace.new(NAMESPACE, redis: ::Redis.new(url: Gitlab::Redis::SharedState.url))
-      end
-
-      def self.redis
-        @redis ||= redis_connection
-      end
-    end
-  end
-end

lib/gitlab/middleware/readonly_geo.rb

@@ -7,7 +7,7 @@ module Gitlab
 
       def initialize(app)
         @app = app
-        @whitelisted = internal_routes + geo_routes
+        @whitelisted = internal_routes
       end
 
       def call(env)
@@ -36,11 +36,6 @@ module Gitlab
         API_VERSIONS.flat_map { |version| "api/v#{version}/internal" }
       end
 
-      def geo_routes
-        geo_routes = %w(refresh_wikis)
-        API_VERSIONS.flat_map { |version| geo_routes.map { |route| "api/v#{version}/geo/#{route}" } }
-      end
-
       def disallowed_request?
         DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes
       end

spec/ee/spec/services/ee/wiki_pages/create_service_spec.rb

@@ -30,12 +30,6 @@ describe WikiPages::CreateService do
 
         service.execute
       end
-
-      it 'triggers wiki update on secondary nodes' do
-        expect(Gitlab::Geo).to receive(:notify_wiki_update).with(instance_of(Project))
-
-        service.execute
-      end
     end
   end
 end

spec/ee/spec/services/ee/wiki_pages/destroy_service_spec.rb

@@ -23,12 +23,6 @@ describe WikiPages::DestroyService do
 
         service.execute(page)
       end
-
-      it 'triggers wiki update on secondary nodes' do
-        expect(Gitlab::Geo).to receive(:notify_wiki_update).with(instance_of(Project))
-
-        service.execute(page)
-      end
     end
   end
 end

spec/ee/spec/services/ee/wiki_pages/update_service_spec.rb

@@ -31,12 +31,6 @@ describe WikiPages::UpdateService do
 
         service.execute(page)
       end
-
-      it 'triggers wiki update on secondary nodes' do
-        expect(Gitlab::Geo).to receive(:notify_wiki_update).with(instance_of(Project))
-
-        service.execute(page)
-      end
     end
   end
 end

spec/lib/gitlab/geo/update_queue_spec.rb

-require 'spec_helper'
-
-describe Gitlab::Geo::UpdateQueue do
-  subject { described_class.new('test_queue') }
-  let(:dummy_data) { { 'id' => 1, 'clone_url' => 'git@localhost:repo/path.git' } }
-  let(:dummy_data2) { { 'id' => 99, 'clone_url' => 'git@localhost:other_repo/path.git' } }
-  let(:multiple_dummy_data) { [dummy_data, dummy_data2] * 10 }
-
-  before do
-    subject.empty!
-  end
-
-  describe '#store' do
-    before do
-      subject.store(dummy_data)
-    end
-
-    it 'stores data to the queue' do
-      expect(subject).not_to be_empty
-    end
-
-    it 'stored data is equal to original' do
-      expect(subject.first).to eq(dummy_data)
-    end
-  end
-
-  context 'when queue has elements' do
-    before do
-      subject.store(dummy_data)
-      subject.store(dummy_data2)
-    end
-
-    describe '#first' do
-      it { expect(subject.first).to eq(dummy_data) }
-    end
-
-    describe '#last' do
-      it { expect(subject.last).to eq(dummy_data2) }
-    end
-  end
-
-  describe '#fetch_batched_data' do
-    before do
-      subject.store_batched_data(multiple_dummy_data)
-    end
-
-    it 'returns same stored data' do
-      expect(subject.fetch_batched_data).to eq(multiple_dummy_data)
-    end
-  end
-
-  describe '#store_batched_data' do
-    let(:ordered_data) { [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }, { 'a' => 4 }, { 'a' => 5 }] }
-
-    it 'stores multiple items to the queue' do
-      expect { subject.store_batched_data(multiple_dummy_data) }.to change { subject.batch_size }.by(multiple_dummy_data.size)
-    end
-
-    it 'returns data in equal order to original' do
-      subject.store_batched_data(ordered_data)
-
-      expect(subject.first).to eq(ordered_data.first)
-      expect(subject.last).to eq(ordered_data.last)
-    end
-  end
-
-  describe '#batch_size' do
-    before do
-      allow(subject).to receive(:queue_size) { queue_size }
-    end
-
-    context 'when queue size is smaller than BATCH_SIZE' do
-      let(:queue_size) { described_class::BATCH_SIZE - 20 }
-
-      it 'equals to the queue size' do
-        expect(subject.batch_size).to eq(queue_size)
-      end
-    end
-
-    context 'when queue size is bigger than BATCH_SIZE' do
-      let(:queue_size) { described_class::BATCH_SIZE + 20 }
-
-      it 'equals to the BATCH_SIZE' do
-        expect(subject.batch_size).to eq(described_class::BATCH_SIZE)
-      end
-    end
-  end
-
-  describe '#queue_size' do
-    it 'returns the ammount of items in queue' do
-      expect { subject.store(dummy_data) }.to change { subject.queue_size }.by(1)
-    end
-  end
-
-  describe '#empty?' do
-    it 'returns true when empty' do
-      is_expected.to be_empty
-    end
-
-    it 'returns false when there are enqueue data' do
-      subject.store(dummy_data)
-      is_expected.not_to be_empty
-    end
-  end
-end

spec/lib/gitlab/geo_spec.rb

@@ -176,7 +176,7 @@ describe Gitlab::Geo do
   end
 
   describe '.configure_cron_jobs!' do
-    JOBS = %w(ldap_test geo_bulk_notify_worker geo_repository_sync_worker geo_file_download_dispatch_worker).freeze
+    JOBS = %w(ldap_test geo_repository_sync_worker geo_file_download_dispatch_worker).freeze
 
     def init_cron_job(job_name, class_name)
       job = Sidekiq::Cron::Job.new(
@@ -202,7 +202,6 @@ describe Gitlab::Geo do
 
       described_class.configure_cron_jobs!
 
-      expect(described_class.bulk_notify_job).to be_enabled
       expect(described_class.repository_sync_job).not_to be_enabled
       expect(described_class.file_download_job).not_to be_enabled
       expect(Sidekiq::Cron::Job.find('ldap_test')).to be_enabled
@@ -215,7 +214,6 @@ describe Gitlab::Geo do
       described_class.configure_cron_jobs!
 
       expect(Sidekiq::Cron::Job.find('ldap_test')).not_to be_enabled
-      expect(described_class.bulk_notify_job).not_to be_enabled
       expect(described_class.repository_sync_job).to be_enabled
       expect(described_class.file_download_job).to be_enabled
     end
@@ -226,7 +224,6 @@ describe Gitlab::Geo do
 
       described_class.configure_cron_jobs!
 
-      expect(described_class.bulk_notify_job).not_to be_enabled
       expect(described_class.repository_sync_job).not_to be_enabled
       expect(described_class.file_download_job).not_to be_enabled
       expect(Sidekiq::Cron::Job.find('ldap_test')).to be_enabled

spec/lib/gitlab/middleware/readonly_geo_spec.rb

@@ -65,12 +65,12 @@ describe Gitlab::Middleware::ReadonlyGeo do
       expect(subject).to disallow_request
     end
 
-    it 'expects a POST Geo request to be allowed after a disallowed request' do
+    it 'expects a internal POST request to be allowed after a disallowed request' do
       response = request.post('/test_request')
 
       expect(response).to be_a_redirect
 
-      response = request.post("/api/#{API::API.version}/geo/refresh_wikis")
+      response = request.post("/api/#{API::API.version}/internal")
 
       expect(response).not_to be_a_redirect
     end
@@ -97,8 +97,8 @@ describe Gitlab::Middleware::ReadonlyGeo do
         expect(subject).not_to disallow_request
       end
 
-      it 'expects a POST Geo request to be allowed' do
-        response = request.post("/api/#{API::API.version}/geo/refresh_wikis")
+      it 'expects a GET status request to be allowed' do
+        response = request.get("/api/#{API::API.version}/geo/status")
 
         expect(response).not_to be_a_redirect
         expect(subject).not_to disallow_request

spec/models/geo_node_spec.rb

@@ -206,22 +206,6 @@ describe GeoNode, type: :model do
     end
   end
 
-  describe '#notify_projects_url' do
-    let(:refresh_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/refresh_projects" }
-
-    it 'returns api url based on node uri' do
-      expect(new_node.notify_projects_url).to eq(refresh_url)
-    end
-  end
-
-  describe '#notify_wikis_url' do
-    let(:refresh_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/refresh_wikis" }
-
-    it 'returns api url based on node uri' do
-      expect(new_node.notify_wikis_url).to eq(refresh_url)
-    end
-  end
-
   describe '#geo_transfers_url' do
     let(:transfers_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/transfers/lfs/1" }
 

spec/requests/api/geo_spec.rb

@@ -15,16 +15,6 @@ describe API::Geo do
     allow(Gitlab::Geo).to receive(:current_node) { secondary_node }
   end
 
-  describe 'POST /geo/refresh_wikis disabled node' do
-    it 'responds with forbidden' do
-      secondary_node.enabled = false
-
-      post api('/geo/refresh_wikis', admin), nil
-
-      expect(response).to have_http_status(403)
-    end
-  end
-
   describe 'GET /geo/transfers/attachment/1' do
     let!(:secondary_node) { create(:geo_node) }
     let(:note) { create(:note, :with_attachment) }

spec/services/geo/enqueue_wiki_update_service_spec.rb

-require 'spec_helper'
-
-describe Geo::EnqueueWikiUpdateService do
-  subject { described_class.new(project) }
-  let(:project) { double(:project) }
-  let(:fake_url) { 'git@localhost:repo/path.git' }
-  let(:fake_id) { 999 }
-  let(:queue) { subject.instance_variable_get(:@queue) }
-
-  before do
-    queue.empty!
-    expect(project).to receive_message_chain(:wiki, :url_to_repo) { fake_url }
-    expect(project).to receive(:id) { fake_id }
-  end
-
-  describe '#execute' do
-    let(:stored_data) { queue.first }
-
-    before do
-      subject.execute
-    end
-
-    it 'persists id and clone_url to redis queue' do
-      expect(stored_data).to have_key('id')
-      expect(stored_data).to have_key('clone_url')
-    end
-
-    it 'persisted id is equal to original' do
-      expect(stored_data['id']).to eq(fake_id)
-    end
-
-    it 'persisted clone_url is equal to original' do
-      expect(stored_data['clone_url']).to eq(fake_url)
-    end
-  end
-end

spec/services/geo/schedule_wiki_repo_update_service_spec.rb

-require 'spec_helper'
-
-describe Geo::ScheduleWikiRepoUpdateService do
-  describe '#execute' do
-    let(:group) { create(:group) }
-    let(:project_1) { create(:project) }
-    let(:project_2) { create(:project, group: group) }
-
-    let(:projects) do
-      [
-        { 'id' => project_1.id, 'clone_url' => 'git@example.com:mike/diaspora.git' },
-        { 'id' => project_2.id, 'clone_url' => 'git@example.com:asd/vim.git' }
-      ]
-    end
-
-    subject { described_class.new(projects) }
-
-    it "enqueues a batch of IDs of wiki's projects to have their wiki repositories updated" do
-      create(:geo_node, :current)
-
-      expect(GeoWikiRepositoryUpdateWorker).to receive(:perform_async)
-        .once.with(project_1.id, 'git@example.com:mike/diaspora.git').and_return(spy)
-      expect(GeoWikiRepositoryUpdateWorker).to receive(:perform_async)
-        .once.with(project_2.id, 'git@example.com:asd/vim.git').and_return(spy)
-
-      subject.execute
-    end
-
-    context 'when node has namespace restrictions' do
-      it "does not enqueue IDs of wiki's projects that do not belong to selected namespaces to replicate" do
-        create(:geo_node, :current, namespaces: [group])
-
-        expect(GeoWikiRepositoryUpdateWorker).not_to receive(:perform_async)
-          .with(project_1.id, 'git@example.com:mike/diaspora.git')
-        expect(GeoWikiRepositoryUpdateWorker).to receive(:perform_async)
-          .once.with(project_2.id, 'git@example.com:asd/vim.git').and_return(spy)
-
-        subject.execute
-      end
-    end
-  end
-end