Skip to content

G
gitlab-ce
Commit bd286c4e authored by Toon Claes's avatar Toon Claes
Browse files
Options

Also remove refresh_wikis 

After removing the `refresh_projects` Geo API endpoint, also remove
the `refresh_wikis` endpoint, because refreshing wikis is also done by
the Geo Log Cursor.
parent 6f6473f7
Hide whitespace changes
Inline Side-by-side
Showing with 14 additions and 485 deletions
app/controllers/projects/wikis_controller.rb
View file @ bd286c4e
......@@ -48,9 +48,6 @@ class Projects::WikisController < Projects::ApplicationController
@page = WikiPages::UpdateService.new(@project, current_user, wiki_params).execute(@page)
if @page.valid?
# Triggers repository update on secondary nodes when Geo is enabled
Gitlab::Geo.notify_wiki_update(@project) if Gitlab::Geo.primary?
redirect_to(
project_wiki_path(@project, @page),
notice: 'Wiki was successfully updated.'
......@@ -67,8 +64,6 @@ class Projects::WikisController < Projects::ApplicationController
@page = WikiPages::CreateService.new(@project, current_user, wiki_params).execute
if @page.persisted?
# Triggers repository update on secondary nodes when Geo is enabled
Gitlab::Geo.notify_wiki_update(@project) if Gitlab::Geo.primary?
redirect_to(
project_wiki_path(@project, @page),
notice: 'Wiki was successfully updated.'
......
app/models/geo_node.rb
View file @ bd286c4e
......@@ -63,14 +63,6 @@ class GeoNode < ActiveRecord::Base
self.relative_url_root = new_uri.path != '/' ? new_uri.path : ''
end
def notify_projects_url
geo_api_url('refresh_projects')
end
def notify_wikis_url
geo_api_url('refresh_wikis')
end
def geo_transfers_url(file_type, file_id)
geo_api_url("transfers/#{file_type}/#{file_id}")
end
......
app/services/geo/enqueue_wiki_update_service.rb deleted 100644 → 0
View file @ 6f6473f7
module Geo
class EnqueueWikiUpdateService
attr_reader :project
def initialize(project)
@queue = Gitlab::Geo::UpdateQueue.new('updated_wikis')
@project = project
end
def execute
@queue.store({ id: @project.id, clone_url: @project.wiki.url_to_repo })
end
end
end
app/services/geo/notify_nodes_service.rb deleted 100644 → 0
View file @ 6f6473f7
module Geo
class NotifyNodesService < BaseNotify
def initialize
@wiki_queue = Gitlab::Geo::UpdateQueue.new('updated_wikis')
end
def execute
process(@wiki_queue, :notify_wikis_url)
end
private
def process(queue, notify_url_method)
return if queue.empty?
projects = queue.fetch_batched_data
content = { projects: projects }.to_json
::Gitlab::Geo.secondary_nodes.each do |node|
next unless node.enabled?
notify_url = node.__send__(notify_url_method.to_sym) # rubocop:disable GitlabSecurity/PublicSend
success, details = notify(notify_url, content)
unless success
Gitlab::Geo::Logger.error(
class: self.class.name,
message: "GitLab failed to notify",
error: details,
node_url: node.url,
notify_url: notify_url)
queue.store_batched_data(projects)
end
end
end
end
end
app/services/geo/schedule_wiki_repo_update_service.rb deleted 100644 → 0
View file @ 6f6473f7
module Geo
class ScheduleWikiRepoUpdateService
attr_reader :projects
def initialize(projects)
@projects = projects
end
def execute
@projects.each do |project|
next unless Gitlab::Geo.current_node&.projects_include?(project['id'].to_i)
GeoWikiRepositoryUpdateWorker.perform_async(project['id'], project['clone_url'])
end
end
end
end
app/workers/geo_bulk_notify_worker.rb deleted 100644 → 0
View file @ 6f6473f7
class GeoBulkNotifyWorker
include Sidekiq::Worker
include CronjobQueue
def perform
Geo::NotifyNodesService.new.execute
end
end
app/workers/geo_repository_move_worker.rb 0 → 100644
View file @ bd286c4e
class GeoRepositoryMoveWorker
include Sidekiq::Worker
include GeoQueue
def perform(id, name, old_path_with_namespace, new_path_with_namespace)
Geo::MoveRepositoryService.new(id, name, old_path_with_namespace, new_path_with_namespace).execute
end
end
app/workers/geo_wiki_repository_update_worker.rb deleted 100644 → 0
View file @ 6f6473f7
class GeoWikiRepositoryUpdateWorker
include Sidekiq::Worker
include Gitlab::ShellAdapter
include GeoQueue
attr_accessor :project
def perform(project_id, clone_url)
@project = Project.find(project_id)
fetch_repository(clone_url)
end
private
def fetch_repository(remote_url)
# Second .wiki call returns a Gollum::Wiki, and it will always create the physical repository when not found
if @project.wiki.wiki.exist?
@project.wiki.repository.fetch_geo_mirror(remote_url)
end
end
end
config/gitlab.yml.example
View file @ bd286c4e
......@@ -219,11 +219,6 @@ production: &base
ldap_sync_worker:
cron: "30 1 * * *"
# GitLab Geo nodes notification worker
# NOTE: This will only take effect if Geo is enabled
geo_bulk_notify_worker:
cron: "*/10 * * * * *"
# GitLab Geo repository sync worker
# NOTE: This will only take effect if Geo is enabled
geo_repository_sync_worker:
......
config/initializers/1_settings.rb
View file @ bd286c4e
......@@ -425,9 +425,6 @@ Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker'
Settings.cron_jobs['ldap_group_sync_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['ldap_group_sync_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['ldap_group_sync_worker']['job_class'] = 'LdapAllGroupsSyncWorker'
Settings.cron_jobs['geo_bulk_notify_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['geo_bulk_notify_worker']['cron'] ||= '*/10 * * * * *'
Settings.cron_jobs['geo_bulk_notify_worker']['job_class'] ||= 'GeoBulkNotifyWorker'
Settings.cron_jobs['geo_repository_sync_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['geo_repository_sync_worker']['cron'] ||= '*/5 * * * *'
Settings.cron_jobs['geo_repository_sync_worker']['job_class'] ||= 'Geo::RepositorySyncWorker'
......
ee/app/services/ee/wiki_pages/base_service.rb
View file @ bd286c4e
......@@ -18,9 +18,6 @@ module EE
if ::Gitlab::Geo.primary?
# Create wiki repository updated event on Geo event log
::Geo::RepositoryUpdatedEventStore.new(project, source: Geo::RepositoryUpdatedEvent::WIKI).create
# Triggers repository update on secondary nodes
::Gitlab::Geo.notify_wiki_update(project)
end
end
end
......
ee/app/workers/ee/post_receive.rb
View file @ bd286c4e
......@@ -24,9 +24,6 @@ module EE
if ::Gitlab::Geo.enabled?
# Create wiki repository updated event on Geo event log
::Geo::RepositoryUpdatedEventStore.new(post_received.project, source: Geo::RepositoryUpdatedEvent::WIKI).create
# Triggers repository update on secondary nodes
::Gitlab::Geo.notify_wiki_update(post_received.project)
end
end
......
lib/api/geo.rb
View file @ bd286c4e
......@@ -38,18 +38,6 @@ module API
present GeoNodeStatus.new(id: Gitlab::Geo.current_node.id), with: Entities::GeoNodeStatus
end
# Enqueue a batch of IDs of wiki's projects to have their
# wiki repositories updated
#
# Example request:
# POST /geo/refresh_wikis
post 'refresh_wikis' do
authenticated_as_admin!
require_node_to_be_enabled!
required_attributes! [:projects]
::Geo::ScheduleWikiRepoUpdateService.new(params[:projects]).execute
end
end
helpers do
......
lib/gitlab/geo.rb
View file @ bd286c4e
......@@ -11,7 +11,6 @@ module Gitlab
geo_oauth_application
).freeze
PRIMARY_JOBS = %i(bulk_notify_job).freeze
SECONDARY_JOBS = %i(repository_sync_job file_download_job).freeze
def self.current_node
......@@ -66,14 +65,6 @@ module Gitlab
GeoNode.where(host: host, port: port).exists?
end
def self.notify_wiki_update(project)
::Geo::EnqueueWikiUpdateService.new(project).execute
end
def self.bulk_notify_job
Sidekiq::Cron::Job.find('geo_bulk_notify_worker')
end
def self.repository_sync_job
Sidekiq::Cron::Job.find('geo_repository_sync_worker')
end
......@@ -93,7 +84,6 @@ module Gitlab
end
def self.disable_all_geo_jobs!
PRIMARY_JOBS.each { |job| self.__send__(job).try(:disable!) } # rubocop:disable GitlabSecurity/PublicSend
SECONDARY_JOBS.each { |job| self.__send__(job).try(:disable!) } # rubocop:disable GitlabSecurity/PublicSend
end
......
lib/gitlab/geo/update_queue.rb deleted 100644 → 0
View file @ 6f6473f7
module Gitlab
module Geo
class UpdateQueue
BATCH_SIZE = 250
NAMESPACE = 'geo:gitlab'.freeze
def initialize(queue)
@queue = queue
end
def store(data)
redis.rpush(@queue, data.to_json)
expire_queue_size!
end
def first
data = fetch(0, 0)
data.first unless data.empty?
end
def last
data = fetch(-1, -1)
data.first unless data.empty?
end
def fetch_batched_data
projects = []
bsize = batch_size
redis.multi do
projects = redis.lrange(@queue, 0, bsize - 1)
redis.ltrim(@queue, bsize, -1)
end
expire_queue_size!
deserialize(projects.value)
end
def store_batched_data(projects)
redis.pipelined do
projects.reverse_each do |project|
# enqueue again to the head of the queue
redis.lpush(@queue, project.to_json)
end
end
expire_queue_size!
end
def batch_size
queue_size > BATCH_SIZE ? BATCH_SIZE : queue_size
end
def queue_size
@queue_size ||= fetch_queue_size
end
def empty?
queue_size == 0
end
def empty!
redis.del(@queue)
end
protected
def fetch(start, stop)
deserialize(redis.lrange(@queue, start, stop))
end
def fetch_queue_size
redis.llen(@queue)
end
def expire_queue_size!
@queue_size = nil
end
def deserialize(data)
data.map! { |item| JSON.parse(item) } unless data.empty?
data
end
def redis
self.class.redis
end
def self.redis_connection
::Redis::Namespace.new(NAMESPACE, redis: ::Redis.new(url: Gitlab::Redis::SharedState.url))
end
def self.redis
@redis ||= redis_connection
end
end
end
end
lib/gitlab/middleware/readonly_geo.rb
View file @ bd286c4e
......@@ -7,7 +7,7 @@ module Gitlab
def initialize(app)
@app = app
@whitelisted = internal_routes + geo_routes
@whitelisted = internal_routes
end
def call(env)
......@@ -36,11 +36,6 @@ module Gitlab
API_VERSIONS.flat_map { |version| "api/v#{version}/internal" }
end
def geo_routes
geo_routes = %w(refresh_wikis)
API_VERSIONS.flat_map { |version| geo_routes.map { |route| "api/v#{version}/geo/#{route}" } }
end
def disallowed_request?
DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes
end
......
spec/ee/spec/services/ee/wiki_pages/create_service_spec.rb
View file @ bd286c4e
......@@ -30,12 +30,6 @@ describe WikiPages::CreateService do
service.execute
end
it 'triggers wiki update on secondary nodes' do
expect(Gitlab::Geo).to receive(:notify_wiki_update).with(instance_of(Project))
service.execute
end
end
end
end
spec/ee/spec/services/ee/wiki_pages/destroy_service_spec.rb
View file @ bd286c4e
......@@ -23,12 +23,6 @@ describe WikiPages::DestroyService do
service.execute(page)
end
it 'triggers wiki update on secondary nodes' do
expect(Gitlab::Geo).to receive(:notify_wiki_update).with(instance_of(Project))
service.execute(page)
end
end
end
end
spec/ee/spec/services/ee/wiki_pages/update_service_spec.rb
View file @ bd286c4e
......@@ -31,12 +31,6 @@ describe WikiPages::UpdateService do
service.execute(page)
end
it 'triggers wiki update on secondary nodes' do
expect(Gitlab::Geo).to receive(:notify_wiki_update).with(instance_of(Project))
service.execute(page)
end
end
end
end
spec/lib/gitlab/geo/update_queue_spec.rb deleted 100644 → 0
View file @ 6f6473f7
require 'spec_helper'
describe Gitlab::Geo::UpdateQueue do
subject { described_class.new('test_queue') }
let(:dummy_data) { { 'id' => 1, 'clone_url' => 'git@localhost:repo/path.git' } }
let(:dummy_data2) { { 'id' => 99, 'clone_url' => 'git@localhost:other_repo/path.git' } }
let(:multiple_dummy_data) { [dummy_data, dummy_data2] * 10 }
before do
subject.empty!
end
describe '#store' do
before do
subject.store(dummy_data)
end
it 'stores data to the queue' do
expect(subject).not_to be_empty
end
it 'stored data is equal to original' do
expect(subject.first).to eq(dummy_data)
end
end
context 'when queue has elements' do
before do
subject.store(dummy_data)
subject.store(dummy_data2)
end
describe '#first' do
it { expect(subject.first).to eq(dummy_data) }
end
describe '#last' do
it { expect(subject.last).to eq(dummy_data2) }
end
end
describe '#fetch_batched_data' do
before do
subject.store_batched_data(multiple_dummy_data)
end
it 'returns same stored data' do
expect(subject.fetch_batched_data).to eq(multiple_dummy_data)
end
end
describe '#store_batched_data' do
let(:ordered_data) { [{ 'a' => 1 }, { 'a' => 2 }, { 'a' => 3 }, { 'a' => 4 }, { 'a' => 5 }] }
it 'stores multiple items to the queue' do
expect { subject.store_batched_data(multiple_dummy_data) }.to change { subject.batch_size }.by(multiple_dummy_data.size)
end
it 'returns data in equal order to original' do
subject.store_batched_data(ordered_data)
expect(subject.first).to eq(ordered_data.first)
expect(subject.last).to eq(ordered_data.last)
end
end
describe '#batch_size' do
before do
allow(subject).to receive(:queue_size) { queue_size }
end
context 'when queue size is smaller than BATCH_SIZE' do
let(:queue_size) { described_class::BATCH_SIZE - 20 }
it 'equals to the queue size' do
expect(subject.batch_size).to eq(queue_size)
end
end
context 'when queue size is bigger than BATCH_SIZE' do
let(:queue_size) { described_class::BATCH_SIZE + 20 }
it 'equals to the BATCH_SIZE' do
expect(subject.batch_size).to eq(described_class::BATCH_SIZE)
end
end
end
describe '#queue_size' do
it 'returns the ammount of items in queue' do
expect { subject.store(dummy_data) }.to change { subject.queue_size }.by(1)
end
end
describe '#empty?' do
it 'returns true when empty' do
is_expected.to be_empty
end
it 'returns false when there are enqueue data' do
subject.store(dummy_data)
is_expected.not_to be_empty
end
end
end
spec/lib/gitlab/geo_spec.rb
View file @ bd286c4e
......@@ -176,7 +176,7 @@ describe Gitlab::Geo do
end
describe '.configure_cron_jobs!' do
JOBS = %w(ldap_test geo_bulk_notify_worker geo_repository_sync_worker geo_file_download_dispatch_worker).freeze
JOBS = %w(ldap_test geo_repository_sync_worker geo_file_download_dispatch_worker).freeze
def init_cron_job(job_name, class_name)
job = Sidekiq::Cron::Job.new(
......@@ -202,7 +202,6 @@ describe Gitlab::Geo do
described_class.configure_cron_jobs!
expect(described_class.bulk_notify_job).to be_enabled
expect(described_class.repository_sync_job).not_to be_enabled
expect(described_class.file_download_job).not_to be_enabled
expect(Sidekiq::Cron::Job.find('ldap_test')).to be_enabled
......@@ -215,7 +214,6 @@ describe Gitlab::Geo do
described_class.configure_cron_jobs!
expect(Sidekiq::Cron::Job.find('ldap_test')).not_to be_enabled
expect(described_class.bulk_notify_job).not_to be_enabled
expect(described_class.repository_sync_job).to be_enabled
expect(described_class.file_download_job).to be_enabled
end
......@@ -226,7 +224,6 @@ describe Gitlab::Geo do
described_class.configure_cron_jobs!
expect(described_class.bulk_notify_job).not_to be_enabled
expect(described_class.repository_sync_job).not_to be_enabled
expect(described_class.file_download_job).not_to be_enabled
expect(Sidekiq::Cron::Job.find('ldap_test')).to be_enabled
......
spec/lib/gitlab/middleware/readonly_geo_spec.rb
View file @ bd286c4e
......@@ -65,12 +65,12 @@ describe Gitlab::Middleware::ReadonlyGeo do
expect(subject).to disallow_request
end
it 'expects a POST Geo request to be allowed after a disallowed request' do
it 'expects a internal POST request to be allowed after a disallowed request' do
response = request.post('/test_request')
expect(response).to be_a_redirect
response = request.post("/api/#{API::API.version}/geo/refresh_wikis")
response = request.post("/api/#{API::API.version}/internal")
expect(response).not_to be_a_redirect
end
......@@ -97,8 +97,8 @@ describe Gitlab::Middleware::ReadonlyGeo do
expect(subject).not_to disallow_request
end
it 'expects a POST Geo request to be allowed' do
response = request.post("/api/#{API::API.version}/geo/refresh_wikis")
it 'expects a GET status request to be allowed' do
response = request.get("/api/#{API::API.version}/geo/status")
expect(response).not_to be_a_redirect
expect(subject).not_to disallow_request
......
spec/models/geo_node_spec.rb
View file @ bd286c4e
......@@ -206,22 +206,6 @@ describe GeoNode, type: :model do
end
end
describe '#notify_projects_url' do
let(:refresh_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/refresh_projects" }
it 'returns api url based on node uri' do
expect(new_node.notify_projects_url).to eq(refresh_url)
end
end
describe '#notify_wikis_url' do
let(:refresh_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/refresh_wikis" }
it 'returns api url based on node uri' do
expect(new_node.notify_wikis_url).to eq(refresh_url)
end
end
describe '#geo_transfers_url' do
let(:transfers_url) { "https://localhost:3000/gitlab/api/#{api_version}/geo/transfers/lfs/1" }
......
spec/requests/api/geo_spec.rb
View file @ bd286c4e
......@@ -15,16 +15,6 @@ describe API::Geo do
allow(Gitlab::Geo).to receive(:current_node) { secondary_node }
end
describe 'POST /geo/refresh_wikis disabled node' do
it 'responds with forbidden' do
secondary_node.enabled = false
post api('/geo/refresh_wikis', admin), nil
expect(response).to have_http_status(403)
end
end
describe 'GET /geo/transfers/attachment/1' do
let!(:secondary_node) { create(:geo_node) }
let(:note) { create(:note, :with_attachment) }
......
spec/services/geo/enqueue_wiki_update_service_spec.rb deleted 100644 → 0
View file @ 6f6473f7
require 'spec_helper'
describe Geo::EnqueueWikiUpdateService do
subject { described_class.new(project) }
let(:project) { double(:project) }
let(:fake_url) { 'git@localhost:repo/path.git' }
let(:fake_id) { 999 }
let(:queue) { subject.instance_variable_get(:@queue) }
before do
queue.empty!
expect(project).to receive_message_chain(:wiki, :url_to_repo) { fake_url }
expect(project).to receive(:id) { fake_id }
end
describe '#execute' do
let(:stored_data) { queue.first }
before do
subject.execute
end
it 'persists id and clone_url to redis queue' do
expect(stored_data).to have_key('id')
expect(stored_data).to have_key('clone_url')
end
it 'persisted id is equal to original' do
expect(stored_data['id']).to eq(fake_id)
end
it 'persisted clone_url is equal to original' do
expect(stored_data['clone_url']).to eq(fake_url)
end
end
end
spec/services/geo/schedule_wiki_repo_update_service_spec.rb deleted 100644 → 0
View file @ 6f6473f7
require 'spec_helper'
describe Geo::ScheduleWikiRepoUpdateService do
describe '#execute' do
let(:group) { create(:group) }
let(:project_1) { create(:project) }
let(:project_2) { create(:project, group: group) }
let(:projects) do
[
{ 'id' => project_1.id, 'clone_url' => 'git@example.com:mike/diaspora.git' },
{ 'id' => project_2.id, 'clone_url' => 'git@example.com:asd/vim.git' }
]
end
subject { described_class.new(projects) }
it "enqueues a batch of IDs of wiki's projects to have their wiki repositories updated" do
create(:geo_node, :current)
expect(GeoWikiRepositoryUpdateWorker).to receive(:perform_async)
.once.with(project_1.id, 'git@example.com:mike/diaspora.git').and_return(spy)
expect(GeoWikiRepositoryUpdateWorker).to receive(:perform_async)
.once.with(project_2.id, 'git@example.com:asd/vim.git').and_return(spy)
subject.execute
end
context 'when node has namespace restrictions' do
it "does not enqueue IDs of wiki's projects that do not belong to selected namespaces to replicate" do
create(:geo_node, :current, namespaces: [group])
expect(GeoWikiRepositoryUpdateWorker).not_to receive(:perform_async)
.with(project_1.id, 'git@example.com:mike/diaspora.git')
expect(GeoWikiRepositoryUpdateWorker).to receive(:perform_async)
.once.with(project_2.id, 'git@example.com:asd/vim.git').and_return(spy)
subject.execute
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7