Commit b72e6cb7 authored by Nick Thomas's avatar Nick Thomas

Merge branch '3508-hashed-storage-sends-a-geo-repository-renamed-event' into 'master'

Does not move projects backed by hashed storage when handling renamed events

Closes #3508

See merge request gitlab-org/gitlab-ee!3066
parents 21816434 201f7f3f
...@@ -22,6 +22,10 @@ module Geo ...@@ -22,6 +22,10 @@ module Geo
class_name: 'Geo::RepositoriesChangedEvent', class_name: 'Geo::RepositoriesChangedEvent',
foreign_key: :repositories_changed_event_id foreign_key: :repositories_changed_event_id
belongs_to :hashed_storage_migrated_event,
class_name: 'Geo::HashedStorageMigratedEvent',
foreign_key: :hashed_storage_migrated_event_id
def self.latest_event def self.latest_event
order(id: :desc).first order(id: :desc).first
end end
...@@ -31,7 +35,8 @@ module Geo ...@@ -31,7 +35,8 @@ module Geo
repository_updated_event || repository_updated_event ||
repository_deleted_event || repository_deleted_event ||
repository_renamed_event || repository_renamed_event ||
repositories_changed_event repositories_changed_event ||
hashed_storage_migrated_event
end end
def project_id def project_id
......
module Geo
class HashedStorageMigratedEvent < ActiveRecord::Base
include Geo::Model
belongs_to :project
validates :project, :repository_storage_name, :repository_storage_path,
:old_disk_path, :new_disk_path, :old_wiki_disk_path,
:new_wiki_disk_path, :new_storage_version, presence: true
end
end
module Geo
class HashedStorageMigratedEventStore < EventStore
self.event_type = :hashed_storage_migrated_event
private
def build_event
Geo::HashedStorageMigratedEvent.new(
project: project,
old_storage_version: old_storage_version,
new_storage_version: project.storage_version,
repository_storage_name: project.repository.storage,
repository_storage_path: project.repository_storage_path,
old_disk_path: old_disk_path,
new_disk_path: project.disk_path,
old_wiki_disk_path: old_wiki_disk_path,
new_wiki_disk_path: project.wiki.disk_path
)
end
def old_storage_version
params.fetch(:old_storage_version)
end
def old_disk_path
params.fetch(:old_disk_path)
end
def old_wiki_disk_path
params.fetch(:old_wiki_disk_path)
end
end
end
module Geo
class HashedStorageMigrationService
attr_reader :project_id, :old_disk_path, :new_disk_path, :old_storage_version
def initialize(project_id, old_disk_path:, new_disk_path:, old_storage_version:)
@project_id = project_id
@old_disk_path = old_disk_path
@new_disk_path = new_disk_path
@old_storage_version = old_storage_version
end
def async_execute
Geo::HashedStorageMigrationWorker.perform_async(
project_id,
old_disk_path,
new_disk_path,
old_storage_version
)
end
def execute
project = Project.find(project_id)
project.expire_caches_before_rename(old_disk_path)
if migrating_from_legacy_storage?(project)
Geo::MoveRepositoryService.new(project, old_disk_path, new_disk_path).execute
end
true
end
private
def migrating_from_legacy_storage?(project)
from_legacy_storage? && project.hashed_storage?(:repository)
end
def from_legacy_storage?
old_storage_version.nil? || old_storage_version.zero?
end
end
end
...@@ -2,34 +2,26 @@ module Geo ...@@ -2,34 +2,26 @@ module Geo
class MoveRepositoryService class MoveRepositoryService
include Gitlab::ShellAdapter include Gitlab::ShellAdapter
attr_reader :id, :name, :old_path_with_namespace, :new_path_with_namespace attr_reader :project, :old_disk_path, :new_disk_path
def initialize(id, name, old_path_with_namespace, new_path_with_namespace) def initialize(project, old_disk_path, new_disk_path)
@id = id @project = project
@name = name @old_disk_path = old_disk_path
@old_path_with_namespace = old_path_with_namespace @new_disk_path = new_disk_path
@new_path_with_namespace = new_path_with_namespace
end
def async_execute
GeoRepositoryMoveWorker.perform_async(id, name, old_path_with_namespace, new_path_with_namespace)
end end
def execute def execute
project = Project.find(id)
project.expire_caches_before_rename(old_path_with_namespace)
# Make sure target directory exists (used when transfering repositories) # Make sure target directory exists (used when transfering repositories)
project.ensure_storage_path_exists project.ensure_storage_path_exists
if gitlab_shell.mv_repository(project.repository_storage_path, if gitlab_shell.mv_repository(project.repository_storage_path,
old_path_with_namespace, new_path_with_namespace) old_disk_path, new_disk_path)
# If repository moved successfully we need to send update instructions to users. # If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository # However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions # So we basically we mute exceptions in next actions
begin begin
gitlab_shell.mv_repository(project.repository_storage_path, gitlab_shell.mv_repository(project.repository_storage_path,
"#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") "#{old_disk_path}.wiki", "#{new_disk_path}.wiki")
rescue rescue
# Returning false does not rollback after_* transaction but gives # Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks # us information about failing some of tasks
...@@ -38,7 +30,7 @@ module Geo ...@@ -38,7 +30,7 @@ module Geo
else else
# if we cannot move namespace directory we should rollback # if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs # db changes in order to prevent out of sync between db and fs
raise Exception.new('repository cannot be renamed') raise StandardError.new('Repository cannot be renamed')
end end
true true
......
module Geo
class RenameRepositoryService
attr_reader :project_id, :old_disk_path, :new_disk_path
def initialize(project_id, old_disk_path, new_disk_path)
@project_id = project_id
@old_disk_path = old_disk_path
@new_disk_path = new_disk_path
end
def async_execute
Geo::RenameRepositoryWorker.perform_async(project_id, old_disk_path, new_disk_path)
end
def execute
project = Project.find(project_id)
project.expire_caches_before_rename(old_disk_path)
return true if project.hashed_storage?(:repository)
Geo::MoveRepositoryService.new(project, old_disk_path, new_disk_path).execute
end
end
end
...@@ -4,7 +4,7 @@ module Projects ...@@ -4,7 +4,7 @@ module Projects
prepend ::EE::Projects::HashedStorageMigrationService prepend ::EE::Projects::HashedStorageMigrationService
attr_reader :old_disk_path, :new_disk_path attr_reader :old_disk_path, :new_disk_path, :old_wiki_disk_path, :old_storage_version
def initialize(project, logger = nil) def initialize(project, logger = nil)
@project = project @project = project
...@@ -17,6 +17,7 @@ module Projects ...@@ -17,6 +17,7 @@ module Projects
@old_disk_path = project.disk_path @old_disk_path = project.disk_path
has_wiki = project.wiki.repository_exists? has_wiki = project.wiki.repository_exists?
@old_storage_version = project.storage_version
project.storage_version = Storage::HashedProject::STORAGE_VERSION project.storage_version = Storage::HashedProject::STORAGE_VERSION
project.ensure_storage_path_exists project.ensure_storage_path_exists
...@@ -25,7 +26,8 @@ module Projects ...@@ -25,7 +26,8 @@ module Projects
result = move_repository(@old_disk_path, @new_disk_path) result = move_repository(@old_disk_path, @new_disk_path)
if has_wiki if has_wiki
result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki") @old_wiki_disk_path = "#{@old_disk_path}.wiki"
result &&= move_repository(@old_wiki_disk_path, "#{@new_disk_path}.wiki")
end end
unless result unless result
......
module Geo
class HashedStorageMigrationWorker
include Sidekiq::Worker
include GeoQueue
def perform(project_id, old_disk_path, new_disk_path, old_storage_version)
Geo::HashedStorageMigrationService.new(
project_id,
old_disk_path: old_disk_path,
new_disk_path: new_disk_path,
old_storage_version: old_storage_version
).execute
end
end
end
module Geo
class RenameRepositoryWorker
include Sidekiq::Worker
include GeoQueue
def perform(project_id, old_disk_path, new_disk_path)
Geo::RenameRepositoryService.new(project_id, old_disk_path, new_disk_path).execute
end
end
end
class GeoRepositoryMoveWorker
include Sidekiq::Worker
include GeoQueue
def perform(id, name, old_path_with_namespace, new_path_with_namespace)
Geo::MoveRepositoryService.new(id, name, old_path_with_namespace, new_path_with_namespace).execute
end
end
---
title: Geo - Does not move projects backed by hashed storage when handling renamed events
merge_request: 3066
author:
type: fixed
class CreateGeoHashedStorageMigratedEvents < ActiveRecord::Migration
DOWNTIME = false
def change
create_table :geo_hashed_storage_migrated_events, id: :bigserial do |t|
t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
t.text :repository_storage_name, null: false
t.text :repository_storage_path, null: false
t.text :old_disk_path, null: false
t.text :new_disk_path, null: false
t.text :old_wiki_disk_path, null: false
t.text :new_wiki_disk_path, null: false
t.integer :old_storage_version, limit: 2
t.integer :new_storage_version, null: false, limit: 2
end
add_column :geo_event_log, :hashed_storage_migrated_event_id, :integer, limit: 8
end
end
class AddGeoHashedStorageMigratedEventsForeignKey < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :geo_event_log, :geo_hashed_storage_migrated_events,
column: :hashed_storage_migrated_event_id, on_delete: :cascade
end
def down
remove_foreign_key :geo_event_log, column: :hashed_storage_migrated_event_id
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20171107090120) do ActiveRecord::Schema.define(version: 20171107144726) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -784,6 +784,7 @@ ActiveRecord::Schema.define(version: 20171107090120) do ...@@ -784,6 +784,7 @@ ActiveRecord::Schema.define(version: 20171107090120) do
t.integer "repository_renamed_event_id", limit: 8 t.integer "repository_renamed_event_id", limit: 8
t.integer "repositories_changed_event_id", limit: 8 t.integer "repositories_changed_event_id", limit: 8
t.integer "repository_created_event_id", limit: 8 t.integer "repository_created_event_id", limit: 8
t.integer "hashed_storage_migrated_event_id", limit: 8
end end
add_index "geo_event_log", ["repositories_changed_event_id"], name: "index_geo_event_log_on_repositories_changed_event_id", using: :btree add_index "geo_event_log", ["repositories_changed_event_id"], name: "index_geo_event_log_on_repositories_changed_event_id", using: :btree
...@@ -792,6 +793,20 @@ ActiveRecord::Schema.define(version: 20171107090120) do ...@@ -792,6 +793,20 @@ ActiveRecord::Schema.define(version: 20171107090120) do
add_index "geo_event_log", ["repository_renamed_event_id"], name: "index_geo_event_log_on_repository_renamed_event_id", using: :btree add_index "geo_event_log", ["repository_renamed_event_id"], name: "index_geo_event_log_on_repository_renamed_event_id", using: :btree
add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
create_table "geo_hashed_storage_migrated_events", id: :bigserial, force: :cascade do |t|
t.integer "project_id", null: false
t.text "repository_storage_name", null: false
t.text "repository_storage_path", null: false
t.text "old_disk_path", null: false
t.text "new_disk_path", null: false
t.text "old_wiki_disk_path", null: false
t.text "new_wiki_disk_path", null: false
t.integer "old_storage_version", limit: 2
t.integer "new_storage_version", limit: 2, null: false
end
add_index "geo_hashed_storage_migrated_events", ["project_id"], name: "index_geo_hashed_storage_migrated_events_on_project_id", using: :btree
create_table "geo_node_namespace_links", force: :cascade do |t| create_table "geo_node_namespace_links", force: :cascade do |t|
t.integer "geo_node_id", null: false t.integer "geo_node_id", null: false
t.integer "namespace_id", null: false t.integer "namespace_id", null: false
...@@ -2289,11 +2304,13 @@ ActiveRecord::Schema.define(version: 20171107090120) do ...@@ -2289,11 +2304,13 @@ ActiveRecord::Schema.define(version: 20171107090120) do
add_foreign_key "gcp_clusters", "projects", on_delete: :cascade add_foreign_key "gcp_clusters", "projects", on_delete: :cascade
add_foreign_key "gcp_clusters", "services", on_delete: :nullify add_foreign_key "gcp_clusters", "services", on_delete: :nullify
add_foreign_key "gcp_clusters", "users", on_delete: :nullify add_foreign_key "gcp_clusters", "users", on_delete: :nullify
add_foreign_key "geo_event_log", "geo_hashed_storage_migrated_events", column: "hashed_storage_migrated_event_id", name: "fk_27548c6db3", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repositories_changed_events", column: "repositories_changed_event_id", name: "fk_4a99ebfd60", on_delete: :cascade add_foreign_key "geo_event_log", "geo_repositories_changed_events", column: "repositories_changed_event_id", name: "fk_4a99ebfd60", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_created_events", column: "repository_created_event_id", name: "fk_9b9afb1916", on_delete: :cascade add_foreign_key "geo_event_log", "geo_repository_created_events", column: "repository_created_event_id", name: "fk_9b9afb1916", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
add_foreign_key "geo_hashed_storage_migrated_events", "projects", on_delete: :cascade
add_foreign_key "geo_node_namespace_links", "geo_nodes", on_delete: :cascade add_foreign_key "geo_node_namespace_links", "geo_nodes", on_delete: :cascade
add_foreign_key "geo_node_namespace_links", "namespaces", on_delete: :cascade add_foreign_key "geo_node_namespace_links", "namespaces", on_delete: :cascade
add_foreign_key "geo_node_statuses", "geo_nodes", on_delete: :cascade add_foreign_key "geo_node_statuses", "geo_nodes", on_delete: :cascade
......
...@@ -445,7 +445,6 @@ module EE ...@@ -445,7 +445,6 @@ module EE
end end
alias_method :merge_requests_ff_only_enabled?, :merge_requests_ff_only_enabled alias_method :merge_requests_ff_only_enabled?, :merge_requests_ff_only_enabled
# TODO: check storage type and NOOP when not using Legacy
def rename_repo def rename_repo
raise NotImplementedError unless defined?(super) raise NotImplementedError unless defined?(super)
......
...@@ -5,10 +5,11 @@ module EE ...@@ -5,10 +5,11 @@ module EE
raise NotImplementedError.new unless defined?(super) raise NotImplementedError.new unless defined?(super)
super do super do
::Geo::RepositoryRenamedEventStore.new( ::Geo::HashedStorageMigratedEventStore.new(
project, project,
old_path: File.basename(old_disk_path), old_storage_version: old_storage_version,
old_path_with_namespace: old_disk_path old_disk_path: old_disk_path,
old_wiki_disk_path: old_wiki_disk_path
).create ).create
end end
end end
......
...@@ -76,6 +76,8 @@ module Gitlab ...@@ -76,6 +76,8 @@ module Gitlab
handle_repositories_changed(event_log.repositories_changed_event) handle_repositories_changed(event_log.repositories_changed_event)
elsif event_log.repository_renamed_event elsif event_log.repository_renamed_event
handle_repository_renamed(event_log) handle_repository_renamed(event_log)
elsif event_log.hashed_storage_migrated_event
handle_hashed_storage_migrated(event_log)
end end
end end
end end
...@@ -182,8 +184,8 @@ module Gitlab ...@@ -182,8 +184,8 @@ module Gitlab
old_path = event.old_path_with_namespace old_path = event.old_path_with_namespace
new_path = event.new_path_with_namespace new_path = event.new_path_with_namespace
job_id = ::Geo::MoveRepositoryService job_id = ::Geo::RenameRepositoryService
.new(event.project_id, '', old_path, new_path) .new(event.project_id, old_path, new_path)
.async_execute .async_execute
log_event_info( log_event_info(
...@@ -195,6 +197,28 @@ module Gitlab ...@@ -195,6 +197,28 @@ module Gitlab
job_id: job_id) job_id: job_id)
end end
def handle_hashed_storage_migrated(event_log)
event = event_log.hashed_storage_migrated_event
return unless event.project_id
job_id = ::Geo::HashedStorageMigrationService.new(
event.project_id,
old_disk_path: event.old_disk_path,
new_disk_path: event.new_disk_path,
old_storage_version: event.old_storage_version
).async_execute
log_event_info(
event_log.created_at,
message: 'Migrating project to hashed storage',
project_id: event.project_id,
old_storage_version: event.old_storage_version,
new_storage_version: event.new_storage_version,
old_disk_path: event.old_disk_path,
new_disk_path: event.new_disk_path,
job_id: job_id)
end
def find_or_initialize_registry(project_id, attrs) def find_or_initialize_registry(project_id, attrs)
registry = ::Geo::ProjectRegistry.find_or_initialize_by(project_id: project_id) registry = ::Geo::ProjectRegistry.find_or_initialize_by(project_id: project_id)
registry.assign_attributes(attrs) registry.assign_attributes(attrs)
......
...@@ -74,7 +74,8 @@ describe Namespace do ...@@ -74,7 +74,8 @@ describe Namespace do
it 'logs the Geo::RepositoryRenamedEvent for each project inside namespace' do it 'logs the Geo::RepositoryRenamedEvent for each project inside namespace' do
parent = create(:namespace) parent = create(:namespace)
child = create(:group, name: 'child', path: 'child', parent: parent) child = create(:group, name: 'child', path: 'child', parent: parent)
project_1 = create(:project_empty_repo, namespace: parent) project_legacy_storage = create(:project_empty_repo, namespace: parent)
create(:project, :hashed, namespace: child)
create(:project_empty_repo, namespace: child) create(:project_empty_repo, namespace: child)
full_path_was = "#{parent.full_path}_old" full_path_was = "#{parent.full_path}_old"
new_path = parent.full_path new_path = parent.full_path
...@@ -85,10 +86,10 @@ describe Namespace do ...@@ -85,10 +86,10 @@ describe Namespace do
allow(parent).to receive(:full_path).and_return(new_path) allow(parent).to receive(:full_path).and_return(new_path)
allow(gitlab_shell).to receive(:mv_namespace) allow(gitlab_shell).to receive(:mv_namespace)
.with(project_1.repository_storage_path, full_path_was, new_path) .with(project_legacy_storage.repository_storage_path, full_path_was, new_path)
.and_return(true) .and_return(true)
expect { parent.move_dir }.to change(Geo::RepositoryRenamedEvent, :count).by(2) expect { parent.move_dir }.to change(Geo::RepositoryRenamedEvent, :count).by(3)
end end
end end
end end
......
...@@ -936,14 +936,19 @@ describe Project do ...@@ -936,14 +936,19 @@ describe Project do
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new } let(:gitlab_shell) { Gitlab::Shell.new }
before do it 'logs the Geo::RepositoryRenamedEvent for project backed by hashed storage' do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) project_hashed_storage = create(:project, :hashed)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'logs the Geo::RepositoryRenamedEvent' do allow(project_hashed_storage).to receive(:gitlab_shell).and_return(gitlab_shell)
stub_container_registry_config(enabled: false) allow(project_hashed_storage).to receive(:previous_changes).and_return('path' => ['foo'])
allow(gitlab_shell).to receive(:mv_repository).twice.and_return(true)
expect { project_hashed_storage.rename_repo }.to change(Geo::RepositoryRenamedEvent, :count)
end
it 'logs the Geo::RepositoryRenamedEvent for project backed by legacy storage' do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
allow(gitlab_shell).to receive(:mv_repository).twice.and_return(true) allow(gitlab_shell).to receive(:mv_repository).twice.and_return(true)
expect(Geo::RepositoryRenamedEventStore).to receive(:new) expect(Geo::RepositoryRenamedEventStore).to receive(:new)
......
...@@ -10,19 +10,19 @@ describe Projects::HashedStorageMigrationService do ...@@ -10,19 +10,19 @@ describe Projects::HashedStorageMigrationService do
set(:primary) { create(:geo_node, :primary) } set(:primary) { create(:geo_node, :primary) }
set(:secondary) { create(:geo_node) } set(:secondary) { create(:geo_node) }
it 'creates a Geo::RepositoryRenamedEvent on success' do it 'creates a Geo::HashedStorageMigratedEvent on success' do
expect { service.execute }.to change { Geo::EventLog.count }.by(1) expect { service.execute }.to change(Geo::EventLog, :count).by(1)
event = Geo::EventLog.first.event event = Geo::EventLog.first.event
expect(event).to be_a(Geo::RepositoryRenamedEvent) expect(event).to be_a(Geo::HashedStorageMigratedEvent)
expect(event).to have_attributes( expect(event).to have_attributes(
old_path: project.path, old_storage_version: nil,
new_path: project.path, new_storage_version: Storage::HashedProject::STORAGE_VERSION,
old_path_with_namespace: legacy_storage.disk_path, old_disk_path: legacy_storage.disk_path,
new_path_with_namespace: hashed_storage.disk_path, new_disk_path: hashed_storage.disk_path,
old_wiki_path_with_namespace: legacy_storage.disk_path + '.wiki', old_wiki_disk_path: legacy_storage.disk_path + '.wiki',
new_wiki_path_with_namespace: hashed_storage.disk_path + '.wiki' new_wiki_disk_path: hashed_storage.disk_path + '.wiki'
) )
end end
......
...@@ -13,7 +13,11 @@ FactoryGirl.define do ...@@ -13,7 +13,11 @@ FactoryGirl.define do
end end
trait :renamed_event do trait :renamed_event do
repository_renamed_event factory: :geo_repository_rename_event repository_renamed_event factory: :geo_repository_renamed_event
end
trait :hashed_storage_migration_event do
hashed_storage_migrated_event factory: :geo_hashed_storage_migrated_event
end end
end end
...@@ -44,24 +48,11 @@ FactoryGirl.define do ...@@ -44,24 +48,11 @@ FactoryGirl.define do
deleted_project_name { project.name } deleted_project_name { project.name }
end end
factory :geo_repository_renamed_event, class: Geo::RepositoryRenamedEvent do
project
repository_storage_name { project.repository_storage }
repository_storage_path { project.repository_storage_path }
old_path_with_namespace { project.full_path }
new_path_with_namespace { project.full_path }
old_wiki_path_with_namespace { project.wiki.path_with_namespace }
new_wiki_path_with_namespace { project.wiki.path_with_namespace }
old_path { project.path }
new_path { project.path }
end
factory :geo_repositories_changed_event, class: Geo::RepositoriesChangedEvent do factory :geo_repositories_changed_event, class: Geo::RepositoriesChangedEvent do
geo_node geo_node
end end
factory :geo_repository_rename_event, class: Geo::RepositoryRenamedEvent do factory :geo_repository_renamed_event, class: Geo::RepositoryRenamedEvent do
project { create(:project, :repository) } project { create(:project, :repository) }
repository_storage_name { project.repository_storage } repository_storage_name { project.repository_storage }
...@@ -73,4 +64,16 @@ FactoryGirl.define do ...@@ -73,4 +64,16 @@ FactoryGirl.define do
old_path { project.path } old_path { project.path }
new_path { project.path + '_new' } new_path { project.path + '_new' }
end end
factory :geo_hashed_storage_migrated_event, class: Geo::HashedStorageMigratedEvent do
project { create(:project, :repository) }
repository_storage_name { project.repository_storage }
repository_storage_path { project.repository_storage_path }
old_disk_path { project.path_with_namespace }
new_disk_path { project.path_with_namespace + '_new' }
old_wiki_disk_path { project.wiki.path_with_namespace }
new_wiki_disk_path { project.wiki.path_with_namespace + '_new' }
new_storage_version { Project::LATEST_STORAGE_VERSION }
end
end end
...@@ -197,21 +197,42 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do ...@@ -197,21 +197,42 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
context 'when processing a repository renamed event' do context 'when processing a repository renamed event' do
let(:event_log) { create(:geo_event_log, :renamed_event) } let(:event_log) { create(:geo_event_log, :renamed_event) }
let(:project) { event_log.repository_rename_event.project }
let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) } let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
let(:repository_rename_event) { event_log.repository_renamed_event } let(:repository_renamed_event) { event_log.repository_renamed_event }
it 'does not create a new project registry' do it 'does not create a new project registry' do
expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count) expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
end end
it 'schedules a GeoRepositoryDestroyWorker' do it 'schedules a Geo::RenameRepositoryWorker' do
project_id = repository_rename_event.project_id project_id = repository_renamed_event.project_id
old_path_with_namespace = repository_rename_event.old_path_with_namespace old_path_with_namespace = repository_renamed_event.old_path_with_namespace
new_path_with_namespace = repository_rename_event.new_path_with_namespace new_path_with_namespace = repository_renamed_event.new_path_with_namespace
expect(::Geo::RenameRepositoryWorker).to receive(:perform_async)
.with(project_id, old_path_with_namespace, new_path_with_namespace)
daemon.run_once!
end
end
context 'when processing a hashed storage migration event' do
let(:event_log) { create(:geo_event_log, :hashed_storage_migration_event) }
let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
let(:hashed_storage_migrated_event) { event_log.hashed_storage_migrated_event }
it 'does not create a new project registry' do
expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
end
it 'schedules a Geo::HashedStorageMigrationWorker' do
project = hashed_storage_migrated_event.project
old_disk_path = hashed_storage_migrated_event.old_disk_path
new_disk_path = hashed_storage_migrated_event.new_disk_path
old_storage_version = project.storage_version
expect(::GeoRepositoryMoveWorker).to receive(:perform_async) expect(::Geo::HashedStorageMigrationWorker).to receive(:perform_async)
.with(project_id, '', old_path_with_namespace, new_path_with_namespace) .with(project.id, old_disk_path, new_disk_path, old_storage_version)
daemon.run_once! daemon.run_once!
end end
......
...@@ -7,6 +7,7 @@ RSpec.describe Geo::EventLog, type: :model do ...@@ -7,6 +7,7 @@ RSpec.describe Geo::EventLog, type: :model do
it { is_expected.to belong_to(:repository_deleted_event).class_name('Geo::RepositoryDeletedEvent').with_foreign_key('repository_deleted_event_id') } it { is_expected.to belong_to(:repository_deleted_event).class_name('Geo::RepositoryDeletedEvent').with_foreign_key('repository_deleted_event_id') }
it { is_expected.to belong_to(:repository_renamed_event).class_name('Geo::RepositoryRenamedEvent').with_foreign_key('repository_renamed_event_id') } it { is_expected.to belong_to(:repository_renamed_event).class_name('Geo::RepositoryRenamedEvent').with_foreign_key('repository_renamed_event_id') }
it { is_expected.to belong_to(:repository_updated_event).class_name('Geo::RepositoryUpdatedEvent').with_foreign_key('repository_updated_event_id') } it { is_expected.to belong_to(:repository_updated_event).class_name('Geo::RepositoryUpdatedEvent').with_foreign_key('repository_updated_event_id') }
it { is_expected.to belong_to(:hashed_storage_migrated_event).class_name('Geo::HashedStorageMigratedEvent').with_foreign_key('hashed_storage_migrated_event_id') }
end end
describe '#event' do describe '#event' do
...@@ -48,6 +49,13 @@ RSpec.describe Geo::EventLog, type: :model do ...@@ -48,6 +49,13 @@ RSpec.describe Geo::EventLog, type: :model do
expect(subject.event).to eq repositories_changed_event expect(subject.event).to eq repositories_changed_event
end end
it 'returns hashed_storage_migrated_event when set' do
hashed_storage_migrated_event = build(:geo_hashed_storage_migrated_event)
subject.hashed_storage_migrated_event = hashed_storage_migrated_event
expect(subject.event).to eq hashed_storage_migrated_event
end
end end
describe '#project_id' do describe '#project_id' do
......
require 'spec_helper'
RSpec.describe Geo::HashedStorageMigratedEvent, type: :model do
describe 'relationships' do
it { is_expected.to belong_to(:project) }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:project) }
it { is_expected.to validate_presence_of(:repository_storage_name) }
it { is_expected.to validate_presence_of(:repository_storage_path) }
it { is_expected.to validate_presence_of(:old_disk_path) }
it { is_expected.to validate_presence_of(:new_disk_path) }
it { is_expected.to validate_presence_of(:old_wiki_disk_path) }
it { is_expected.to validate_presence_of(:new_wiki_disk_path) }
it { is_expected.to validate_presence_of(:new_storage_version) }
end
end
require 'spec_helper'
describe Geo::HashedStorageMigratedEventStore do
set(:project) { create(:project, :hashed, path: 'bar') }
set(:secondary_node) { create(:geo_node) }
let(:old_disk_path) { "#{project.namespace.full_path}/foo" }
let(:old_wiki_disk_path) { "#{old_disk_path}.wiki" }
subject(:event_store) { described_class.new(project, old_storage_version: nil, old_disk_path: old_disk_path, old_wiki_disk_path: old_wiki_disk_path) }
describe '#create' do
it 'does not create an event when not running on a primary node' do
allow(Gitlab::Geo).to receive(:primary?) { false }
expect { event_store.create }.not_to change(Geo::HashedStorageMigratedEvent, :count)
end
context 'when running on a primary node' do
before do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'does not create an event when there are no secondary nodes' do
allow(Gitlab::Geo).to receive(:secondary_nodes) { [] }
expect { subject.create }.not_to change(Geo::HashedStorageMigratedEvent, :count)
end
it 'creates a hashed migration event' do
expect { event_store.create }.to change(Geo::HashedStorageMigratedEvent, :count).by(1)
end
it 'tracks project attributes' do
event_store.create
event = Geo::HashedStorageMigratedEvent.last
expect(event).to have_attributes(
repository_storage_name: project.repository_storage,
repository_storage_path: project.repository_storage_path,
old_storage_version: nil,
new_storage_version: project.storage_version,
old_disk_path: old_disk_path,
new_disk_path: project.disk_path,
old_wiki_disk_path: old_wiki_disk_path,
new_wiki_disk_path: project.wiki.disk_path
)
end
end
end
end
require 'spec_helper'
describe Geo::HashedStorageMigrationService do
let(:project) { create(:project, :repository, :hashed) }
let(:new_path) { "#{project.full_path}+renamed" }
describe '#execute' do
it 'moves project backed by legacy storage' do
service = described_class.new(
project.id,
old_disk_path: project.full_path,
new_disk_path: new_path,
old_storage_version: nil
)
expect_any_instance_of(Geo::MoveRepositoryService).to receive(:execute).once
service.execute
end
it 'does not move project backed by hashed storage' do
service = described_class.new(
project.id,
old_disk_path: project.full_path,
new_disk_path: "#{project.full_path}+renamed",
old_storage_version: project.storage_version
)
expect_any_instance_of(Geo::MoveRepositoryService).not_to receive(:execute).once
service.execute
end
end
describe '#async_execute' do
subject(:service) { described_class.new(project.id, old_disk_path: project.full_path, new_disk_path: new_path, old_storage_version: nil) }
it 'starts the worker' do
expect(Geo::HashedStorageMigrationWorker).to receive(:perform_async)
service.async_execute
end
it 'returns job id' do
allow(Geo::HashedStorageMigrationWorker).to receive(:perform_async).and_return('foo')
expect(service.async_execute).to eq('foo')
end
end
end
...@@ -2,33 +2,19 @@ require 'spec_helper' ...@@ -2,33 +2,19 @@ require 'spec_helper'
describe Geo::MoveRepositoryService do describe Geo::MoveRepositoryService do
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:new_path) { project.full_path + '+renamed' } let(:new_path) { "#{project.full_path}+renamed" }
let(:full_new_path) { File.join(project.repository_storage_path, new_path) }
subject { described_class.new(project.id, project.name, project.full_path, new_path) }
describe '#execute' do describe '#execute' do
it 'renames the path' do it 'renames the path' do
old_path = project.repository.path_to_repo old_path = project.repository.path_to_repo
expect(File.directory?(old_path)).to be_truthy full_new_path = File.join(project.repository_storage_path, new_path)
expect(subject.execute).to eq(true) service = described_class.new(project, project.full_path, new_path)
expect(File.directory?(old_path)).to be_truthy
expect(service.execute).to eq(true)
expect(File.directory?(old_path)).to be_falsey expect(File.directory?(old_path)).to be_falsey
expect(File.directory?("#{full_new_path}.git")).to be_truthy expect(File.directory?("#{full_new_path}.git")).to be_truthy
end end
end end
describe '#async_execute' do
it 'starts the worker' do
expect(GeoRepositoryMoveWorker).to receive(:perform_async)
subject.async_execute
end
it 'returns job id' do
allow(GeoRepositoryMoveWorker).to receive(:perform_async).and_return('foo')
expect(subject.async_execute).to eq('foo')
end
end
end end
require 'spec_helper'
describe Geo::RenameRepositoryService do
let(:project) { create(:project, :repository) }
let(:new_path) { "#{project.full_path}+renamed" }
describe '#execute' do
it 'moves project backed by legacy storage' do
service = described_class.new(project.id, project.full_path, new_path)
expect_any_instance_of(Geo::MoveRepositoryService).to receive(:execute).once
service.execute
end
it 'does not move project backed by hashed storage' do
project_hashed_storage = create(:project, :hashed)
service = described_class.new(project_hashed_storage.id, project_hashed_storage.full_path, new_path)
expect_any_instance_of(Geo::MoveRepositoryService).not_to receive(:execute)
service.execute
end
end
describe '#async_execute' do
subject(:service) { described_class.new(project.id, project.full_path, new_path) }
it 'starts the worker' do
expect(Geo::RenameRepositoryWorker).to receive(:perform_async)
service.async_execute
end
it 'returns job id' do
allow(Geo::RenameRepositoryWorker).to receive(:perform_async).and_return('foo')
expect(service.async_execute).to eq('foo')
end
end
end
...@@ -6,13 +6,13 @@ describe Geo::RepositoryRenamedEventStore do ...@@ -6,13 +6,13 @@ describe Geo::RepositoryRenamedEventStore do
let(:old_path) { 'foo' } let(:old_path) { 'foo' }
let(:old_path_with_namespace) { "#{project.namespace.full_path}/foo" } let(:old_path_with_namespace) { "#{project.namespace.full_path}/foo" }
subject { described_class.new(project, old_path: old_path, old_path_with_namespace: old_path_with_namespace) } subject(:event_store) { described_class.new(project, old_path: old_path, old_path_with_namespace: old_path_with_namespace) }
describe '#create' do describe '#create' do
it 'does not create an event when not running on a primary node' do it 'does not create an event when not running on a primary node' do
allow(Gitlab::Geo).to receive(:primary?) { false } allow(Gitlab::Geo).to receive(:primary?) { false }
expect { subject.create }.not_to change(Geo::RepositoryRenamedEvent, :count) expect { event_store.create }.not_to change(Geo::RepositoryRenamedEvent, :count)
end end
context 'when running on a primary node' do context 'when running on a primary node' do
...@@ -27,11 +27,11 @@ describe Geo::RepositoryRenamedEventStore do ...@@ -27,11 +27,11 @@ describe Geo::RepositoryRenamedEventStore do
end end
it 'creates a renamed event' do it 'creates a renamed event' do
expect { subject.create }.to change(Geo::RepositoryRenamedEvent, :count).by(1) expect { event_store.create }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
end end
it 'tracks old and new paths for project repositories' do it 'tracks old and new paths for project repositories' do
subject.create event_store.create
event = Geo::RepositoryRenamedEvent.last event = Geo::RepositoryRenamedEvent.last
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment