Commit 89e685fb authored by Sean McGivern's avatar Sean McGivern

Merge branch 'da-geo-log-namespace-project-rename-events' into 'master'

Add Geo repository renamed event log

Closes #2641

See merge request !2238
parents e79f1acc b2a35fcc
......@@ -29,6 +29,26 @@ module EE
validates :plan, inclusion: { in: EE_PLANS.keys }, allow_blank: true
end
def move_dir
raise NotImplementedError unless defined?(super)
succeeded = super
if succeeded
all_projects.each do |project|
old_path_with_namespace = File.join(full_path_was, project.path)
::Geo::RepositoryRenamedEventStore.new(
project,
old_path: project.path,
old_path_with_namespace: old_path_with_namespace
).create
end
end
succeeded
end
# Checks features (i.e. https://about.gitlab.com/products/) availabily
# for a given Namespace plan. This method should consider ancestor groups
# being licensed.
......
......@@ -425,6 +425,21 @@ module EE
end
alias_method :merge_requests_ff_only_enabled?, :merge_requests_ff_only_enabled
def rename_repo
raise NotImplementedError unless defined?(super)
super
path_was = previous_changes['path'].first
old_path_with_namespace = File.join(namespace.full_path, path_was)
::Geo::RepositoryRenamedEventStore.new(
self,
old_path: path_was,
old_path_with_namespace: old_path_with_namespace
).create
end
private
def licensed_feature_available?(feature)
......
......@@ -9,5 +9,9 @@ module Geo
belongs_to :repository_deleted_event,
class_name: 'Geo::RepositoryDeletedEvent',
foreign_key: :repository_deleted_event_id
belongs_to :repository_renamed_event,
class_name: 'Geo::RepositoryRenamedEvent',
foreign_key: :repository_renamed_event_id
end
end
module Geo
class RepositoryRenamedEvent < ActiveRecord::Base
include Geo::Model
belongs_to :project
validates :project, :repository_storage_name, :repository_storage_path,
:old_path_with_namespace, :new_path_with_namespace,
:old_wiki_path_with_namespace, :new_wiki_path_with_namespace,
:old_path, :new_path, presence: true
end
end
......@@ -144,6 +144,7 @@ class Namespace < ActiveRecord::Base
# So we basically we mute exceptions in next actions
begin
send_update_instructions
true
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
......
module EE
module Projects
module TransferService
private
def execute_system_hooks
raise NotImplementedError unless defined?(super)
super
::Geo::RepositoryRenamedEventStore.new(
project,
old_path: project.path,
old_path_with_namespace: @old_path
).create
end
end
end
end
module Geo
# Base class for event store classes.
#
# Each store should also specify its event type by calling
# `self.event_type = ...` in the body of the class. The value of
# this method should be a symbol such as `:repository_updated_event`
# or `:repository_deleted_event`. For example:
#
# class RepositoryUpdatedEventStore < EventStore
# self.event_type = :repository_updated_event
# end
#
# The event type is used to determine which attribute we should set
# on an instance of the Geo::EventLog class.
#
# Event store classes should implement the instance method `build_event`.
# The `build_event` method is supposed to return an instance of the event
# that will be logged.
class EventStore
class << self
attr_accessor :event_type
end
attr_reader :project, :params
def initialize(project, params = {})
@project = project
@params = params
end
def create
return unless Gitlab::Geo.primary?
Geo::EventLog.create!("#{self.class.event_type}" => build_event)
rescue ActiveRecord::RecordInvalid, NoMethodError => e
log("#{self.event_type.to_s.humanize} could not be created", e)
end
private
def build_event
raise NotImplementedError,
"#{self.class} does not implement #{__method__}"
end
def log(message, error)
Rails.logger.error("#{self.class.name}: #{message} for project #{project.path_with_namespace} (#{project.id}): #{error}")
end
end
end
module Geo
class RepositoryDeletedEventStore
attr_reader :project, :repo_path, :wiki_path
class RepositoryDeletedEventStore < EventStore
self.event_type = :repository_deleted_event
def initialize(project, repo_path:, wiki_path:)
@project = project
@repo_path = repo_path
@wiki_path = wiki_path
end
def create
return unless Gitlab::Geo.primary?
private
Geo::EventLog.transaction do
event_log = Geo::EventLog.new
deleted_event = Geo::RepositoryDeletedEvent.new(
project: project,
repository_storage_name: project.repository.storage,
repository_storage_path: project.repository_storage_path,
deleted_path: repo_path,
deleted_wiki_path: wiki_path,
deleted_project_name: project.name)
event_log.repository_deleted_event = deleted_event
event_log.save
end
def build_event
Geo::RepositoryDeletedEvent.new(
project: project,
repository_storage_name: project.repository.storage,
repository_storage_path: project.repository_storage_path,
deleted_path: params.fetch(:repo_path),
deleted_wiki_path: params.fetch(:wiki_path),
deleted_project_name: project.name)
end
end
end
module Geo
class RepositoryRenamedEventStore < EventStore
self.event_type = :repository_renamed_event
private
def build_event
Geo::RepositoryRenamedEvent.new(
project: project,
repository_storage_name: project.repository.storage,
repository_storage_path: project.repository_storage_path,
old_path_with_namespace: old_path_with_namespace,
new_path_with_namespace: project.full_path,
old_wiki_path_with_namespace: old_wiki_path_with_namespace,
new_wiki_path_with_namespace: new_wiki_path_with_namespace,
old_path: params.fetch(:old_path),
new_path: project.path
)
end
def old_path_with_namespace
params.fetch(:old_path_with_namespace)
end
def old_wiki_path_with_namespace
"#{old_path_with_namespace}.wiki"
end
def new_wiki_path_with_namespace
project.wiki.path_with_namespace
end
end
end
module Geo
class RepositoryUpdatedEventStore
attr_reader :project, :source, :refs, :changes
def initialize(project, refs: [], changes: [], source: Geo::RepositoryUpdatedEvent::REPOSITORY)
@project = project
@refs = refs
@changes = changes
@source = source
end
def create
return unless Gitlab::Geo.primary?
Geo::EventLog.transaction do
event_log = Geo::EventLog.new
event_log.repository_updated_event = build_event
event_log.save!
end
rescue ActiveRecord::RecordInvalid
log("#{Geo::PushEvent.sources.key(source).humanize} updated event could not be created")
end
class RepositoryUpdatedEventStore < EventStore
self.event_type = :repository_updated_event
private
......@@ -35,6 +16,18 @@ module Geo
)
end
def refs
params.fetch(:refs, [])
end
def changes
params.fetch(:changes, [])
end
def source
params.fetch(:source, Geo::RepositoryUpdatedEvent::REPOSITORY)
end
def ref
refs.first if refs.length == 1
end
......@@ -54,9 +47,5 @@ module Geo
def push_remove_branch?
changes.any? { |change| Gitlab::Git.branch_ref?(change[:ref]) && Gitlab::Git.blank_ref?(change[:after]) }
end
def log(message)
Rails.logger.info("#{self.class.name}: #{message} for project #{project.path_with_namespace} (#{project.id})")
end
end
end
......@@ -11,6 +11,8 @@ module Projects
include Gitlab::ShellAdapter
TransferError = Class.new(StandardError)
prepend ::EE::Projects::TransferService
def execute(new_namespace)
@new_namespace = new_namespace
......
---
title: Add Geo repository renamed event log
merge_request:
author:
class CreateGeoRepositoryRenamedEvents < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :geo_repository_renamed_events, id: :bigserial do |t|
t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
t.text :repository_storage_name, null: false
t.text :repository_storage_path, null: false
t.text :old_path_with_namespace, null: false
t.text :new_path_with_namespace, null: false
t.text :old_wiki_path_with_namespace, null: false
t.text :new_wiki_path_with_namespace, null: false
t.text :old_path, null: false
t.text :new_path, null: false
end
add_column :geo_event_log, :repository_renamed_event_id, :integer, limit: 8
end
end
class AddGeoRepositoryRenamedEventsForeignKey < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_foreign_key :geo_event_log, :geo_repository_renamed_events,
column: :repository_renamed_event_id, on_delete: :cascade
end
def down
remove_foreign_key :geo_event_log, column: :repository_renamed_event_id
end
end
......@@ -588,6 +588,7 @@ ActiveRecord::Schema.define(version: 20170627211700) do
t.datetime "created_at", null: false
t.integer "repository_updated_event_id", limit: 8
t.integer "repository_deleted_event_id", limit: 8
t.integer "repository_renamed_event_id", limit: 8
end
add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
......@@ -625,6 +626,20 @@ ActiveRecord::Schema.define(version: 20170627211700) do
add_index "geo_repository_deleted_events", ["project_id"], name: "index_geo_repository_deleted_events_on_project_id", using: :btree
create_table "geo_repository_renamed_events", id: :bigserial, force: :cascade do |t|
t.integer "project_id", null: false
t.text "repository_storage_name", null: false
t.text "repository_storage_path", null: false
t.text "old_path_with_namespace", null: false
t.text "new_path_with_namespace", null: false
t.text "old_wiki_path_with_namespace", null: false
t.text "new_wiki_path_with_namespace", null: false
t.text "old_path", null: false
t.text "new_path", null: false
end
add_index "geo_repository_renamed_events", ["project_id"], name: "index_geo_repository_renamed_events_on_project_id", using: :btree
create_table "geo_repository_updated_events", id: :bigserial, force: :cascade do |t|
t.datetime "created_at", null: false
t.integer "branches_affected", null: false
......@@ -1862,7 +1877,9 @@ ActiveRecord::Schema.define(version: 20170627211700) do
add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade
add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
add_foreign_key "geo_repository_renamed_events", "projects", on_delete: :cascade
add_foreign_key "geo_repository_updated_events", "projects", on_delete: :cascade
add_foreign_key "index_statuses", "projects", name: "fk_74b2492545", on_delete: :cascade
add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade
......
......@@ -42,6 +42,34 @@ describe Namespace, models: true do
end
end
describe '#move_dir' do
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary, :current) }
let(:gitlab_shell) { Gitlab::Shell.new }
it 'logs the Geo::RepositoryRenamedEvent for each project inside namespace' do
parent = create(:namespace)
child = create(:group, name: 'child', path: 'child', parent: parent)
project_1 = create(:project_empty_repo, namespace: parent)
create(:project_empty_repo, namespace: child)
full_path_was = "#{parent.full_path}_old"
new_path = parent.full_path
allow(parent).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(parent).to receive(:path_changed?).and_return(true)
allow(parent).to receive(:full_path_was).and_return(full_path_was)
allow(parent).to receive(:full_path).and_return(new_path)
allow(gitlab_shell).to receive(:mv_namespace)
.ordered
.with(project_1.repository_storage_path, full_path_was, new_path)
.and_return(true)
expect { parent.move_dir }.to change(Geo::RepositoryRenamedEvent, :count).by(2)
end
end
end
describe '#feature_available?' do
let(:plan_license) { Namespace::BRONZE_PLAN }
let(:group) { create(:group, plan: plan_license) }
......
......@@ -604,4 +604,37 @@ describe Project, models: true do
end
end
end
describe '#rename_repo' do
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary, :current) }
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
before do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'logs the Geo::RepositoryRenamedEvent' do
stub_container_registry_config(enabled: false)
allow(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
.and_return(true)
allow(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
.and_return(true)
expect(Geo::RepositoryRenamedEventStore).to receive(:new)
.with(instance_of(Project), old_path: 'foo', old_path_with_namespace: "#{project.namespace.full_path}/foo")
.and_call_original
expect { project.rename_repo }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
end
end
end
end
......@@ -3,5 +3,6 @@ require 'spec_helper'
RSpec.describe Geo::EventLog, type: :model do
describe 'relationships' do
it { is_expected.to belong_to(:repository_updated_event).class_name('Geo::RepositoryUpdatedEvent').with_foreign_key('repository_updated_event_id') }
it { is_expected.to belong_to(:repository_renamed_event).class_name('Geo::RepositoryRenamedEvent').with_foreign_key('repository_renamed_event_id') }
end
end
require 'spec_helper'
RSpec.describe Geo::RepositoryRenamedEvent, type: :model do
describe 'relationships' do
it { is_expected.to belong_to(:project) }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:project) }
it { is_expected.to validate_presence_of(:repository_storage_name) }
it { is_expected.to validate_presence_of(:repository_storage_path) }
it { is_expected.to validate_presence_of(:old_path_with_namespace) }
it { is_expected.to validate_presence_of(:new_path_with_namespace) }
it { is_expected.to validate_presence_of(:old_wiki_path_with_namespace) }
it { is_expected.to validate_presence_of(:new_wiki_path_with_namespace) }
it { is_expected.to validate_presence_of(:old_path) }
it { is_expected.to validate_presence_of(:new_path) }
end
end
......@@ -10,32 +10,22 @@ describe Projects::DestroyService, services: true do
let!(:wiki_path) { project.path_with_namespace + '.wiki' }
let!(:storage_name) { project.repository_storage }
let!(:storage_path) { project.repository_storage_path }
let!(:geo_node) { create(:geo_node, :primary, :current) }
subject { described_class.new(project, user, {}) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: [])
end
context 'Geo primary' do
it 'logs the event' do
# Run sidekiq immediatly to check that renamed repository will be removed
Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
event = Geo::RepositoryDeletedEvent.first
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary, :current) }
expect(Geo::EventLog.count).to eq(1)
expect(Geo::RepositoryDeletedEvent.count).to eq(1)
expect(event.project_id).to eq(project_id)
expect(event.deleted_path).to eq(project_path)
expect(event.deleted_wiki_path).to eq(wiki_path)
expect(event.deleted_project_name).to eq(project_name)
expect(event.repository_storage_name).to eq(storage_name)
expect(event.repository_storage_path).to eq(storage_path)
it 'logs an event to the Geo event log' do
# Run Sidekiq immediately to check that renamed repository will be removed
Sidekiq::Testing.inline! do
expect { subject.execute }.to change(Geo::RepositoryDeletedEvent, :count).by(1)
end
end
end
def destroy_project(project, user, params = {})
described_class.new(project, user, params).execute
end
end
# rubocop:disable RSpec/FilePath
require 'spec_helper'
describe Projects::TransferService, services: true do
let(:user) { create(:user) }
let(:group) { create(:group) }
let(:project) { create(:project, :repository, namespace: user.namespace) }
subject { described_class.new(project, user) }
before do
group.add_owner(user)
end
context 'when running on a primary node' do
let!(:geo_node) { create(:geo_node, :primary, :current) }
it 'logs an event to the Geo event log' do
expect { subject.execute(group) }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
end
end
end
require 'spec_helper'
describe Geo::RepositoryDeletedEventStore, services: true do
let(:project) { create(:empty_project, path: 'bar') }
let!(:project_id) { project.id }
let!(:project_name) { project.name }
let!(:repo_path) { project.full_path }
let!(:wiki_path) { "#{project.full_path}.wiki" }
let!(:storage_name) { project.repository_storage }
let!(:storage_path) { project.repository_storage_path }
subject { described_class.new(project, repo_path: repo_path, wiki_path: wiki_path) }
describe '#create' do
it 'does not create an event when not running on a primary node' do
allow(Gitlab::Geo).to receive(:primary?) { false }
expect { subject.create }.not_to change(Geo::RepositoryDeletedEvent, :count)
end
context 'when running on a primary node' do
before do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'creates a deleted event' do
expect { subject.create }.to change(Geo::RepositoryDeletedEvent, :count).by(1)
end
it 'tracks information for the deleted project' do
subject.create
event = Geo::RepositoryDeletedEvent.last
expect(event.project_id).to eq(project_id)
expect(event.deleted_path).to eq(repo_path)
expect(event.deleted_wiki_path).to eq(wiki_path)
expect(event.deleted_project_name).to eq(project_name)
expect(event.repository_storage_name).to eq(storage_name)
expect(event.repository_storage_path).to eq(storage_path)
end
end
end
end
require 'spec_helper'
describe Geo::RepositoryRenamedEventStore, services: true do
let(:project) { create(:empty_project, path: 'bar') }
let(:old_path) { 'foo' }
let(:old_path_with_namespace) { "#{project.namespace.full_path}/foo" }
subject { described_class.new(project, old_path: old_path, old_path_with_namespace: old_path_with_namespace) }
describe '#create' do
it 'does not create an event when not running on a primary node' do
allow(Gitlab::Geo).to receive(:primary?) { false }
expect { subject.create }.not_to change(Geo::RepositoryRenamedEvent, :count)
end
context 'when running on a primary node' do
before do
allow(Gitlab::Geo).to receive(:primary?) { true }
end
it 'creates a renamed event' do
expect { subject.create }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
end
it 'tracks old and new paths for project repositories' do
subject.create
event = Geo::RepositoryRenamedEvent.last
expect(event.repository_storage_name).to eq(project.repository_storage)
expect(event.repository_storage_path).to eq(project.repository_storage_path)
expect(event.old_path_with_namespace).to eq(old_path_with_namespace)
expect(event.new_path_with_namespace).to eq(project.full_path)
expect(event.old_wiki_path_with_namespace).to eq("#{old_path_with_namespace}.wiki")
expect(event.new_wiki_path_with_namespace).to eq("#{project.full_path}.wiki")
expect(event.old_path).to eq(old_path)
expect(event.new_path).to eq(project.path)
end
end
end
end
......@@ -37,7 +37,7 @@ describe Projects::TransferService, services: true do
end
it 'executes system hooks' do
expect_any_instance_of(Projects::TransferService).to receive(:execute_system_hooks)
expect_any_instance_of(SystemHooksService).to receive(:execute_hooks_for).with(project, :transfer)
transfer_project(project, user, group)
end
......@@ -80,7 +80,7 @@ describe Projects::TransferService, services: true do
end
it "doesn't run system hooks" do
expect_any_instance_of(Projects::TransferService).not_to receive(:execute_system_hooks)
expect_any_instance_of(SystemHooksService).not_to receive(:execute_hooks_for).with(project, :transfer)
attempt_project_transfer
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment