Merge branch 'da-geo-log-namespace-project-rename-events' into 'master'

Add Geo repository renamed event log

Closes #2641

See merge request !2238

app/models/ee/namespace.rb

@@ -29,6 +29,26 @@ module EE
       validates :plan, inclusion: { in: EE_PLANS.keys }, allow_blank: true
     end
 
+    def move_dir
+      raise NotImplementedError unless defined?(super)
+
+      succeeded = super
+
+      if succeeded
+        all_projects.each do |project|
+          old_path_with_namespace = File.join(full_path_was, project.path)
+
+          ::Geo::RepositoryRenamedEventStore.new(
+            project,
+            old_path: project.path,
+            old_path_with_namespace: old_path_with_namespace
+          ).create
+        end
+      end
+
+      succeeded
+    end
+
     # Checks features (i.e. https://about.gitlab.com/products/) availabily
     # for a given Namespace plan. This method should consider ancestor groups
     # being licensed.

app/models/ee/project.rb

@@ -425,6 +425,21 @@ module EE
     end
     alias_method :merge_requests_ff_only_enabled?, :merge_requests_ff_only_enabled
 
+    def rename_repo
+      raise NotImplementedError unless defined?(super)
+
+      super
+
+      path_was = previous_changes['path'].first
+      old_path_with_namespace = File.join(namespace.full_path, path_was)
+
+      ::Geo::RepositoryRenamedEventStore.new(
+        self,
+        old_path: path_was,
+        old_path_with_namespace: old_path_with_namespace
+      ).create
+    end
+
     private
 
     def licensed_feature_available?(feature)

app/models/geo/event_log.rb

@@ -9,5 +9,9 @@ module Geo
     belongs_to :repository_deleted_event,
       class_name: 'Geo::RepositoryDeletedEvent',
       foreign_key: :repository_deleted_event_id
+
+    belongs_to :repository_renamed_event,
+      class_name: 'Geo::RepositoryRenamedEvent',
+      foreign_key: :repository_renamed_event_id
   end
 end

app/models/geo/repository_renamed_event.rb

+module Geo
+  class RepositoryRenamedEvent < ActiveRecord::Base
+    include Geo::Model
+
+    belongs_to :project
+
+    validates :project, :repository_storage_name, :repository_storage_path,
+              :old_path_with_namespace, :new_path_with_namespace,
+              :old_wiki_path_with_namespace, :new_wiki_path_with_namespace,
+              :old_path, :new_path, presence: true
+  end
+end

app/models/namespace.rb

@@ -144,6 +144,7 @@ class Namespace < ActiveRecord::Base
     # So we basically we mute exceptions in next actions
     begin
       send_update_instructions
+      true
     rescue
       # Returning false does not rollback after_* transaction but gives
       # us information about failing some of tasks

app/services/ee/projects/transfer_service.rb

+module EE
+  module Projects
+    module TransferService
+      private
+
+      def execute_system_hooks
+        raise NotImplementedError unless defined?(super)
+
+        super
+
+        ::Geo::RepositoryRenamedEventStore.new(
+          project,
+          old_path: project.path,
+          old_path_with_namespace: @old_path
+        ).create
+      end
+    end
+  end
+end

app/services/geo/event_store.rb

+module Geo
+  # Base class for event store classes.
+  #
+  # Each store should also specify its event type by calling
+  # `self.event_type = ...` in the body of the class. The value of
+  # this method should be a symbol such as `:repository_updated_event`
+  # or `:repository_deleted_event`. For example:
+  #
+  #     class RepositoryUpdatedEventStore < EventStore
+  #       self.event_type = :repository_updated_event
+  #     end
+  #
+  # The event type is used to determine which attribute we should set
+  # on an instance of the Geo::EventLog class.
+  #
+  # Event store classes should implement the instance method `build_event`.
+  # The `build_event` method is supposed to return an instance of the event
+  # that will be logged.
+  class EventStore
+    class << self
+      attr_accessor :event_type
+    end
+
+    attr_reader :project, :params
+
+    def initialize(project, params = {})
+      @project = project
+      @params  = params
+    end
+
+    def create
+      return unless Gitlab::Geo.primary?
+
+      Geo::EventLog.create!("#{self.class.event_type}" => build_event)
+    rescue ActiveRecord::RecordInvalid, NoMethodError => e
+      log("#{self.event_type.to_s.humanize} could not be created", e)
+    end
+
+    private
+
+    def build_event
+      raise NotImplementedError,
+        "#{self.class} does not implement #{__method__}"
+    end
+
+    def log(message, error)
+      Rails.logger.error("#{self.class.name}: #{message} for project #{project.path_with_namespace} (#{project.id}): #{error}")
+    end
+  end
+end

app/services/geo/repository_deleted_event_store.rb

 module Geo
-  class RepositoryDeletedEventStore
-    attr_reader :project, :repo_path, :wiki_path
+  class RepositoryDeletedEventStore < EventStore
+    self.event_type = :repository_deleted_event
 
-    def initialize(project, repo_path:, wiki_path:)
-      @project = project
-      @repo_path = repo_path
-      @wiki_path = wiki_path
-    end
-
-    def create
-      return unless Gitlab::Geo.primary?
+    private
 
-      Geo::EventLog.transaction do
-        event_log = Geo::EventLog.new
-        deleted_event = Geo::RepositoryDeletedEvent.new(
-          project: project,
-          repository_storage_name: project.repository.storage,
-          repository_storage_path: project.repository_storage_path,
-          deleted_path: repo_path,
-          deleted_wiki_path: wiki_path,
-          deleted_project_name: project.name)
-        event_log.repository_deleted_event = deleted_event
-        event_log.save
-      end
+    def build_event
+      Geo::RepositoryDeletedEvent.new(
+        project: project,
+        repository_storage_name: project.repository.storage,
+        repository_storage_path: project.repository_storage_path,
+        deleted_path: params.fetch(:repo_path),
+        deleted_wiki_path: params.fetch(:wiki_path),
+        deleted_project_name: project.name)
     end
   end
 end

app/services/geo/repository_renamed_event_store.rb

+module Geo
+  class RepositoryRenamedEventStore < EventStore
+    self.event_type = :repository_renamed_event
+
+    private
+
+    def build_event
+      Geo::RepositoryRenamedEvent.new(
+        project: project,
+        repository_storage_name: project.repository.storage,
+        repository_storage_path: project.repository_storage_path,
+        old_path_with_namespace: old_path_with_namespace,
+        new_path_with_namespace: project.full_path,
+        old_wiki_path_with_namespace: old_wiki_path_with_namespace,
+        new_wiki_path_with_namespace: new_wiki_path_with_namespace,
+        old_path: params.fetch(:old_path),
+        new_path: project.path
+      )
+    end
+
+    def old_path_with_namespace
+      params.fetch(:old_path_with_namespace)
+    end
+
+    def old_wiki_path_with_namespace
+      "#{old_path_with_namespace}.wiki"
+    end
+
+    def new_wiki_path_with_namespace
+      project.wiki.path_with_namespace
+    end
+  end
+end

app/services/geo/repository_updated_event_store.rb

 module Geo
-  class RepositoryUpdatedEventStore
-    attr_reader :project, :source, :refs, :changes
-
-    def initialize(project, refs: [], changes: [], source: Geo::RepositoryUpdatedEvent::REPOSITORY)
-      @project = project
-      @refs    = refs
-      @changes = changes
-      @source  = source
-    end
-
-    def create
-      return unless Gitlab::Geo.primary?
-
-      Geo::EventLog.transaction do
-        event_log = Geo::EventLog.new
-        event_log.repository_updated_event = build_event
-        event_log.save!
-      end
-    rescue ActiveRecord::RecordInvalid
-      log("#{Geo::PushEvent.sources.key(source).humanize} updated event could not be created")
-    end
+  class RepositoryUpdatedEventStore < EventStore
+    self.event_type = :repository_updated_event
 
     private
 
@@ -35,6 +16,18 @@ module Geo
       )
     end
 
+    def refs
+      params.fetch(:refs, [])
+    end
+
+    def changes
+      params.fetch(:changes, [])
+    end
+
+    def source
+      params.fetch(:source, Geo::RepositoryUpdatedEvent::REPOSITORY)
+    end
+
     def ref
       refs.first if refs.length == 1
     end
@@ -54,9 +47,5 @@ module Geo
     def push_remove_branch?
       changes.any? { |change| Gitlab::Git.branch_ref?(change[:ref]) && Gitlab::Git.blank_ref?(change[:after]) }
     end
-
-    def log(message)
-      Rails.logger.info("#{self.class.name}: #{message} for project #{project.path_with_namespace} (#{project.id})")
-    end
   end
 end

app/services/projects/transfer_service.rb

@@ -11,6 +11,8 @@ module Projects
     include Gitlab::ShellAdapter
     TransferError = Class.new(StandardError)
 
+    prepend ::EE::Projects::TransferService
+
     def execute(new_namespace)
       @new_namespace = new_namespace
 

changelogs/unreleased-ee/da-geo-log-namespace-project-rename-events.yml

+---
+title: Add Geo repository renamed event log
+merge_request:
+author:

db/migrate/20170623195236_create_geo_repository_renamed_events.rb

+class CreateGeoRepositoryRenamedEvents < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :geo_repository_renamed_events, id: :bigserial do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
+      t.text :repository_storage_name, null: false
+      t.text :repository_storage_path, null: false
+      t.text :old_path_with_namespace, null: false
+      t.text :new_path_with_namespace, null: false
+      t.text :old_wiki_path_with_namespace, null: false
+      t.text :new_wiki_path_with_namespace, null: false
+      t.text :old_path, null: false
+      t.text :new_path, null: false
+    end
+
+    add_column :geo_event_log, :repository_renamed_event_id, :integer, limit: 8
+  end
+end

db/migrate/20170623202610_add_geo_repository_renamed_events_foreign_key.rb

+class AddGeoRepositoryRenamedEventsForeignKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :geo_event_log, :geo_repository_renamed_events,
+                               column: :repository_renamed_event_id, on_delete: :cascade
+  end
+
+  def down
+    remove_foreign_key :geo_event_log, column: :repository_renamed_event_id
+  end
+end

db/schema.rb

@@ -588,6 +588,7 @@ ActiveRecord::Schema.define(version: 20170627211700) do
     t.datetime "created_at", null: false
     t.integer "repository_updated_event_id", limit: 8
     t.integer "repository_deleted_event_id", limit: 8
+    t.integer "repository_renamed_event_id", limit: 8
   end
 
   add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
@@ -625,6 +626,20 @@ ActiveRecord::Schema.define(version: 20170627211700) do
 
   add_index "geo_repository_deleted_events", ["project_id"], name: "index_geo_repository_deleted_events_on_project_id", using: :btree
 
+  create_table "geo_repository_renamed_events", id: :bigserial, force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.text "repository_storage_name", null: false
+    t.text "repository_storage_path", null: false
+    t.text "old_path_with_namespace", null: false
+    t.text "new_path_with_namespace", null: false
+    t.text "old_wiki_path_with_namespace", null: false
+    t.text "new_wiki_path_with_namespace", null: false
+    t.text "old_path", null: false
+    t.text "new_path", null: false
+  end
+
+  add_index "geo_repository_renamed_events", ["project_id"], name: "index_geo_repository_renamed_events_on_project_id", using: :btree
+
   create_table "geo_repository_updated_events", id: :bigserial, force: :cascade do |t|
     t.datetime "created_at", null: false
     t.integer "branches_affected", null: false
@@ -1862,7 +1877,9 @@ ActiveRecord::Schema.define(version: 20170627211700) do
   add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
   add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
+  add_foreign_key "geo_event_log", "geo_repository_renamed_events", column: "repository_renamed_event_id", name: "fk_86c84214ec", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
+  add_foreign_key "geo_repository_renamed_events", "projects", on_delete: :cascade
   add_foreign_key "geo_repository_updated_events", "projects", on_delete: :cascade
   add_foreign_key "index_statuses", "projects", name: "fk_74b2492545", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade

spec/models/ee/namespace_spec.rb

@@ -42,6 +42,34 @@ describe Namespace, models: true do
     end
   end
 
+  describe '#move_dir' do
+    context 'when running on a primary node' do
+      let!(:geo_node) { create(:geo_node, :primary, :current) }
+      let(:gitlab_shell) { Gitlab::Shell.new }
+
+      it 'logs the Geo::RepositoryRenamedEvent for each project inside namespace' do
+        parent = create(:namespace)
+        child = create(:group, name: 'child', path: 'child', parent: parent)
+        project_1 = create(:project_empty_repo, namespace: parent)
+        create(:project_empty_repo, namespace: child)
+        full_path_was = "#{parent.full_path}_old"
+        new_path = parent.full_path
+
+        allow(parent).to receive(:gitlab_shell).and_return(gitlab_shell)
+        allow(parent).to receive(:path_changed?).and_return(true)
+        allow(parent).to receive(:full_path_was).and_return(full_path_was)
+        allow(parent).to receive(:full_path).and_return(new_path)
+
+        allow(gitlab_shell).to receive(:mv_namespace)
+          .ordered
+          .with(project_1.repository_storage_path, full_path_was, new_path)
+          .and_return(true)
+
+        expect { parent.move_dir }.to change(Geo::RepositoryRenamedEvent, :count).by(2)
+      end
+    end
+  end
+
   describe '#feature_available?' do
     let(:plan_license) { Namespace::BRONZE_PLAN }
     let(:group) { create(:group, plan: plan_license) }

spec/models/ee/project_spec.rb

@@ -604,4 +604,37 @@ describe Project, models: true do
       end
     end
   end
+
+  describe '#rename_repo' do
+    context 'when running on a primary node' do
+      let!(:geo_node) { create(:geo_node, :primary, :current) }
+      let(:project) { create(:project, :repository) }
+      let(:gitlab_shell) { Gitlab::Shell.new }
+
+      before do
+        allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
+        allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
+      end
+
+      it 'logs the Geo::RepositoryRenamedEvent' do
+        stub_container_registry_config(enabled: false)
+
+        allow(gitlab_shell).to receive(:mv_repository)
+          .ordered
+          .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
+          .and_return(true)
+
+        allow(gitlab_shell).to receive(:mv_repository)
+          .ordered
+          .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
+          .and_return(true)
+
+        expect(Geo::RepositoryRenamedEventStore).to receive(:new)
+          .with(instance_of(Project), old_path: 'foo', old_path_with_namespace: "#{project.namespace.full_path}/foo")
+          .and_call_original
+
+        expect { project.rename_repo }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
+      end
+    end
+  end
 end

spec/models/geo/event_log_spec.rb

@@ -3,5 +3,6 @@ require 'spec_helper'
 RSpec.describe Geo::EventLog, type: :model do
   describe 'relationships' do
     it { is_expected.to belong_to(:repository_updated_event).class_name('Geo::RepositoryUpdatedEvent').with_foreign_key('repository_updated_event_id') }
+    it { is_expected.to belong_to(:repository_renamed_event).class_name('Geo::RepositoryRenamedEvent').with_foreign_key('repository_renamed_event_id') }
   end
 end

spec/models/geo/repository_renamed_event_spec.rb

+require 'spec_helper'
+
+RSpec.describe Geo::RepositoryRenamedEvent, type: :model do
+  describe 'relationships' do
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:repository_storage_name) }
+    it { is_expected.to validate_presence_of(:repository_storage_path) }
+    it { is_expected.to validate_presence_of(:old_path_with_namespace) }
+    it { is_expected.to validate_presence_of(:new_path_with_namespace) }
+    it { is_expected.to validate_presence_of(:old_wiki_path_with_namespace) }
+    it { is_expected.to validate_presence_of(:new_wiki_path_with_namespace) }
+    it { is_expected.to validate_presence_of(:old_path) }
+    it { is_expected.to validate_presence_of(:new_path) }
+  end
+end

spec/services/ee/projects/destroy_service_spec.rb

@@ -10,32 +10,22 @@ describe Projects::DestroyService, services: true do
   let!(:wiki_path) { project.path_with_namespace + '.wiki' }
   let!(:storage_name) { project.repository_storage }
   let!(:storage_path) { project.repository_storage_path }
-  let!(:geo_node) { create(:geo_node, :primary, :current) }
+
+  subject { described_class.new(project, user, {}) }
 
   before do
     stub_container_registry_config(enabled: true)
     stub_container_registry_tags(repository: :any, tags: [])
   end
 
-  context 'Geo primary' do
-    it 'logs the event' do
-      # Run sidekiq immediatly to check that renamed repository will be removed
-      Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
-
-      event = Geo::RepositoryDeletedEvent.first
+  context 'when running on a primary node' do
+    let!(:geo_node) { create(:geo_node, :primary, :current) }
 
-      expect(Geo::EventLog.count).to eq(1)
-      expect(Geo::RepositoryDeletedEvent.count).to eq(1)
-      expect(event.project_id).to eq(project_id)
-      expect(event.deleted_path).to eq(project_path)
-      expect(event.deleted_wiki_path).to eq(wiki_path)
-      expect(event.deleted_project_name).to eq(project_name)
-      expect(event.repository_storage_name).to eq(storage_name)
-      expect(event.repository_storage_path).to eq(storage_path)
+    it 'logs an event to the Geo event log' do
+      # Run Sidekiq immediately to check that renamed repository will be removed
+      Sidekiq::Testing.inline! do
+        expect { subject.execute }.to change(Geo::RepositoryDeletedEvent, :count).by(1)
+      end
     end
   end
-
-  def destroy_project(project, user, params = {})
-    described_class.new(project, user, params).execute
-  end
 end

spec/services/ee/projects/transfer_service_spec.rb

+# rubocop:disable RSpec/FilePath
+require 'spec_helper'
+
+describe Projects::TransferService, services: true do
+  let(:user) { create(:user) }
+  let(:group) { create(:group) }
+  let(:project) { create(:project, :repository, namespace: user.namespace) }
+
+  subject { described_class.new(project, user) }
+
+  before do
+    group.add_owner(user)
+  end
+
+  context 'when running on a primary node' do
+    let!(:geo_node) { create(:geo_node, :primary, :current) }
+
+    it 'logs an event to the Geo event log' do
+      expect { subject.execute(group) }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
+    end
+  end
+end

spec/services/geo/repository_deleted_event_store_spec.rb

+require 'spec_helper'
+
+describe Geo::RepositoryDeletedEventStore, services: true do
+  let(:project) { create(:empty_project, path: 'bar') }
+  let!(:project_id) { project.id }
+  let!(:project_name) { project.name }
+  let!(:repo_path) { project.full_path }
+  let!(:wiki_path) { "#{project.full_path}.wiki" }
+  let!(:storage_name) { project.repository_storage }
+  let!(:storage_path) { project.repository_storage_path }
+
+  subject { described_class.new(project, repo_path: repo_path, wiki_path: wiki_path) }
+
+  describe '#create' do
+    it 'does not create an event when not running on a primary node' do
+      allow(Gitlab::Geo).to receive(:primary?) { false }
+
+      expect { subject.create }.not_to change(Geo::RepositoryDeletedEvent, :count)
+    end
+
+    context 'when running on a primary node' do
+      before do
+        allow(Gitlab::Geo).to receive(:primary?) { true }
+      end
+
+      it 'creates a deleted event' do
+        expect { subject.create }.to change(Geo::RepositoryDeletedEvent, :count).by(1)
+      end
+
+      it 'tracks information for the deleted project' do
+        subject.create
+
+        event = Geo::RepositoryDeletedEvent.last
+
+        expect(event.project_id).to eq(project_id)
+        expect(event.deleted_path).to eq(repo_path)
+        expect(event.deleted_wiki_path).to eq(wiki_path)
+        expect(event.deleted_project_name).to eq(project_name)
+        expect(event.repository_storage_name).to eq(storage_name)
+        expect(event.repository_storage_path).to eq(storage_path)
+      end
+    end
+  end
+end

spec/services/geo/repository_renamed_event_store_spec.rb

+require 'spec_helper'
+
+describe Geo::RepositoryRenamedEventStore, services: true do
+  let(:project) { create(:empty_project, path: 'bar') }
+  let(:old_path) { 'foo' }
+  let(:old_path_with_namespace) { "#{project.namespace.full_path}/foo" }
+
+  subject { described_class.new(project, old_path: old_path, old_path_with_namespace: old_path_with_namespace) }
+
+  describe '#create' do
+    it 'does not create an event when not running on a primary node' do
+      allow(Gitlab::Geo).to receive(:primary?) { false }
+
+      expect { subject.create }.not_to change(Geo::RepositoryRenamedEvent, :count)
+    end
+
+    context 'when running on a primary node' do
+      before do
+        allow(Gitlab::Geo).to receive(:primary?) { true }
+      end
+
+      it 'creates a renamed event' do
+        expect { subject.create }.to change(Geo::RepositoryRenamedEvent, :count).by(1)
+      end
+
+      it 'tracks old and new paths for project repositories' do
+        subject.create
+
+        event = Geo::RepositoryRenamedEvent.last
+
+        expect(event.repository_storage_name).to eq(project.repository_storage)
+        expect(event.repository_storage_path).to eq(project.repository_storage_path)
+        expect(event.old_path_with_namespace).to eq(old_path_with_namespace)
+        expect(event.new_path_with_namespace).to eq(project.full_path)
+        expect(event.old_wiki_path_with_namespace).to eq("#{old_path_with_namespace}.wiki")
+        expect(event.new_wiki_path_with_namespace).to eq("#{project.full_path}.wiki")
+        expect(event.old_path).to eq(old_path)
+        expect(event.new_path).to eq(project.path)
+      end
+    end
+  end
+end

spec/services/projects/transfer_service_spec.rb

@@ -37,7 +37,7 @@ describe Projects::TransferService, services: true do
     end
 
     it 'executes system hooks' do
-      expect_any_instance_of(Projects::TransferService).to receive(:execute_system_hooks)
+      expect_any_instance_of(SystemHooksService).to receive(:execute_hooks_for).with(project, :transfer)
 
       transfer_project(project, user, group)
     end
@@ -80,7 +80,7 @@ describe Projects::TransferService, services: true do
     end
 
     it "doesn't run system hooks" do
-      expect_any_instance_of(Projects::TransferService).not_to receive(:execute_system_hooks)
+      expect_any_instance_of(SystemHooksService).not_to receive(:execute_hooks_for).with(project, :transfer)
 
       attempt_project_transfer
     end