Merge branch 'project-topics-data-migration' into 'master'

Data migration ('tags' to 'topics') for projects [RUN ALL RSPEC]

See merge request gitlab-org/gitlab!61237

app/models/project.rb

@@ -128,8 +128,41 @@ class Project < ApplicationRecord
   after_initialize :use_hashed_storage
   after_create :check_repository_absence!
 
-  acts_as_ordered_taggable
-  alias_method :topics, :tag_list
+  acts_as_ordered_taggable_on :topics
+  # The 'tag_list' alias and the 'has_many' associations are required during the 'tags -> topics' migration
+  # TODO: eliminate 'tag_list', 'topic_taggings' and 'tags' in the further process of the migration
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/331081
+  alias_attribute :tag_list, :topic_list
+  has_many :topic_taggings, -> { includes(:tag).order("#{ActsAsTaggableOn::Tagging.table_name}.id") },
+                     as: :taggable,
+                     class_name: 'ActsAsTaggableOn::Tagging',
+                     after_add: :dirtify_tag_list,
+                     after_remove: :dirtify_tag_list
+  has_many :topics, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") },
+                     class_name: 'ActsAsTaggableOn::Tag',
+                     through: :topic_taggings,
+                     source: :tag
+  has_many :tags, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") },
+                     class_name: 'ActsAsTaggableOn::Tag',
+                     through: :topic_taggings,
+                     source: :tag
+
+  # Overwriting 'topic_list' and 'topic_list=' is necessary to ensure functionality during the background migration [1].
+  # [1] https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61237
+  # TODO: remove 'topic_list' and 'topic_list=' once the background migration is complete
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/331081
+  def topic_list
+    # Return both old topics (context 'tags') and new topics (context 'topics')
+    tag_list_on('tags') + tag_list_on('topics')
+  end
+
+  def topic_list=(new_tags)
+    # Old topics with context 'tags' are added as new topics with context 'topics'
+    super(new_tags)
+
+    # Remove old topics with context 'tags'
+    set_tag_list_on('tags', '')
+  end
 
   attr_accessor :old_path_with_namespace
   attr_accessor :template_name

app/presenters/project_presenter.rb

@@ -401,16 +401,16 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated
   end
 
   def topics_to_show
-    project.topics.take(MAX_TOPICS_TO_SHOW) # rubocop: disable CodeReuse/ActiveRecord
+    project.topic_list.take(MAX_TOPICS_TO_SHOW) # rubocop: disable CodeReuse/ActiveRecord
   end
 
   def topics_not_shown
-    project.topics - topics_to_show
+    project.topic_list - topics_to_show
   end
 
   def count_of_extra_topics_not_shown
-    if project.topics.count > MAX_TOPICS_TO_SHOW
-      project.topics.count - MAX_TOPICS_TO_SHOW
+    if project.topic_list.count > MAX_TOPICS_TO_SHOW
+      project.topic_list.count - MAX_TOPICS_TO_SHOW
     else
       0
     end

changelogs/unreleased/project-topics-data-migration.yml

+---
+title: Migrate 'tags' to 'topics' for project in the database context
+merge_request: 61237
+author: Jonas Wälter @wwwjon
+type: changed

db/post_migrate/20210511095657_add_temporary_index_for_project_topics_to_taggings.rb

+# frozen_string_literal: true
+
+class AddTemporaryIndexForProjectTopicsToTaggings < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  INDEX_NAME = 'tmp_index_taggings_on_id_where_taggable_type_project_and_tags'
+  INDEX_CONDITION = "taggable_type = 'Project' AND context = 'tags'"
+
+  disable_ddl_transaction!
+
+  def up
+    # this index is used in 20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics
+    add_concurrent_index :taggings, :id, where: INDEX_CONDITION, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :taggings, INDEX_NAME
+  end
+end

db/post_migrate/20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics.rb

+# frozen_string_literal: true
+
+class ScheduleMigrateProjectTaggingsContextFromTagsToTopics < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 30_000
+  DELAY_INTERVAL = 2.minutes
+  MIGRATION = 'MigrateProjectTaggingsContextFromTagsToTopics'
+
+  disable_ddl_transaction!
+
+  class Tagging < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'taggings'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      Tagging.where(taggable_type: 'Project', context: 'tags'),
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+  end
+end

db/post_migrate/20210517075444_remove_temporary_index_for_project_topics_to_taggings.rb

+# frozen_string_literal: true
+
+class RemoveTemporaryIndexForProjectTopicsToTaggings < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  INDEX_NAME = 'tmp_index_taggings_on_id_where_taggable_type_project_and_tags'
+  INDEX_CONDITION = "taggable_type = 'Project' AND context = 'tags'"
+
+  disable_ddl_transaction!
+
+  def up
+    # this index was used in 20210511095658_schedule_migrate_project_taggings_context_from_tags_to_topics
+    remove_concurrent_index_by_name :taggings, INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :taggings, :id, where: INDEX_CONDITION, name: INDEX_NAME
+  end
+end

db/schema_migrations/20210511095657

+4d11cdf876786db5e827ea1a50b70e2d5b3814fd7c0b0c083ab61adad9685364
\ No newline at end of file

db/schema_migrations/20210511095658

+7387c23bbbc376e26c057179ebe2796be183462acb1fc509d451f0fede13ed93
\ No newline at end of file

db/schema_migrations/20210517075444

+ec08c18ac37f2ae7298650df58345755eada20aaa5b7ed3dfd54ee5cea88ebdd
\ No newline at end of file

lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # The class to migrate the context of project taggings from `tags` to `topics`
+    class MigrateProjectTaggingsContextFromTagsToTopics
+      # Temporary AR table for taggings
+      class Tagging < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'taggings'
+      end
+
+      def perform(start_id, stop_id)
+        Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation|
+          relation.update_all(context: 'topics')
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/import_export.yml

@@ -153,6 +153,7 @@ excluded_attributes:
     - :bfg_object_map
     - :detected_repository_languages
     - :tag_list
+    - :topic_list
     - :mirror_user_id
     - :mirror_trigger_builds
     - :only_mirror_protected_branches

spec/finders/projects_finder_spec.rb

@@ -139,7 +139,7 @@ RSpec.describe ProjectsFinder do
 
       describe 'filter by tags' do
         before do
-          public_project.tag_list.add('foo')
+          public_project.tag_list = 'foo'
           public_project.save!
         end
 

spec/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::MigrateProjectTaggingsContextFromTagsToTopics, schema: 20210511095658 do
+  it 'correctly migrates project taggings context from tags to topics' do
+    taggings = table(:taggings)
+
+    project_old_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'tags')
+    project_new_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'topics')
+    project_other_context_tagging_1 = taggings.create!(taggable_type: 'Project', context: 'other')
+    project_old_tagging_2 = taggings.create!(taggable_type: 'Project', context: 'tags')
+    project_old_tagging_3 = taggings.create!(taggable_type: 'Project', context: 'tags')
+
+    subject.perform(project_old_tagging_1.id, project_old_tagging_2.id)
+
+    project_old_tagging_1.reload
+    project_new_tagging_1.reload
+    project_other_context_tagging_1.reload
+    project_old_tagging_2.reload
+    project_old_tagging_3.reload
+
+    expect(project_old_tagging_1.context).to eq('topics')
+    expect(project_new_tagging_1.context).to eq('topics')
+    expect(project_other_context_tagging_1.context).to eq('other')
+    expect(project_old_tagging_2.context).to eq('topics')
+    expect(project_old_tagging_3.context).to eq('tags')
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -343,8 +343,9 @@ project:
 - external_approval_rules
 - taggings
 - base_tags
-- tag_taggings
 - tags
+- topic_taggings
+- topics
 - chat_services
 - cluster
 - clusters

spec/models/project_spec.rb

@@ -6964,6 +6964,55 @@ RSpec.describe Project, factory_default: :keep do
     end
   end
 
+  describe 'topics' do
+    let_it_be(:project) { create(:project, tag_list: 'topic1, topic2, topic3') }
+
+    it 'topic_list returns correct string array' do
+      expect(project.topic_list).to match_array(%w[topic1 topic2 topic3])
+    end
+
+    it 'topics returns correct tag records' do
+      expect(project.topics.first.class.name).to eq('ActsAsTaggableOn::Tag')
+      expect(project.topics.map(&:name)).to match_array(%w[topic1 topic2 topic3])
+    end
+
+    context 'aliases' do
+      it 'tag_list returns correct string array' do
+        expect(project.tag_list).to match_array(%w[topic1 topic2 topic3])
+      end
+
+      it 'tags returns correct tag records' do
+        expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag')
+        expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3])
+      end
+    end
+
+    context 'intermediate state during background migration' do
+      before do
+        project.taggings.first.update!(context: 'tags')
+        project.instance_variable_set("@tag_list", nil)
+        project.reload
+      end
+
+      it 'tag_list returns string array including old and new topics' do
+        expect(project.tag_list).to match_array(%w[topic1 topic2 topic3])
+      end
+
+      it 'tags returns old and new tag records' do
+        expect(project.tags.first.class.name).to eq('ActsAsTaggableOn::Tag')
+        expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3])
+        expect(project.taggings.map(&:context)).to match_array(%w[tags topics topics])
+      end
+
+      it 'update tag_list adds new topics and removes old topics' do
+        project.update!(tag_list: 'topic1, topic2, topic3, topic4')
+
+        expect(project.tags.map(&:name)).to match_array(%w[topic1 topic2 topic3 topic4])
+        expect(project.taggings.map(&:context)).to match_array(%w[topics topics topics topics])
+      end
+    end
+  end
+
   def finish_job(export_job)
     export_job.start
     export_job.finish

spec/requests/api/project_attributes.yml

@@ -41,6 +41,7 @@ itself: # project
     - reset_approvals_on_push
     - runners_token_encrypted
     - storage_version
+    - topic_list
     - updated_at
   remapped_attributes:
     avatar: avatar_url
@@ -67,6 +68,7 @@ itself: # project
     - readme_url
     - shared_with_groups
     - ssh_url_to_repo
+    - tag_list
     - web_url
 
 build_auto_devops: # auto_devops