Implement feature flag to rollout decomposed primary CI database

Per our rollout plan https://gitlab.com/groups/gitlab-org/-/epics/6160
of separating the CI database from the main database the "phase 4" will
involve gradually changing a percentage of "read-write" (primary)
traffic to go through a new connection that still points to the same
database.

Previously we added the `GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci`
environment variable to allow us to share the same primary connection
when there is a separate `ci` entry in `config/database.yml`. But we
need to a way to override this environment variable so this MR
introduces a new feature flag `force_no_sharing_primary_model` which
overrides the "reuse" connection and forces it to not reuse the
connection.

config/feature_flags/development/force_no_sharing_primary_model.yml

+---
+name: force_no_sharing_primary_model
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76188
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/347286
+milestone: '14.8'
+type: development
+group: group::sharding
+default_enabled: false

lib/gitlab/database/load_balancing/configuration.rb

@@ -74,11 +74,24 @@ module Gitlab
         # With connection re-use the primary connection can be overwritten
         # to be used from different model
         def primary_connection_specification_name
-          (@primary_model || @model).connection_specification_name
+          primary_model_or_model_if_enabled.connection_specification_name
         end
 
-        def primary_db_config
-          (@primary_model || @model).connection_db_config
+        def primary_model_or_model_if_enabled
+          if force_no_sharing_primary_model?
+            @model
+          else
+            @primary_model || @model
+          end
+        end
+
+        def force_no_sharing_primary_model?
+          return false unless @primary_model # Doesn't matter since we don't have an overriding primary model
+          return false unless ::Gitlab::SafeRequestStore.active?
+
+          ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do
+            ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model, default_enabled: :yaml)
+          end
         end
 
         def replica_db_config

spec/lib/gitlab/database/load_balancing/configuration_spec.rb

@@ -2,11 +2,18 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::Database::LoadBalancing::Configuration do
+RSpec.describe Gitlab::Database::LoadBalancing::Configuration, :request_store do
   let(:configuration_hash) { {} }
   let(:db_config) { ActiveRecord::DatabaseConfigurations::HashConfig.new('test', 'ci', configuration_hash) }
   let(:model) { double(:model, connection_db_config: db_config) }
 
+  before do
+    # It's confusing to think about these specs with this enabled by default so
+    # we make it disabled by default and just write the specific spec for when
+    # it's enabled
+    stub_feature_flags(force_no_sharing_primary_model: false)
+  end
+
   describe '.for_model' do
     context 'when load balancing is not configured' do
       it 'uses the default settings' do
@@ -233,11 +240,23 @@ RSpec.describe Gitlab::Database::LoadBalancing::Configuration do
     end
 
     context 'when GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci=main' do
-      it 'the primary connection uses main connection' do
+      before do
         stub_env('GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci', 'main')
+      end
 
+      it 'the primary connection uses main connection' do
         expect(config.primary_connection_specification_name).to eq('ActiveRecord::Base')
       end
+
+      context 'when force_no_sharing_primary_model feature flag is enabled' do
+        before do
+          stub_feature_flags(force_no_sharing_primary_model: true)
+        end
+
+        it 'the primary connection uses ci connection' do
+          expect(config.primary_connection_specification_name).to eq('Ci::ApplicationRecord')
+        end
+      end
     end
 
     context 'when GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci=unknown' do

spec/lib/gitlab/database/load_balancing/setup_spec.rb

@@ -130,6 +130,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil,
           request_store_active: false,
           ff_use_model_load_balancing: nil,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'ci_replica', write: 'ci' }
@@ -140,6 +141,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main',
           request_store_active: false,
           ff_use_model_load_balancing: nil,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'ci_replica', write: 'main' }
@@ -150,6 +152,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil,
           request_store_active: false,
           ff_use_model_load_balancing: nil,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'main_replica', write: 'main' }
@@ -160,60 +163,77 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main',
           request_store_active: false,
           ff_use_model_load_balancing: nil,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'main_replica', write: 'main' }
           }
         },
-        "with FF disabled without RequestStore it uses main" => {
+        "with FF use_model_load_balancing disabled without RequestStore it uses main" => {
           env_GITLAB_USE_MODEL_LOAD_BALANCING: nil,
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil,
           request_store_active: false,
           ff_use_model_load_balancing: false,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'main_replica', write: 'main' }
           }
         },
-        "with FF enabled without RequestStore sticking of FF does not work, so it fallbacks to use main" => {
+        "with FF use_model_load_balancing enabled without RequestStore sticking of FF does not work, so it fallbacks to use main" => {
           env_GITLAB_USE_MODEL_LOAD_BALANCING: nil,
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil,
           request_store_active: false,
           ff_use_model_load_balancing: true,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'main_replica', write: 'main' }
           }
         },
-        "with FF disabled with RequestStore it uses main" => {
+        "with FF use_model_load_balancing disabled with RequestStore it uses main" => {
           env_GITLAB_USE_MODEL_LOAD_BALANCING: nil,
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil,
           request_store_active: true,
           ff_use_model_load_balancing: false,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'main_replica', write: 'main' }
           }
         },
-        "with FF enabled with RequestStore it sticks FF and uses CI connection" => {
+        "with FF use_model_load_balancing enabled with RequestStore it sticks FF and uses CI connection" => {
           env_GITLAB_USE_MODEL_LOAD_BALANCING: nil,
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: nil,
           request_store_active: true,
           ff_use_model_load_balancing: true,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'ci_replica', write: 'ci' }
           }
         },
-        "with re-use and FF enabled with RequestStore it sticks FF and uses CI connection for reads" => {
+        "with re-use and ff_use_model_load_balancing enabled and FF force_no_sharing_primary_model disabled with RequestStore it sticks FF and uses CI connection for reads" => {
           env_GITLAB_USE_MODEL_LOAD_BALANCING: nil,
           env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main',
           request_store_active: true,
           ff_use_model_load_balancing: true,
+          ff_force_no_sharing_primary_model: false,
           expectations: {
             main: { read: 'main_replica', write: 'main' },
             ci: { read: 'ci_replica', write: 'main' }
           }
+        },
+        "with re-use and ff_use_model_load_balancing enabled and FF force_no_sharing_primary_model enabled with RequestStore it sticks FF and uses CI connection for reads" => {
+          env_GITLAB_USE_MODEL_LOAD_BALANCING: nil,
+          env_GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci: 'main',
+          request_store_active: true,
+          ff_use_model_load_balancing: true,
+          ff_force_no_sharing_primary_model: true,
+          expectations: {
+            main: { read: 'main_replica', write: 'main' },
+            ci: { read: 'ci_replica', write: 'ci' }
+          }
         }
       }
     end
@@ -243,6 +263,7 @@ RSpec.describe Gitlab::Database::LoadBalancing::Setup do
       around do |example|
         if request_store_active
           Gitlab::WithRequestStore.with_request_store do
+            stub_feature_flags(force_no_sharing_primary_model: ff_force_no_sharing_primary_model)
             RequestStore.clear!
 
             example.run