Commit 09d7cbd0 authored by Kamil Trzciński's avatar Kamil Trzciński

Update `QueryAnalyzers::PreventCrossDatabaseModification` to use context

Remove all duplicate processing logic and make it fit well into `rspec`.
This requires adding `allow_cross_database_modification` flag.
parent deaa7c85
...@@ -6,90 +6,77 @@ module Gitlab ...@@ -6,90 +6,77 @@ module Gitlab
class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base
CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError) CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError)
# This method will allow cross database modifications within the block def self.allow_cross_database_modification?
# Example: Thread.current[:prevent_cross_database_modification_allowed]
# end
# allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do
# create(:build) # inserts ci_build and project record in one transaction
# end
def self.allow_cross_database_modification_within_transaction(url:)
cross_database_context = self.cross_database_context
return yield unless cross_database_context && cross_database_context[:enabled]
transaction_tracker_enabled_was = cross_database_context[:enabled]
cross_database_context[:enabled] = false
yield def self.allow_cross_database_modification=(value)
ensure Thread.current[:prevent_cross_database_modification_allowed] = value
cross_database_context[:enabled] = transaction_tracker_enabled_was if cross_database_context
end end
def self.with_cross_database_modification_prevented(log_only: false) def self.with_allow_cross_database_modification(value, &blk)
reset_cross_database_context! previous = self.allow_cross_database_modification?
cross_database_context.merge!(enabled: true, log_only: log_only) self.allow_cross_database_modification = value
yield if block_given? yield
ensure ensure
cleanup_with_cross_database_modification_prevented if block_given? self.allow_cross_database_modification = previous
end end
def self.cleanup_with_cross_database_modification_prevented # This method will allow cross database modifications within the block
if cross_database_context # Example:
cross_database_context[:enabled] = false #
end # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do
# create(:build) # inserts ci_build and project record in one transaction
# end
def self.allow_cross_database_modification_within_transaction(url:, &blk)
self.with_allow_cross_database_modification(true, &blk)
end end
def self.cross_database_context # This method will prevent cross database modifications within the block
Thread.current[:transaction_tracker] # if it was allowed previously
def self.with_cross_database_modification_prevented(&blk)
self.with_allow_cross_database_modification(false, &blk)
end end
def self.reset_cross_database_context! def self.begin!
Thread.current[:transaction_tracker] = initial_data super
end
def self.initial_data context.merge!({
{
enabled: false,
transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 },
modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }, modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }
log_only: false })
}
end end
def self.enabled? def self.enabled?
true ::Feature::FlipperFeature.table_exists? &&
Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml)
end end
# rubocop:disable Metrics/AbcSize # rubocop:disable Metrics/AbcSize
def self.analyze(parsed) def self.analyze(parsed)
return false unless cross_database_context return if self.allow_cross_database_modification?
return false unless cross_database_context[:enabled]
connection = parsed.connection
return false if connection.pool.instance_of?(ActiveRecord::ConnectionAdapters::NullPool)
return if in_factory_bot_create? return if in_factory_bot_create?
database = connection.pool.db_config.name database = ::Gitlab::Database.db_config_name(parsed.connection)
sql = parsed.sql sql = parsed.sql
# We ignore BEGIN in tests as this is the outer transaction for # We ignore BEGIN in tests as this is the outer transaction for
# DatabaseCleaner # DatabaseCleaner
if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN')) if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN'))
cross_database_context[:transaction_depth_by_db][database] += 1 context[:transaction_depth_by_db][database] += 1
return return
elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT')) elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT'))
cross_database_context[:transaction_depth_by_db][database] -= 1 context[:transaction_depth_by_db][database] -= 1
if cross_database_context[:transaction_depth_by_db][database] <= 0 if context[:transaction_depth_by_db][database] <= 0
cross_database_context[:modified_tables_by_db][database].clear context[:modified_tables_by_db][database].clear
end end
return return
end end
return if cross_database_context[:transaction_depth_by_db].values.all?(&:zero?) return if context[:transaction_depth_by_db].values.all?(&:zero?)
# PgQuery might fail in some cases due to limited nesting: # PgQuery might fail in some cases due to limited nesting:
# https://github.com/pganalyze/pg_query/issues/209 # https://github.com/pganalyze/pg_query/issues/209
...@@ -107,8 +94,8 @@ module Gitlab ...@@ -107,8 +94,8 @@ module Gitlab
# databases # databases
return if tables == ['schema_migrations'] return if tables == ['schema_migrations']
cross_database_context[:modified_tables_by_db][database].merge(tables) context[:modified_tables_by_db][database].merge(tables)
all_tables = cross_database_context[:modified_tables_by_db].values.map(&:to_a).flatten all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten
schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables)
if schemas.many? if schemas.many?
...@@ -120,13 +107,11 @@ module Gitlab ...@@ -120,13 +107,11 @@ module Gitlab
message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ." message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ."
end end
begin raise CrossDatabaseModificationAcrossUnsupportedTablesError, message
raise CrossDatabaseModificationAcrossUnsupportedTablesError, message
rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e
::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql })
raise unless cross_database_context[:log_only]
end
end end
rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e
::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql })
raise if raise_exception?
rescue StandardError => e rescue StandardError => e
# Extra safety net to ensure we never raise in production # Extra safety net to ensure we never raise in production
# if something goes wrong in this logic # if something goes wrong in this logic
...@@ -142,6 +127,11 @@ module Gitlab ...@@ -142,6 +127,11 @@ module Gitlab
def self.in_factory_bot_create? def self.in_factory_bot_create?
Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' } Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' }
end end
# When in test we raise exception
def self.raise_exception?
Rails.env.test?
end
end end
end end
end end
......
...@@ -2,10 +2,18 @@ ...@@ -2,10 +2,18 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification do RSpec.describe Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification, query_analyzers: false do
let_it_be(:pipeline, refind: true) { create(:ci_pipeline) } let_it_be(:pipeline, refind: true) { create(:ci_pipeline) }
let_it_be(:project, refind: true) { create(:project) } let_it_be(:project, refind: true) { create(:project) }
before do
allow(Gitlab::Database::QueryAnalyzer.instance).to receive(:all_analyzers).and_return([described_class])
end
around do |example|
Gitlab::Database::QueryAnalyzer.instance.within { example.run }
end
shared_examples 'successful examples' do shared_examples 'successful examples' do
context 'outside transaction' do context 'outside transaction' do
it { expect { run_queries }.not_to raise_error } it { expect { run_queries }.not_to raise_error }
......
...@@ -21,10 +21,11 @@ RSpec.describe Gitlab::Middleware::QueryAnalyzer, query_analyzers: false do ...@@ -21,10 +21,11 @@ RSpec.describe Gitlab::Middleware::QueryAnalyzer, query_analyzers: false do
end end
end end
it 'detects cross modifications and logs them' do it 'detects cross modifications and logs them without raising exception' do
allow(Rails.env).to receive(:test?).and_return(false)
expect(::Gitlab::ErrorTracking).to receive(:track_exception) expect(::Gitlab::ErrorTracking).to receive(:track_exception)
subject expect { subject }.not_to raise_error
end end
context 'when the detect_cross_database_modification is disabled' do context 'when the detect_cross_database_modification is disabled' do
......
...@@ -23,10 +23,11 @@ RSpec.describe Gitlab::SidekiqMiddleware::QueryAnalyzer, query_analyzers: false ...@@ -23,10 +23,11 @@ RSpec.describe Gitlab::SidekiqMiddleware::QueryAnalyzer, query_analyzers: false
end end
end end
it 'detects cross modifications and logs them' do it 'detects cross modifications and logs them without raising exception' do
allow(Rails.env).to receive(:test?).and_return(false)
expect(::Gitlab::ErrorTracking).to receive(:track_exception) expect(::Gitlab::ErrorTracking).to receive(:track_exception)
subject expect { subject }.not_to raise_error
end end
context 'when the detect_cross_database_modification is disabled' do context 'when the detect_cross_database_modification is disabled' do
......
# frozen_string_literal: true # frozen_string_literal: true
module PreventCrossDatabaseModificationSpecHelpers module PreventCrossDatabaseModificationSpecHelpers
def with_cross_database_modification_prevented(...) delegate :with_cross_database_modification_prevented,
::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.with_cross_database_modification_prevented(...) :allow_cross_database_modification_within_transaction,
end to: :'::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification'
def cleanup_with_cross_database_modification_prevented
::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.cleanup_with_cross_database_modification_prevented
end
end end
CROSS_DB_MODIFICATION_ALLOW_LIST = Set.new(YAML.load_file(File.join(__dir__, 'cross-database-modification-allowlist.yml'))).freeze CROSS_DB_MODIFICATION_ALLOW_LIST = Set.new(YAML.load_file(File.join(__dir__, 'cross-database-modification-allowlist.yml'))).freeze
...@@ -18,12 +14,11 @@ RSpec.configure do |config| ...@@ -18,12 +14,11 @@ RSpec.configure do |config|
# Using before and after blocks because the around block causes problems with the let_it_be # Using before and after blocks because the around block causes problems with the let_it_be
# record creations. It makes an extra savepoint which breaks the transaction count logic. # record creations. It makes an extra savepoint which breaks the transaction count logic.
config.before do |example_file| config.before do |example_file|
if CROSS_DB_MODIFICATION_ALLOW_LIST.exclude?(example_file.file_path_rerun_argument) ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification =
with_cross_database_modification_prevented CROSS_DB_MODIFICATION_ALLOW_LIST.include?(example_file.file_path_rerun_argument)
end
end end
config.after do |example_file| config.after do |example_file|
cleanup_with_cross_database_modification_prevented ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification = false
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment