Merge branch 'ab/partition-management' into 'master'

Create time-space partitions in separate schema

See merge request gitlab-org/gitlab!34504

changelogs/unreleased/ab-partition-management.yml

+---
+title: Create time-space partitions in separate schema
+merge_request: 34504
+author:
+type: other

config/initializers/active_record_schema_ignore_tables.rb

 # Ignore table used temporarily in background migration
 ActiveRecord::SchemaDumper.ignore_tables = ["untracked_files_for_uploads"]
+
+# Ignore dynamically managed partitions in static application schema
+ActiveRecord::SchemaDumper.ignore_tables += ["partitions_dynamic.*"]

db/migrate/20200615101135_create_dynamic_partitions_schema.rb

+# frozen_string_literal: true
+
+class CreateDynamicPartitionsSchema < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::SchemaHelpers
+
+  DOWNTIME = false
+
+  def up
+    execute 'CREATE SCHEMA partitions_dynamic'
+
+    create_comment(:schema, :partitions_dynamic, <<~EOS.strip)
+      Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.
+    EOS
+  end
+
+  def down
+    execute 'DROP SCHEMA partitions_dynamic'
+  end
+end

db/structure.sql

 SET search_path=public;
 
+CREATE SCHEMA partitions_dynamic;
+
+COMMENT ON SCHEMA partitions_dynamic IS 'Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.';
+
 CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
 
 CREATE TABLE public.abuse_reports (
@@ -14002,6 +14006,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200609212701
 20200613104045
 20200615083635
+20200615101135
 20200615121217
 20200615123055
 20200615232735

lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb

@@ -8,6 +8,7 @@ module Gitlab
 
         WHITELISTED_TABLES = %w[audit_events].freeze
         ERROR_SCOPE = 'table partitioning'
+        DYNAMIC_PARTITIONS_SCHEMA = 'partitions_dynamic'
 
         # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column.
         # One partition is created per month between the given `min_date` and `max_date`.
@@ -125,7 +126,7 @@ module Gitlab
           min_date = min_date.beginning_of_month.to_date
           max_date = max_date.next_month.beginning_of_month.to_date
 
-          create_range_partition_safely("#{table_name}_000000", table_name, 'MINVALUE', to_sql_date_literal(min_date))
+          create_range_partition_safely("#{table_name}_000000", table_name, 'MINVALUE', to_sql_date_literal(min_date), schema: DYNAMIC_PARTITIONS_SCHEMA)
 
           while min_date < max_date
             partition_name = "#{table_name}_#{min_date.strftime('%Y%m')}"
@@ -133,7 +134,7 @@ module Gitlab
             lower_bound = to_sql_date_literal(min_date)
             upper_bound = to_sql_date_literal(next_date)
 
-            create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
+            create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound, schema: DYNAMIC_PARTITIONS_SCHEMA)
             min_date = next_date
           end
         end
@@ -142,8 +143,8 @@ module Gitlab
           connection.quote(date.strftime('%Y-%m-%d'))
         end
 
-        def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
-          if table_exists?(partition_name)
+        def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound, schema:)
+          if table_exists?("#{schema}.#{partition_name}")
             # rubocop:disable Gitlab/RailsLogger
             Rails.logger.warn "Partition not created because it already exists" \
               " (this may be due to an aborted migration or similar): partition_name: #{partition_name}"
@@ -151,7 +152,7 @@ module Gitlab
             return
           end
 
-          create_range_partition(partition_name, table_name, lower_bound, upper_bound)
+          create_range_partition(partition_name, table_name, lower_bound, upper_bound, schema: schema)
         end
 
         def create_sync_trigger(source_table, target_table, unique_key)

lib/gitlab/database/schema_helpers.rb

@@ -69,9 +69,11 @@ module Gitlab
 
       private
 
-      def create_range_partition(partition_name, table_name, lower_bound, upper_bound)
+      def create_range_partition(partition_name, table_name, lower_bound, upper_bound, schema:)
+        raise ArgumentError, 'explicit schema is required but currently missing' unless schema
+
         execute(<<~SQL)
-          CREATE TABLE #{partition_name} PARTITION OF #{table_name}
+          CREATE TABLE #{schema}.#{partition_name} PARTITION OF #{table_name}
           FOR VALUES FROM (#{lower_bound}) TO (#{upper_bound})
         SQL
       end

spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb

@@ -241,7 +241,7 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers
 
   describe '#drop_partitioned_table_for' do
     let(:expected_tables) do
-      %w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
+      %w[000000 201912 202001 202002].map { |suffix| "partitions_dynamic.#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
     end
 
     context 'when the table is not whitelisted' do

spec/support/helpers/partitioning_helpers.rb

@@ -8,8 +8,8 @@ module PartitioningHelpers
     expect(columns_with_part_type).to match_array(actual_columns)
   end
 
-  def expect_range_partition_of(partition_name, table_name, min_value, max_value)
-    definition = find_partition_definition(partition_name)
+  def expect_range_partition_of(partition_name, table_name, min_value, max_value, schema: 'partitions_dynamic')
+    definition = find_partition_definition(partition_name, schema: schema)
 
     expect(definition).not_to be_nil
     expect(definition['base_table']).to eq(table_name.to_s)
@@ -40,7 +40,7 @@ module PartitioningHelpers
     SQL
   end
 
-  def find_partition_definition(partition)
+  def find_partition_definition(partition, schema: 'partitions_dynamic')
     connection.select_one(<<~SQL)
       select
         parent_class.relname as base_table,
@@ -48,7 +48,10 @@ module PartitioningHelpers
       from pg_class
       inner join pg_inherits i on pg_class.oid = inhrelid
       inner join pg_class parent_class on parent_class.oid = inhparent
-      where pg_class.relname = '#{partition}' and pg_class.relispartition;
+      inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+      where pg_namespace.nspname = '#{schema}'
+        and pg_class.relname = '#{partition}'
+        and pg_class.relispartition
     SQL
   end
 end