Merge branch 'morefice/drop-big-detached-partitions-on-weekends' into 'master'

Drop big detached partitions on weekends

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184045



Merged-by: Simon Tomlinson <stomlinson@gitlab.com>
Approved-by: Simon Tomlinson <stomlinson@gitlab.com>
Reviewed-by: Max Orefice <morefice@gitlab.com>
Reviewed-by: Simon Tomlinson <stomlinson@gitlab.com>
Co-authored-by: Maxime Orefice <morefice@gitlab.com>

lib/gitlab/database/partitioning/partition_manager.rb

@@ -13,6 +13,7 @@ class PartitionManager
         STATEMENT_TIMEOUT = 1.hour
         MANAGEMENT_LEASE_KEY = 'database_partition_management_%s'
         RETAIN_DETACHED_PARTITIONS_FOR = 1.week
+        MAX_PARTITION_SIZE = 150.gigabytes
 
         def initialize(model, connection: nil)
           @model = model
@@ -120,8 +121,7 @@ def detach_one_partition(partition)
 
           connection.execute partition.to_detach_sql
 
-          Postgresql::DetachedPartition.create!(table_name: partition.partition_name,
-            drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now)
+          schedule_detached_partition_cleanup(partition)
 
           Gitlab::AppLogger.info(
             message: "Detached Partition",
@@ -245,6 +245,33 @@ def parent_table_has_loose_foreign_key?
           has_loose_foreign_key?(model.table_name)
         end
         strong_memoize_attr :parent_table_has_loose_foreign_key?
+
+        def schedule_detached_partition_cleanup(partition)
+          identifier = identifier(partition)
+
+          if above_threshold?(identifier)
+            Postgresql::DetachedPartition.create!(
+              table_name: partition.partition_name,
+              drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now.next_occurring(:saturday)
+            )
+          else
+            Postgresql::DetachedPartition.create!(
+              table_name: partition.partition_name,
+              drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now
+            )
+          end
+        end
+
+        def above_threshold?(identifier)
+          Gitlab::Database::PostgresPartition
+            .for_identifier(identifier)
+            .above_threshold(MAX_PARTITION_SIZE)
+            .exists?
+        end
+
+        def identifier(partition)
+          "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition.partition_name}"
+        end
       end
     end
   end

spec/lib/gitlab/database/partitioning/partition_manager_spec.rb

@@ -352,6 +352,27 @@ def num_tables
         expect { subject }.not_to change { find_partitions(my_model.table_name).size }
       end
     end
+
+    context 'when scheduling partition drops for large partitions' do
+      before do
+        allow_next_instance_of(described_class) do |instance|
+          allow(instance).to receive(:above_threshold?).and_return(true)
+        end
+      end
+
+      it 'schedules large partitions for weekend drops' do
+        next_saturday = Date.parse('2021-07-03')
+
+        expect(Postgresql::DetachedPartition).to receive(:create!).with(
+          table_name: "#{partitioned_table_name}_202104",
+          drop_after: anything
+        ) do |args|
+          expect(args[:drop_after].to_date).to eq(next_saturday)
+        end
+
+        subject
+      end
+    end
   end
 
   describe 'analyze partitioned table' do