Update PumaWorkerKiller defaults

* Separate memory limit (550mb) for master.
* Bump worker memory limit from 650mb to 850mb.
* Use timer-based restarter rather than rely on OOM behavior.

Based on metrics and observation of ops.gitlab.net[0].

Closes: https://gitlab.com/gitlab-org/gitlab/issues/32646

[0]: https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/7924

changelogs/unreleased/bjk-32646_puma_killer.yml

+---
+title: Update PumaWorkerKiller defaults
+merge_request: 17758
+author:
+type: performance

lib/gitlab/cluster/puma_worker_killer_initializer.rb

@@ -3,7 +3,7 @@
 module Gitlab
   module Cluster
     class PumaWorkerKillerInitializer
-      def self.start(puma_options, puma_per_worker_max_memory_mb: 650)
+      def self.start(puma_options, puma_per_worker_max_memory_mb: 850, puma_master_max_memory_mb: 550)
         require 'puma_worker_killer'
 
         PumaWorkerKiller.config do |config|
@@ -12,10 +12,9 @@ module Gitlab
           # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX
           worker_count = puma_options[:workers] || 1
           # The Puma Worker Killer checks the total RAM used by both the master
-          # and worker processes. Bump the limits to N+1 instead of N workers
-          # to account for this:
+          # and worker processes.
           # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57
-          config.ram = (worker_count + 1) * puma_per_worker_max_memory_mb
+          config.ram = puma_master_max_memory_mb + (worker_count * puma_per_worker_max_memory_mb)
 
           config.frequency = 20 # seconds
 
@@ -23,10 +22,9 @@ module Gitlab
           # of available RAM.
           config.percent_usage = 0.98
 
-          # Ideally we'll never hit the maximum amount of memory. If so the worker
-          # is restarted already, thus periodically restarting workers shouldn't be
-          # needed.
-          config.rolling_restart_frequency = false
+          # Ideally we'll never hit the maximum amount of memory. Restart the workers
+          # regularly rather than rely on OOM behavior for periodic restarting.
+          config.rolling_restart_frequency = 43200 # 12 hours in seconds.
 
           observer = Gitlab::Cluster::PumaWorkerKillerObserver.new
           config.pre_term = observer.callback