Commit 4f9a1406 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Wait 15 minutes before Sidekiq MemoryKiller action

parent eb12e3ab
module Gitlab module Gitlab
module SidekiqMiddleware module SidekiqMiddleware
class MemoryKiller class MemoryKiller
# Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
GRACE_TIME = 15 * 60
# Wait 30 seconds for running jobs to finish during graceful shutdown # Wait 30 seconds for running jobs to finish during graceful shutdown
GRACEFUL_SHUTDOWN_WAIT = 30 SHUTDOWN_WAIT = 30
# Create a mutex so that there will be only one thread waiting to shut
# Sidekiq down
MUTEX = Mutex.new
def call(worker, job, queue) def call(worker, job, queue)
yield yield
current_rss = get_rss current_rss = get_rss
return unless max_rss > 0 && current_rss > max_rss return unless max_rss > 0 && current_rss > max_rss
Tread.new do
# Return if another thread is already waiting to shut Sidekiq down
return unless MUTEX.try_lock
Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
"#{max_rss}" "#{max_rss}"
Sidekiq.logger.warn "spawned thread that will shut down PID "\
"#{Process.pid} in #{grace_time} seconds"
sleep(grace_time)
Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
# SIGUSR1 tells Sidekiq to stop accepting new jobs
Process.kill('SIGUSR1', Process.pid) Process.kill('SIGUSR1', Process.pid)
Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\
"#{Process.pid} in #{graceful_shutdown_wait} seconds" "SIGTERM to PID #{Process.pid}"
# Send the final shutdown signal to Sidekiq from a separate thread so sleep(shutdown_wait)
# that the current job can finish
Thread.new do Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
sleep(graceful_shutdown_wait)
Process.kill('SIGTERM', Process.pid) Process.kill('SIGTERM', Process.pid)
end end
end end
...@@ -38,9 +50,15 @@ module Gitlab ...@@ -38,9 +50,15 @@ module Gitlab
@max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i
end end
def graceful_shutdown_wait def shutdown_wait
@graceful_shutdown_wait ||= ( @graceful_shutdown_wait ||= (
ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT
).to_i
end
def grace_time
@grace_time ||= (
ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME
).to_i ).to_i
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment