Commit 9d2be756 authored by Sean McGivern's avatar Sean McGivern

Limit Sidekiq args logging to 10 KB of JSON

When logging arguments from Sidekiq to JSON, restrict the size of `args`
to 10 KB (when converted to JSON). This is to avoid blowing up with
excessively large job payloads.
parent 5e01cf72
---
title: Prevent Sidekiq arguments over 10 KB in size from being logged to JSON
merge_request: 24493
author:
type: changed
......@@ -11,6 +11,11 @@ Example:
gitlab_rails['env'] = {"SIDEKIQ_LOG_ARGUMENTS" => "1"}
```
Please note: It is not recommend to enable this setting in production because some
Please note: It is not recommend to enable this setting in production because some
Sidekiq jobs (such as sending a password reset email) take secret arguments (for
example the password reset token).
\ No newline at end of file
example the password reset token).
When using [Sidekiq JSON logging](../administration/logs.md#sidekiqlog),
arguments logs are limited to a maximum size of 10 kilobytes of text;
any arguments after this limit will be discarded and replaced with a
single argument containing the string `"..."`.
......@@ -5,6 +5,7 @@ module Gitlab
class StructuredLogger
START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze
DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze
MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes
def call(job, queue)
started_at = current_time
......@@ -64,6 +65,7 @@ module Gitlab
job['pid'] = ::Process.pid
job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
job['args'] = limited_job_args(job['args']) if job['args']
convert_to_iso8601(job, START_TIMESTAMP_FIELDS)
......@@ -93,6 +95,21 @@ module Gitlab
Time.at(timestamp).utc.iso8601(3)
end
def limited_job_args(args)
return unless args.is_a?(Array)
total_length = 0
limited_args = args.take_while do |arg|
total_length += arg.to_json.length
total_length <= MAXIMUM_JOB_ARGUMENTS_LENGTH
end
limited_args.push('...') if total_length > MAXIMUM_JOB_ARGUMENTS_LENGTH
limited_args
end
end
end
end
......@@ -82,15 +82,36 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
end.to raise_error(ArgumentError)
end
end
context 'when the job args are bigger than the maximum allowed' do
it 'keeps args from the front until they exceed the limit' do
Timecop.freeze(timestamp) do
job['args'] = [
1,
2,
'a' * (described_class::MAXIMUM_JOB_ARGUMENTS_LENGTH / 2),
'b' * (described_class::MAXIMUM_JOB_ARGUMENTS_LENGTH / 2),
3
]
expected_args = job['args'].take(3) + ['...']
expect(logger).to receive(:info).with(start_payload.merge('args' => expected_args)).ordered
expect(logger).to receive(:info).with(end_payload.merge('args' => expected_args)).ordered
expect(subject).to receive(:log_job_start).and_call_original
expect(subject).to receive(:log_job_done).and_call_original
subject.call(job, 'test_queue') { }
end
end
end
end
context 'with SIDEKIQ_LOG_ARGUMENTS disabled' do
it 'logs start and end of job' do
it 'logs start and end of job without args' do
Timecop.freeze(timestamp) do
start_payload.delete('args')
expect(logger).to receive(:info).with(start_payload).ordered
expect(logger).to receive(:info).with(end_payload).ordered
expect(logger).to receive(:info).with(start_payload.except('args')).ordered
expect(logger).to receive(:info).with(end_payload.except('args')).ordered
expect(subject).to receive(:log_job_start).and_call_original
expect(subject).to receive(:log_job_done).and_call_original
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment