Commit 22868795 authored by Pawel Chojnacki's avatar Pawel Chojnacki

Ensure test files are deleted after tests

parent d95e6da0
...@@ -6,7 +6,7 @@ Prometheus::Client.configure do |config| ...@@ -6,7 +6,7 @@ Prometheus::Client.configure do |config|
config.initial_mmap_file_size = 4 * 1024 config.initial_mmap_file_size = 4 * 1024
config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] config.multiprocess_files_dir = ENV['prometheus_multiproc_dir']
if Rails.env.development? && Rails.env.test? if Rails.env.development? || Rails.env.test?
config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir') config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
end end
end end
...@@ -27,10 +27,10 @@ module Gitlab ...@@ -27,10 +27,10 @@ module Gitlab
Metric.new(name, value, labels) Metric.new(name, value, labels)
end end
def with_timing(proc) def with_timing
start = Time.now start = Time.now
result = proc.call result = yield
yield result, Time.now.to_f - start.to_f [result, Time.now.to_f - start.to_f]
end end
def catch_timeout(seconds, &block) def catch_timeout(seconds, &block)
......
...@@ -10,52 +10,64 @@ module Gitlab ...@@ -10,52 +10,64 @@ module Gitlab
def readiness def readiness
repository_storages.map do |storage_name| repository_storages.map do |storage_name|
begin begin
tmp_file_path = tmp_file_path(storage_name)
if !storage_stat_test(storage_name) if !storage_stat_test(storage_name)
HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name)
elsif !storage_write_test(tmp_file_path)
HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name)
elsif !storage_read_test(tmp_file_path)
HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name)
else else
HealthChecks::Result.new(true, nil, shard: storage_name) with_temp_file(storage_name) do |tmp_file_path|
if !storage_write_test(tmp_file_path)
HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name)
elsif !storage_read_test(tmp_file_path)
HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name)
else
HealthChecks::Result.new(true, nil, shard: storage_name)
end
end
end end
rescue RuntimeError => ex rescue RuntimeError => ex
message = "unexpected error #{ex} when checking storage #{storage_name}" message = "unexpected error #{ex} when checking storage #{storage_name}"
Rails.logger.error(message) Rails.logger.error(message)
HealthChecks::Result.new(false, message, shard: storage_name) HealthChecks::Result.new(false, message, shard: storage_name)
ensure
delete_test_file(tmp_file_path)
end end
end end
end end
def metrics def metrics
repository_storages.flat_map do |storage_name| res = []
tmp_file_path = tmp_file_path(storage_name) repository_storages.each do |storage_name|
[ res << operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do
operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, -> { storage_stat_test(storage_name) }, shard: storage_name), with_timing { storage_stat_test(storage_name) }
operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, -> { storage_write_test(tmp_file_path) }, shard: storage_name), end
operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, -> { storage_read_test(tmp_file_path) }, shard: storage_name)
].flatten res << operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do
with_temp_file(storage_name) do |tmp_file_path|
with_timing { storage_write_test(tmp_file_path) }
end
end
res << operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do
with_temp_file(storage_name) do |tmp_file_path|
storage_write_test(tmp_file_path) # writes data used by read test
with_timing { storage_read_test(tmp_file_path) }
end
end
end end
res.flatten
end end
private private
def operation_metrics(ok_metric, latency_metric, operation, **labels) def operation_metrics(ok_metric, latency_metric, **labels)
with_timing operation do |result, elapsed| result, elapsed = yield
[ [
metric(latency_metric, elapsed, **labels), metric(latency_metric, elapsed, **labels),
metric(ok_metric, result ? 1 : 0, **labels) metric(ok_metric, result ? 1 : 0, **labels)
] ]
end
rescue RuntimeError => ex rescue RuntimeError => ex
Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}")
[metric(ok_metric, 0, **labels)] [metric(ok_metric, 0, **labels)]
end end
def repository_storages def repository_storages
@repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages
end end
...@@ -68,8 +80,13 @@ module Gitlab ...@@ -68,8 +80,13 @@ module Gitlab
Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block)
end end
def tmp_file_path(storage_name) def with_temp_file(storage_name)
Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } begin
temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path }
yield temp_file_path
ensure
delete_test_file(temp_file_path)
end
end end
def path(storage_name) def path(storage_name)
......
...@@ -131,6 +131,12 @@ describe Gitlab::HealthChecks::FsShardsCheck do ...@@ -131,6 +131,12 @@ describe Gitlab::HealthChecks::FsShardsCheck do
expect(subject).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0)) expect(subject).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
expect(subject).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0)) expect(subject).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
end end
it 'cleans up files used for metrics' do
subject
expect(Dir.entries(tmp_dir).count).to eq(2)
end
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment