2017-04-07 10:27:15 +00:00
|
|
|
require 'spec_helper'
|
|
|
|
|
|
|
|
describe Gitlab::HealthChecks::FsShardsCheck do
|
2017-05-25 18:53:42 +00:00
|
|
|
def command_exists?(command)
|
|
|
|
_, status = Gitlab::Popen.popen(%W{ #{command} 1 echo })
|
2017-07-26 11:23:27 +00:00
|
|
|
status.zero?
|
2017-05-25 18:53:42 +00:00
|
|
|
rescue Errno::ENOENT
|
|
|
|
false
|
|
|
|
end
|
|
|
|
|
|
|
|
def timeout_command
|
|
|
|
@timeout_command ||=
|
|
|
|
if command_exists?('timeout')
|
|
|
|
'timeout'
|
|
|
|
elsif command_exists?('gtimeout')
|
|
|
|
'gtimeout'
|
|
|
|
else
|
|
|
|
''
|
|
|
|
end
|
|
|
|
end
|
2017-05-16 14:25:02 +00:00
|
|
|
|
2017-04-07 10:27:15 +00:00
|
|
|
let(:metric_class) { Gitlab::HealthChecks::Metric }
|
|
|
|
let(:result_class) { Gitlab::HealthChecks::Result }
|
|
|
|
let(:repository_storages) { [:default] }
|
|
|
|
let(:tmp_dir) { Dir.mktmpdir }
|
|
|
|
|
|
|
|
let(:storages_paths) do
|
|
|
|
{
|
|
|
|
default: { path: tmp_dir }
|
|
|
|
}.with_indifferent_access
|
|
|
|
end
|
|
|
|
|
|
|
|
before do
|
|
|
|
allow(described_class).to receive(:repository_storages) { repository_storages }
|
|
|
|
allow(described_class).to receive(:storages_paths) { storages_paths }
|
2017-05-25 18:53:42 +00:00
|
|
|
stub_const('Gitlab::HealthChecks::FsShardsCheck::TIMEOUT_EXECUTABLE', timeout_command)
|
2017-04-07 10:27:15 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
after do
|
|
|
|
FileUtils.remove_entry_secure(tmp_dir) if Dir.exist?(tmp_dir)
|
|
|
|
end
|
|
|
|
|
|
|
|
shared_examples 'filesystem checks' do
|
|
|
|
describe '#readiness' do
|
|
|
|
subject { described_class.readiness }
|
|
|
|
|
|
|
|
context 'storage points to not existing folder' do
|
|
|
|
let(:storages_paths) do
|
|
|
|
{
|
|
|
|
default: { path: 'tmp/this/path/doesnt/exist' }
|
|
|
|
}.with_indifferent_access
|
|
|
|
end
|
|
|
|
|
|
|
|
it { is_expected.to include(result_class.new(false, 'cannot stat storage', shard: :default)) }
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'storage points to directory that has both read and write rights' do
|
|
|
|
before do
|
|
|
|
FileUtils.chmod_R(0755, tmp_dir)
|
|
|
|
end
|
|
|
|
|
|
|
|
it { is_expected.to include(result_class.new(true, nil, shard: :default)) }
|
|
|
|
|
|
|
|
it 'cleans up files used for testing' do
|
|
|
|
expect(described_class).to receive(:storage_write_test).with(any_args).and_call_original
|
|
|
|
|
2017-07-25 22:28:13 +00:00
|
|
|
expect { subject }.not_to change(Dir.entries(tmp_dir), :count)
|
2017-04-07 10:27:15 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
context 'read test fails' do
|
|
|
|
before do
|
|
|
|
allow(described_class).to receive(:storage_read_test).with(any_args).and_return(false)
|
|
|
|
end
|
|
|
|
|
|
|
|
it { is_expected.to include(result_class.new(false, 'cannot read from storage', shard: :default)) }
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'write test fails' do
|
|
|
|
before do
|
|
|
|
allow(described_class).to receive(:storage_write_test).with(any_args).and_return(false)
|
|
|
|
end
|
|
|
|
|
|
|
|
it { is_expected.to include(result_class.new(false, 'cannot write to storage', shard: :default)) }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#metrics' do
|
|
|
|
context 'storage points to not existing folder' do
|
|
|
|
let(:storages_paths) do
|
|
|
|
{
|
|
|
|
default: { path: 'tmp/this/path/doesnt/exist' }
|
|
|
|
}.with_indifferent_access
|
|
|
|
end
|
|
|
|
|
2017-06-29 17:54:10 +00:00
|
|
|
# Unsolved intermittent failure in CI https://gitlab.com/gitlab-org/gitlab-ce/issues/31128
|
2017-06-29 23:53:41 +00:00
|
|
|
around(:each) do |example| # rubocop:disable RSpec/AroundBlock
|
2017-06-29 17:54:10 +00:00
|
|
|
times_to_try = ENV['CI'] ? 4 : 1
|
|
|
|
example.run_with_retry retry: times_to_try
|
|
|
|
end
|
|
|
|
|
2017-06-30 16:00:07 +00:00
|
|
|
it 'provides metrics' do
|
2017-07-25 22:28:13 +00:00
|
|
|
metrics = described_class.metrics
|
|
|
|
|
|
|
|
expect(metrics).to all(have_attributes(labels: { shard: :default }))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_accessible, value: 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_readable, value: 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_writable, value: 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
|
2017-06-30 16:00:07 +00:00
|
|
|
end
|
2017-04-07 10:27:15 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
context 'storage points to directory that has both read and write rights' do
|
|
|
|
before do
|
|
|
|
FileUtils.chmod_R(0755, tmp_dir)
|
|
|
|
end
|
|
|
|
|
2017-06-30 16:00:07 +00:00
|
|
|
it 'provides metrics' do
|
2017-07-25 22:28:13 +00:00
|
|
|
metrics = described_class.metrics
|
|
|
|
|
|
|
|
expect(metrics).to all(have_attributes(labels: { shard: :default }))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_accessible, value: 1))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_readable, value: 1))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_writable, value: 1))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
|
2017-06-30 16:00:07 +00:00
|
|
|
end
|
2017-07-25 12:19:09 +00:00
|
|
|
|
|
|
|
it 'cleans up files used for metrics' do
|
2017-07-25 22:28:13 +00:00
|
|
|
expect { described_class.metrics }.not_to change(Dir.entries(tmp_dir), :count)
|
2017-07-25 12:19:09 +00:00
|
|
|
end
|
2017-04-07 10:27:15 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-05-16 14:25:02 +00:00
|
|
|
context 'when timeout kills fs checks' do
|
|
|
|
before do
|
2017-05-25 18:53:42 +00:00
|
|
|
stub_const('Gitlab::HealthChecks::FsShardsCheck::COMMAND_TIMEOUT', '1')
|
|
|
|
|
|
|
|
allow(described_class).to receive(:exec_with_timeout).and_wrap_original { |m| m.call(%w(sleep 60)) }
|
2017-05-16 14:25:02 +00:00
|
|
|
FileUtils.chmod_R(0755, tmp_dir)
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#readiness' do
|
|
|
|
subject { described_class.readiness }
|
|
|
|
|
|
|
|
it { is_expected.to include(result_class.new(false, 'cannot stat storage', shard: :default)) }
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#metrics' do
|
2017-05-23 10:14:11 +00:00
|
|
|
it 'provides metrics' do
|
2017-07-25 22:28:13 +00:00
|
|
|
metrics = described_class.metrics
|
|
|
|
|
|
|
|
expect(metrics).to all(have_attributes(labels: { shard: :default }))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_accessible, value: 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_readable, value: 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_writable, value: 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
|
|
|
|
expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
|
2017-05-23 10:14:11 +00:00
|
|
|
end
|
2017-05-16 14:25:02 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-07 10:27:15 +00:00
|
|
|
context 'when popen always finds required binaries' do
|
|
|
|
before do
|
2017-05-16 14:25:02 +00:00
|
|
|
allow(described_class).to receive(:exec_with_timeout).and_wrap_original do |method, *args, &block|
|
2017-04-07 10:27:15 +00:00
|
|
|
begin
|
|
|
|
method.call(*args, &block)
|
2017-05-16 14:25:02 +00:00
|
|
|
rescue RuntimeError, Errno::ENOENT
|
2017-04-07 10:27:15 +00:00
|
|
|
raise 'expected not to happen'
|
|
|
|
end
|
|
|
|
end
|
2017-05-16 14:25:02 +00:00
|
|
|
|
2017-05-25 18:53:42 +00:00
|
|
|
stub_const('Gitlab::HealthChecks::FsShardsCheck::COMMAND_TIMEOUT', '10')
|
2017-04-07 10:27:15 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
it_behaves_like 'filesystem checks'
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when popen never finds required binaries' do
|
|
|
|
before do
|
|
|
|
allow(Gitlab::Popen).to receive(:popen).and_raise(Errno::ENOENT)
|
|
|
|
end
|
|
|
|
|
|
|
|
it_behaves_like 'filesystem checks'
|
|
|
|
end
|
|
|
|
end
|