gitlab-org--gitlab-foss/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb

require 'spec_helper'

describe Gitlab::HealthChecks::FsShardsCheck do
  def command_exists?(command)
    _, status = Gitlab::Popen.popen(%W{ #{command} 1 echo })
    status.zero?
  rescue Errno::ENOENT
    false
  end

  def timeout_command
    @timeout_command ||=
      if command_exists?('timeout')
        'timeout'
      elsif command_exists?('gtimeout')
        'gtimeout'
      else
        ''
      end
  end

  let(:metric_class) { Gitlab::HealthChecks::Metric }
  let(:result_class) { Gitlab::HealthChecks::Result }
  let(:repository_storages) { [:default] }
  let(:tmp_dir) { Dir.mktmpdir }

  let(:storages_paths) do
    {
      default: { path: tmp_dir }
    }.with_indifferent_access
  end

  before do
    allow(described_class).to receive(:repository_storages) { repository_storages }
    allow(described_class).to receive(:storages_paths) { storages_paths }
    stub_const('Gitlab::HealthChecks::FsShardsCheck::TIMEOUT_EXECUTABLE', timeout_command)
  end

  after do
    FileUtils.remove_entry_secure(tmp_dir) if Dir.exist?(tmp_dir)
  end

  shared_examples 'filesystem checks' do
    describe '#readiness' do
      subject { described_class.readiness }

      context 'storage has a tripped circuitbreaker', broken_storage: true do
        let(:repository_storages) { ['broken'] }
        let(:storages_paths) do
          Gitlab.config.repositories.storages
        end

        it { is_expected.to include(result_class.new(false, 'circuitbreaker tripped', shard: 'broken')) }
      end

      context 'storage points to not existing folder' do
        let(:storages_paths) do
          {
            default: { path: 'tmp/this/path/doesnt/exist' }
          }.with_indifferent_access
        end

        before do
          allow(described_class).to receive(:storage_circuitbreaker_test) { true }
        end

        it { is_expected.to include(result_class.new(false, 'cannot stat storage', shard: :default)) }
      end

      context 'storage points to directory that has both read and write rights' do
        before do
          FileUtils.chmod_R(0755, tmp_dir)
        end

        it { is_expected.to include(result_class.new(true, nil, shard: :default)) }

        it 'cleans up files used for testing' do
          expect(described_class).to receive(:storage_write_test).with(any_args).and_call_original

          expect { subject }.not_to change(Dir.entries(tmp_dir), :count)
        end

        context 'read test fails' do
          before do
            allow(described_class).to receive(:storage_read_test).with(any_args).and_return(false)
          end

          it { is_expected.to include(result_class.new(false, 'cannot read from storage', shard: :default)) }
        end

        context 'write test fails' do
          before do
            allow(described_class).to receive(:storage_write_test).with(any_args).and_return(false)
          end

          it { is_expected.to include(result_class.new(false, 'cannot write to storage', shard: :default)) }
        end
      end
    end

    describe '#metrics' do
      context 'storage points to not existing folder' do
        let(:storages_paths) do
          {
            default: { path: 'tmp/this/path/doesnt/exist' }
          }.with_indifferent_access
        end

        it 'provides metrics' do
          metrics = described_class.metrics

          expect(metrics).to all(have_attributes(labels: { shard: :default }))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_accessible, value: 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_readable, value: 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_writable, value: 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_circuitbreaker_latency_seconds, value: be >= 0))
        end
      end

      context 'storage points to directory that has both read and write rights' do
        before do
          FileUtils.chmod_R(0755, tmp_dir)
        end

        it 'provides metrics' do
          metrics = described_class.metrics

          expect(metrics).to all(have_attributes(labels: { shard: :default }))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_accessible, value: 1))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_readable, value: 1))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_writable, value: 1))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
          expect(metrics).to include(an_object_having_attributes(name: :filesystem_circuitbreaker_latency_seconds, value: be >= 0))
        end

        it 'cleans up files used for metrics' do
          expect { described_class.metrics }.not_to change(Dir.entries(tmp_dir), :count)
        end
      end
    end
  end

  context 'when timeout kills fs checks' do
    before do
      stub_const('Gitlab::HealthChecks::FsShardsCheck::COMMAND_TIMEOUT', '1')

      allow(described_class).to receive(:exec_with_timeout).and_wrap_original { |m| m.call(%w(sleep 60)) }
      FileUtils.chmod_R(0755, tmp_dir)
    end

    describe '#readiness' do
      subject { described_class.readiness }

      it { is_expected.to include(result_class.new(false, 'cannot stat storage', shard: :default)) }
    end

    describe '#metrics' do
      it 'provides metrics' do
        metrics = described_class.metrics

        expect(metrics).to all(have_attributes(labels: { shard: :default }))
        expect(metrics).to include(an_object_having_attributes(name: :filesystem_accessible, value: 0))
        expect(metrics).to include(an_object_having_attributes(name: :filesystem_readable, value: 0))
        expect(metrics).to include(an_object_having_attributes(name: :filesystem_writable, value: 0))
        expect(metrics).to include(an_object_having_attributes(name: :filesystem_access_latency_seconds, value: be >= 0))
        expect(metrics).to include(an_object_having_attributes(name: :filesystem_read_latency_seconds, value: be >= 0))
        expect(metrics).to include(an_object_having_attributes(name: :filesystem_write_latency_seconds, value: be >= 0))
      end
    end
  end

  context 'when popen always finds required binaries' do
    before do
      allow(described_class).to receive(:exec_with_timeout).and_wrap_original do |method, *args, &block|
        begin
          method.call(*args, &block)
        rescue RuntimeError, Errno::ENOENT
          raise 'expected not to happen'
        end
      end

      stub_const('Gitlab::HealthChecks::FsShardsCheck::COMMAND_TIMEOUT', '10')
    end

    it_behaves_like 'filesystem checks'
  end

  context 'when popen never finds required binaries' do
    before do
      allow(Gitlab::Popen).to receive(:popen).and_raise(Errno::ENOENT)
    end

    it_behaves_like 'filesystem checks'
  end
end