Extract system check rake task logic

These changes make the code more reusable, testable, and most
importantly, overrideable.
This commit is contained in:
Michael Kozono 2018-11-27 16:08:31 -08:00
parent c3c25174e3
commit 6855e6b586
19 changed files with 566 additions and 276 deletions

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module SystemCheck
class GitalyCheck < BaseCheck
set_name 'Gitaly:'
def multi_check
Gitlab::HealthChecks::GitalyCheck.readiness.each do |result|
$stdout.print "#{result.labels[:shard]} ... "
if result.success
$stdout.puts 'OK'.color(:green)
else
$stdout.puts "FAIL: #{result.message}".color(:red)
end
end
end
end
end

View file

@ -0,0 +1,56 @@
# frozen_string_literal: true
module SystemCheck
# Used by gitlab:gitlab_shell:check rake task
class GitlabShellCheck < BaseCheck
set_name 'GitLab Shell:'
def multi_check
check_gitlab_shell
check_gitlab_shell_self_test
end
private
def check_gitlab_shell
required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
$stdout.print "GitLab Shell version >= #{required_version} ? ... "
if current_version.valid? && required_version <= current_version
$stdout.puts "OK (#{current_version})".color(:green)
else
$stdout.puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
end
end
def check_gitlab_shell_self_test
gitlab_shell_repo_base = gitlab_shell_path
check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
$stdout.puts "Running #{check_cmd}"
if system(check_cmd, chdir: gitlab_shell_repo_base)
$stdout.puts 'gitlab-shell self-check successful'.color(:green)
else
$stdout.puts 'gitlab-shell self-check failed'.color(:red)
try_fixing_it(
'Make sure GitLab is running;',
'Check the gitlab-shell configuration file:',
sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}")
)
fix_and_rerun
end
end
# Helper methods
########################
def gitlab_shell_path
Gitlab.config.gitlab_shell.path
end
def gitlab_shell_version
Gitlab::Shell.new.version
end
end
end

View file

@ -0,0 +1,27 @@
# frozen_string_literal: true
module SystemCheck
# Used by gitlab:incoming_email:check rake task
class IncomingEmailCheck < BaseCheck
set_name 'Incoming Email:'
def multi_check
if Gitlab.config.incoming_email.enabled
checks = [
SystemCheck::IncomingEmail::ImapAuthenticationCheck
]
if Rails.env.production?
checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
else
checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
end
SystemCheck.run('Reply by email', checks)
else
$stdout.puts 'Reply by email is disabled in config/gitlab.yml'
end
end
end
end

View file

@ -0,0 +1,60 @@
# frozen_string_literal: true
module SystemCheck
# Used by gitlab:ldap:check rake task
class LdapCheck < BaseCheck
set_name 'LDAP:'
def multi_check
if Gitlab::Auth::LDAP::Config.enabled?
# Only show up to 100 results because LDAP directories can be very big.
# This setting only affects the `rake gitlab:check` script.
limit = ENV['LDAP_CHECK_LIMIT']
limit = 100 if limit.blank?
check_ldap(limit)
else
$stdout.puts 'LDAP is disabled in config/gitlab.yml'
end
end
private
def check_ldap(limit)
servers = Gitlab::Auth::LDAP::Config.providers
servers.each do |server|
$stdout.puts "Server: #{server}"
begin
Gitlab::Auth::LDAP::Adapter.open(server) do |adapter|
check_ldap_auth(adapter)
$stdout.puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
users = adapter.users(adapter.config.uid, '*', limit)
users.each do |user|
$stdout.puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
end
end
rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
$stdout.puts "Could not connect to the LDAP server: #{e.message}".color(:red)
end
end
end
def check_ldap_auth(adapter)
auth = adapter.config.has_auth?
message = if auth && adapter.ldap.bind
'Success'.color(:green)
elsif auth
'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
else
'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
end
$stdout.puts "LDAP authentication... #{message}"
end
end
end

View file

@ -4,7 +4,6 @@ module SystemCheck
module Orphans
class RepositoryCheck < SystemCheck::BaseCheck
set_name 'Orphaned repositories:'
attr_accessor :orphans
def multi_check
Gitlab::GitalyClient::StorageSettings.allow_disk_access do

View file

@ -0,0 +1,38 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:app:check rake task
module AppTask
extend RakeTaskHelpers
def self.name
'GitLab App'
end
def self.checks
[
SystemCheck::App::GitConfigCheck,
SystemCheck::App::DatabaseConfigExistsCheck,
SystemCheck::App::MigrationsAreUpCheck,
SystemCheck::App::OrphanedGroupMembersCheck,
SystemCheck::App::GitlabConfigExistsCheck,
SystemCheck::App::GitlabConfigUpToDateCheck,
SystemCheck::App::LogWritableCheck,
SystemCheck::App::TmpWritableCheck,
SystemCheck::App::UploadsDirectoryExistsCheck,
SystemCheck::App::UploadsPathPermissionCheck,
SystemCheck::App::UploadsPathTmpPermissionCheck,
SystemCheck::App::InitScriptExistsCheck,
SystemCheck::App::InitScriptUpToDateCheck,
SystemCheck::App::ProjectsHaveNamespaceCheck,
SystemCheck::App::RedisVersionCheck,
SystemCheck::App::RubyVersionCheck,
SystemCheck::App::GitVersionCheck,
SystemCheck::App::GitUserDefaultSSHConfigCheck,
SystemCheck::App::ActiveUsersCheck
]
end
end
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:gitaly:check rake task
class GitalyTask
extend RakeTaskHelpers
def self.name
'Gitaly'
end
def self.checks
[SystemCheck::GitalyCheck]
end
end
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:gitlab_shell:check rake task
class GitlabShellTask
extend RakeTaskHelpers
def self.name
'GitLab Shell'
end
def self.checks
[SystemCheck::GitlabShellCheck]
end
end
end
end

View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:check rake task
class GitlabTask
extend RakeTaskHelpers
def self.name
'GitLab'
end
def self.manual_run_checks!
start_checking("#{name} subtasks")
subtasks.each(&:run_checks!)
finished_checking("#{name} subtasks")
end
def self.subtasks
[
SystemCheck::RakeTask::GitlabShellTask,
SystemCheck::RakeTask::GitalyTask,
SystemCheck::RakeTask::SidekiqTask,
SystemCheck::RakeTask::IncomingEmailTask,
SystemCheck::RakeTask::LdapTask,
SystemCheck::RakeTask::AppTask
]
end
end
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:incoming_email:check rake task
class IncomingEmailTask
extend RakeTaskHelpers
def self.name
'Incoming Email'
end
def self.checks
[SystemCheck::IncomingEmailCheck]
end
end
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:ldap:check rake task
class LdapTask
extend RakeTaskHelpers
def self.name
'LDAP'
end
def self.checks
[SystemCheck::LdapCheck]
end
end
end
end

View file

@ -0,0 +1,20 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
module Orphans
# Used by gitlab:orphans:check_namespaces rake task
class NamespaceTask
extend RakeTaskHelpers
def self.name
'Orphans'
end
def self.checks
[SystemCheck::Orphans::NamespaceCheck]
end
end
end
end
end

View file

@ -0,0 +1,20 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
module Orphans
# Used by gitlab:orphans:check_repositories rake task
class RepositoryTask
extend RakeTaskHelpers
def self.name
'Orphans'
end
def self.checks
[SystemCheck::Orphans::RepositoryCheck]
end
end
end
end
end

View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:orphans:check rake task
class OrphansTask
extend RakeTaskHelpers
def self.name
'Orphans'
end
def self.checks
[
SystemCheck::Orphans::NamespaceCheck,
SystemCheck::Orphans::RepositoryCheck
]
end
end
end
end

View file

@ -0,0 +1,32 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Provides the run! method intended to be called from system check rake tasks
module RakeTaskHelpers
include ::SystemCheck::Helpers
def run!
warn_user_is_not_gitlab
if self.respond_to?(:manual_run_checks!)
manual_run_checks!
else
run_checks!
end
end
def run_checks!
SystemCheck.run(name, checks)
end
def name
raise NotImplementedError
end
def checks
raise NotImplementedError
end
end
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
module SystemCheck
module RakeTask
# Used by gitlab:sidekiq:check rake task
class SidekiqTask
extend RakeTaskHelpers
def self.name
'Sidekiq'
end
def self.checks
[SystemCheck::SidekiqCheck]
end
end
end
end

View file

@ -0,0 +1,58 @@
# frozen_string_literal: true
module SystemCheck
# Used by gitlab:sidekiq:check rake task
class SidekiqCheck < BaseCheck
set_name 'Sidekiq:'
def multi_check
check_sidekiq_running
only_one_sidekiq_running
end
private
def check_sidekiq_running
$stdout.print "Running? ... "
if sidekiq_process_count > 0
$stdout.puts "yes".color(:green)
else
$stdout.puts "no".color(:red)
try_fixing_it(
sudo_gitlab("RAILS_ENV=production bin/background_jobs start")
)
for_more_information(
see_installation_guide_section("Install Init Script"),
"see log/sidekiq.log for possible errors"
)
fix_and_rerun
end
end
def only_one_sidekiq_running
process_count = sidekiq_process_count
return if process_count.zero?
$stdout.print 'Number of Sidekiq processes ... '
if process_count == 1
$stdout.puts '1'.color(:green)
else
$stdout.puts "#{process_count}".color(:red)
try_fixing_it(
'sudo service gitlab stop',
"sudo pkill -u #{gitlab_user} -f sidekiq",
"sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq",
'sudo service gitlab start'
)
fix_and_rerun
end
end
def sidekiq_process_count
ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
end
end
end

View file

@ -1,299 +1,66 @@
namespace :gitlab do
desc 'GitLab | Check the configuration of GitLab and its environment'
task check: %w{gitlab:gitlab_shell:check
gitlab:gitaly:check
gitlab:sidekiq:check
gitlab:incoming_email:check
gitlab:ldap:check
gitlab:app:check}
task check: :gitlab_environment do
SystemCheck::RakeTask::GitlabTask.run!
end
namespace :app do
desc 'GitLab | Check the configuration of the GitLab Rails app'
task check: :gitlab_environment do
warn_user_is_not_gitlab
checks = [
SystemCheck::App::GitConfigCheck,
SystemCheck::App::DatabaseConfigExistsCheck,
SystemCheck::App::MigrationsAreUpCheck,
SystemCheck::App::OrphanedGroupMembersCheck,
SystemCheck::App::GitlabConfigExistsCheck,
SystemCheck::App::GitlabConfigUpToDateCheck,
SystemCheck::App::LogWritableCheck,
SystemCheck::App::TmpWritableCheck,
SystemCheck::App::UploadsDirectoryExistsCheck,
SystemCheck::App::UploadsPathPermissionCheck,
SystemCheck::App::UploadsPathTmpPermissionCheck,
SystemCheck::App::InitScriptExistsCheck,
SystemCheck::App::InitScriptUpToDateCheck,
SystemCheck::App::ProjectsHaveNamespaceCheck,
SystemCheck::App::RedisVersionCheck,
SystemCheck::App::RubyVersionCheck,
SystemCheck::App::GitVersionCheck,
SystemCheck::App::GitUserDefaultSSHConfigCheck,
SystemCheck::App::ActiveUsersCheck
]
SystemCheck.run('GitLab', checks)
SystemCheck::RakeTask::AppTask.run!
end
end
namespace :gitlab_shell do
desc "GitLab | Check the configuration of GitLab Shell"
task check: :gitlab_environment do
warn_user_is_not_gitlab
start_checking "GitLab Shell"
check_gitlab_shell
check_gitlab_shell_self_test
finished_checking "GitLab Shell"
end
# Checks
########################
def check_gitlab_shell_self_test
gitlab_shell_repo_base = gitlab_shell_path
check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base)
puts "Running #{check_cmd}"
if system(check_cmd, chdir: gitlab_shell_repo_base)
puts 'gitlab-shell self-check successful'.color(:green)
else
puts 'gitlab-shell self-check failed'.color(:red)
try_fixing_it(
'Make sure GitLab is running;',
'Check the gitlab-shell configuration file:',
sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}")
)
fix_and_rerun
end
end
# Helper methods
########################
def gitlab_shell_path
Gitlab.config.gitlab_shell.path
end
def gitlab_shell_version
Gitlab::Shell.new.version
end
def gitlab_shell_major_version
Gitlab::Shell.version_required.split('.')[0].to_i
end
def gitlab_shell_minor_version
Gitlab::Shell.version_required.split('.')[1].to_i
end
def gitlab_shell_patch_version
Gitlab::Shell.version_required.split('.')[2].to_i
SystemCheck::RakeTask::GitlabShellTask.run!
end
end
namespace :gitaly do
desc 'GitLab | Check the health of Gitaly'
task check: :gitlab_environment do
warn_user_is_not_gitlab
start_checking 'Gitaly'
Gitlab::HealthChecks::GitalyCheck.readiness.each do |result|
print "#{result.labels[:shard]} ... "
if result.success
puts 'OK'.color(:green)
else
puts "FAIL: #{result.message}".color(:red)
end
end
finished_checking 'Gitaly'
SystemCheck::RakeTask::GitalyTask.run!
end
end
namespace :sidekiq do
desc "GitLab | Check the configuration of Sidekiq"
task check: :gitlab_environment do
warn_user_is_not_gitlab
start_checking "Sidekiq"
check_sidekiq_running
only_one_sidekiq_running
finished_checking "Sidekiq"
end
# Checks
########################
def check_sidekiq_running
print "Running? ... "
if sidekiq_process_count > 0
puts "yes".color(:green)
else
puts "no".color(:red)
try_fixing_it(
sudo_gitlab("RAILS_ENV=production bin/background_jobs start")
)
for_more_information(
see_installation_guide_section("Install Init Script"),
"see log/sidekiq.log for possible errors"
)
fix_and_rerun
end
end
def only_one_sidekiq_running
process_count = sidekiq_process_count
return if process_count.zero?
print 'Number of Sidekiq processes ... '
if process_count == 1
puts '1'.color(:green)
else
puts "#{process_count}".color(:red)
try_fixing_it(
'sudo service gitlab stop',
"sudo pkill -u #{gitlab_user} -f sidekiq",
"sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq",
'sudo service gitlab start'
)
fix_and_rerun
end
end
def sidekiq_process_count
ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww))
ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count
SystemCheck::RakeTask::SidekiqTask.run!
end
end
namespace :incoming_email do
desc "GitLab | Check the configuration of Reply by email"
task check: :gitlab_environment do
warn_user_is_not_gitlab
if Gitlab.config.incoming_email.enabled
checks = [
SystemCheck::IncomingEmail::ImapAuthenticationCheck
]
if Rails.env.production?
checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
else
checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck
end
SystemCheck.run('Reply by email', checks)
else
puts 'Reply by email is disabled in config/gitlab.yml'
end
SystemCheck::RakeTask::IncomingEmailTask.run!
end
end
namespace :ldap do
task :check, [:limit] => :gitlab_environment do |_, args|
# Only show up to 100 results because LDAP directories can be very big.
# This setting only affects the `rake gitlab:check` script.
args.with_defaults(limit: 100)
warn_user_is_not_gitlab
start_checking "LDAP"
ENV['LDAP_CHECK_LIMIT'] = args.limit if args.limit.present?
if Gitlab::Auth::LDAP::Config.enabled?
check_ldap(args.limit)
else
puts 'LDAP is disabled in config/gitlab.yml'
end
finished_checking "LDAP"
end
def check_ldap(limit)
servers = Gitlab::Auth::LDAP::Config.providers
servers.each do |server|
puts "Server: #{server}"
begin
Gitlab::Auth::LDAP::Adapter.open(server) do |adapter|
check_ldap_auth(adapter)
puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)"
users = adapter.users(adapter.config.uid, '*', limit)
users.each do |user|
puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
end
end
rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e
puts "Could not connect to the LDAP server: #{e.message}".color(:red)
end
end
end
def check_ldap_auth(adapter)
auth = adapter.config.has_auth?
message = if auth && adapter.ldap.bind
'Success'.color(:green)
elsif auth
'Failed. Check `bind_dn` and `password` configuration values'.color(:red)
else
'Anonymous. No `bind_dn` or `password` configured'.color(:yellow)
end
puts "LDAP authentication... #{message}"
SystemCheck::RakeTask::LdapTask.run!
end
end
namespace :orphans do
desc 'Gitlab | Check for orphaned namespaces and repositories'
task check: :gitlab_environment do
warn_user_is_not_gitlab
checks = [
SystemCheck::Orphans::NamespaceCheck,
SystemCheck::Orphans::RepositoryCheck
]
SystemCheck.run('Orphans', checks)
SystemCheck::RakeTask::OrphansTask.run!
end
desc 'GitLab | Check for orphaned namespaces in the repositories path'
task check_namespaces: :gitlab_environment do
warn_user_is_not_gitlab
checks = [SystemCheck::Orphans::NamespaceCheck]
SystemCheck.run('Orphans', checks)
SystemCheck::RakeTask::Orphans::NamespaceTask.run!
end
desc 'GitLab | Check for orphaned repositories in the repositories path'
task check_repositories: :gitlab_environment do
warn_user_is_not_gitlab
checks = [SystemCheck::Orphans::RepositoryCheck]
SystemCheck.run('Orphans', checks)
end
end
# Helper methods
##########################
def check_gitlab_shell
required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version)
current_version = Gitlab::VersionInfo.parse(gitlab_shell_version)
print "GitLab Shell version >= #{required_version} ? ... "
if current_version.valid? && required_version <= current_version
puts "OK (#{current_version})".color(:green)
else
puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red)
SystemCheck::RakeTask::Orphans::RepositoryTask.run!
end
end
end

View file

@ -1,51 +1,101 @@
require 'rake_helper'
describe 'gitlab:ldap:check rake task' do
include LdapHelpers
describe 'check.rake' do
before do
Rake.application.rake_require 'tasks/gitlab/check'
stub_warn_user_is_not_gitlab
end
context 'when LDAP is not enabled' do
it 'does not attempt to bind or search for users' do
expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers)
expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open)
run_rake_task('gitlab:ldap:check')
shared_examples_for 'system check rake task' do
it 'runs the check' do
expect do
subject
end.to output(/Checking #{name} ... Finished/).to_stdout
end
end
context 'when LDAP is enabled' do
let(:ldap) { double(:ldap) }
let(:adapter) { ldap_adapter('ldapmain', ldap) }
describe 'gitlab:check rake task' do
subject { run_rake_task('gitlab:check') }
let(:name) { 'GitLab subtasks' }
before do
allow(Gitlab::Auth::LDAP::Config)
.to receive_messages(
enabled?: true,
providers: ['ldapmain']
)
allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter)
allow(adapter).to receive(:users).and_return([])
it_behaves_like 'system check rake task'
end
describe 'gitlab:gitlab_shell:check rake task' do
subject { run_rake_task('gitlab:gitlab_shell:check') }
let(:name) { 'GitLab Shell' }
it_behaves_like 'system check rake task'
end
describe 'gitlab:gitaly:check rake task' do
subject { run_rake_task('gitlab:gitaly:check') }
let(:name) { 'Gitaly' }
it_behaves_like 'system check rake task'
end
describe 'gitlab:sidekiq:check rake task' do
subject { run_rake_task('gitlab:sidekiq:check') }
let(:name) { 'Sidekiq' }
it_behaves_like 'system check rake task'
end
describe 'gitlab:incoming_email:check rake task' do
subject { run_rake_task('gitlab:incoming_email:check') }
let(:name) { 'Incoming Email' }
it_behaves_like 'system check rake task'
end
describe 'gitlab:ldap:check rake task' do
include LdapHelpers
subject { run_rake_task('gitlab:ldap:check') }
let(:name) { 'LDAP' }
it_behaves_like 'system check rake task'
context 'when LDAP is not enabled' do
it 'does not attempt to bind or search for users' do
expect(Gitlab::Auth::LDAP::Config).not_to receive(:providers)
expect(Gitlab::Auth::LDAP::Adapter).not_to receive(:open)
subject
end
end
it 'attempts to bind using credentials' do
stub_ldap_config(has_auth?: true)
context 'when LDAP is enabled' do
let(:ldap) { double(:ldap) }
let(:adapter) { ldap_adapter('ldapmain', ldap) }
expect(ldap).to receive(:bind)
before do
allow(Gitlab::Auth::LDAP::Config)
.to receive_messages(
enabled?: true,
providers: ['ldapmain']
)
allow(Gitlab::Auth::LDAP::Adapter).to receive(:open).and_yield(adapter)
allow(adapter).to receive(:users).and_return([])
end
run_rake_task('gitlab:ldap:check')
end
it 'attempts to bind using credentials' do
stub_ldap_config(has_auth?: true)
it 'searches for 100 LDAP users' do
stub_ldap_config(uid: 'uid')
expect(ldap).to receive(:bind)
expect(adapter).to receive(:users).with('uid', '*', 100)
subject
end
run_rake_task('gitlab:ldap:check')
it 'searches for 100 LDAP users' do
stub_ldap_config(uid: 'uid')
expect(adapter).to receive(:users).with('uid', '*', 100)
subject
end
end
end
end