diff --git a/CHANGELOG b/CHANGELOG index efb6dc6f610..ed4b1a12395 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -14,6 +14,7 @@ v 8.8.0 (unreleased) - Reduce delay in destroying a project from 1-minute to immediately - Make build status canceled if any of the jobs was canceled and none failed - Upgrade Sidekiq to 4.1.2 + - Added /health_check endpoint for checking service status - Sanitize repo paths in new project error message - Bump mail_room to 0.7.0 to fix stuck IDLE connections - Remove future dates from contribution calendar graph. diff --git a/Gemfile b/Gemfile index 2854bf8a57b..96d9decb261 100644 --- a/Gemfile +++ b/Gemfile @@ -332,3 +332,6 @@ gem 'oauth2', '~> 1.0.0' # Soft deletion gem "paranoia", "~> 2.0" + +# Health check +gem 'health_check', '~> 1.5.1' diff --git a/Gemfile.lock b/Gemfile.lock index bc47533e5bb..ce1da92e517 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -402,6 +402,8 @@ GEM html2haml (>= 1.0.1) railties (>= 4.0.1) hashie (3.4.3) + health_check (1.5.1) + rails (>= 2.3.0) highline (1.7.8) hipchat (1.5.2) httparty @@ -945,6 +947,7 @@ DEPENDENCIES grape (~> 0.13.0) grape-entity (~> 0.4.2) haml-rails (~> 0.9.0) + health_check (~> 1.5.1) hipchat (~> 1.5.0) html-pipeline (~> 1.11.0) httparty (~> 0.13.3) diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index b9eb7ae7921..8c973f0e4a8 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -19,6 +19,12 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController redirect_to admin_runners_path end + def reset_health_check_token + @application_setting.reset_health_check_access_token! + flash[:notice] = 'New health check access token has been generated!' + redirect_to :back + end + def clear_repository_check_states RepositoryCheck::ClearWorker.perform_async diff --git a/app/controllers/admin/health_check_controller.rb b/app/controllers/admin/health_check_controller.rb new file mode 100644 index 00000000000..241c7be0ea1 --- /dev/null +++ b/app/controllers/admin/health_check_controller.rb @@ -0,0 +1,5 @@ +class Admin::HealthCheckController < Admin::ApplicationController + def show + @errors = HealthCheck::Utils.process_checks('standard') + end +end diff --git a/app/controllers/health_check_controller.rb b/app/controllers/health_check_controller.rb new file mode 100644 index 00000000000..037da7d2bce --- /dev/null +++ b/app/controllers/health_check_controller.rb @@ -0,0 +1,22 @@ +class HealthCheckController < HealthCheck::HealthCheckController + before_action :validate_health_check_access! + + private + + def validate_health_check_access! + render_404 unless token_valid? + end + + def token_valid? + token = params[:token].presence || request.headers['TOKEN'] + token.present? && + ActiveSupport::SecurityUtils.variable_size_secure_compare( + token, + current_application_settings.health_check_access_token + ) + end + + def render_404 + render file: Rails.root.join('public', '404'), layout: false, status: '404' + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c143cf215e6..1a10768655f 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -1,6 +1,7 @@ class ApplicationSetting < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :runners_registration_token + add_authentication_token_field :health_check_access_token CACHE_KEY = 'application_setting.last' @@ -81,6 +82,7 @@ class ApplicationSetting < ActiveRecord::Base end before_save :ensure_runners_registration_token + before_save :ensure_health_check_access_token after_commit do Rails.cache.write(CACHE_KEY, self) @@ -145,4 +147,8 @@ class ApplicationSetting < ActiveRecord::Base def runners_registration_token ensure_runners_registration_token! end + + def health_check_access_token + ensure_health_check_access_token! + end end diff --git a/app/views/admin/health_check/show.html.haml b/app/views/admin/health_check/show.html.haml new file mode 100644 index 00000000000..ad79fd26d0b --- /dev/null +++ b/app/views/admin/health_check/show.html.haml @@ -0,0 +1,49 @@ +- page_title "Health Check" + +%h3.page-title + Health Check +.bs-callout.clearfix + .pull-left + %p + Access token is + %code#health-check-token= current_application_settings.health_check_access_token + = button_to reset_health_check_token_admin_application_settings_path, + method: :put, class: 'btn btn-default', + data: { confirm: 'Are you sure you want to reset the health check token?' } do + = icon('refresh') + Reset health check access token +%p.light + Health information can be reteived as plain text, json, or xml using: + %ul + %li + %code= health_check_url(token: current_application_settings.health_check_access_token) + %li + %code= health_check_url(token: current_application_settings.health_check_access_token, format: :json) + %li + %code= health_check_url(token: current_application_settings.health_check_access_token, format: :xml) + +%p.light + You can also ask for the status of specific services: + %ul + %li + %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :cache) + %li + %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :database) + %li + %code= health_check_url(token: current_application_settings.health_check_access_token, checks: :migrations) + +%hr +.panel.panel-default + .panel-heading + Current Status: + - if @errors.blank? + = icon('circle', class: 'cgreen') + Healthy + - else + = icon('warning', class: 'cred') + Unhealthy + .panel-body + - if @errors.blank? + No Health Problems Detected + - else + = @errors diff --git a/app/views/layouts/nav/_admin.html.haml b/app/views/layouts/nav/_admin.html.haml index 280a1b93729..f292730fe45 100644 --- a/app/views/layouts/nav/_admin.html.haml +++ b/app/views/layouts/nav/_admin.html.haml @@ -41,6 +41,11 @@ = icon('file-text fw') %span Logs + = nav_link(controller: :health_check) do + = link_to admin_health_check_path, title: 'Health Check' do + = icon('medkit fw') + %span + Health Check = nav_link(controller: :broadcast_messages) do = link_to admin_broadcast_messages_path, title: 'Messages' do = icon('bullhorn fw') diff --git a/config/initializers/health_check.rb b/config/initializers/health_check.rb new file mode 100644 index 00000000000..79e2d23ab2e --- /dev/null +++ b/config/initializers/health_check.rb @@ -0,0 +1,3 @@ +HealthCheck.setup do |config| + config.standard_checks = ['database', 'migrations', 'cache'] +end diff --git a/config/routes.rb b/config/routes.rb index dd946305181..ec876eba345 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -73,6 +73,9 @@ Rails.application.routes.draw do mount Sidekiq::Web, at: '/admin/sidekiq', as: :sidekiq end + # Health check + get 'health_check(/:checks)' => 'health_check#index', as: :health_check + # Enable Grack support mount Grack::AuthSpawner, at: '/', constraints: lambda { |request| /[-\/\w\.]+\.git\//.match(request.path_info) }, via: [:get, :post, :put] @@ -254,6 +257,7 @@ Rails.application.routes.draw do end resource :logs, only: [:show] + resource :health_check, controller: 'health_check', only: [:show] resource :background_jobs, controller: 'background_jobs', only: [:show] resources :namespaces, path: '/projects', constraints: { id: /[a-zA-Z.0-9_\-]+/ }, only: [] do @@ -285,6 +289,7 @@ Rails.application.routes.draw do resource :application_settings, only: [:show, :update] do resources :services put :reset_runners_token + put :reset_health_check_token put :clear_repository_check_states end diff --git a/db/migrate/20160509201028_add_health_check_access_token_to_application_settings.rb b/db/migrate/20160509201028_add_health_check_access_token_to_application_settings.rb new file mode 100644 index 00000000000..9d729fec189 --- /dev/null +++ b/db/migrate/20160509201028_add_health_check_access_token_to_application_settings.rb @@ -0,0 +1,5 @@ +class AddHealthCheckAccessTokenToApplicationSettings < ActiveRecord::Migration + def change + add_column :application_settings, :health_check_access_token, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 74facd12088..9b5aa640cb0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20160508194200) do +ActiveRecord::Schema.define(version: 20160509201028) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -81,6 +81,7 @@ ActiveRecord::Schema.define(version: 20160508194200) do t.text "shared_runners_text" t.integer "metrics_packet_size", default: 1 t.text "disabled_oauth_sign_in_sources" + t.string "health_check_access_token" end create_table "audit_events", force: :cascade do |t| diff --git a/spec/controllers/health_check_controller_spec.rb b/spec/controllers/health_check_controller_spec.rb new file mode 100644 index 00000000000..0d8a68bb51a --- /dev/null +++ b/spec/controllers/health_check_controller_spec.rb @@ -0,0 +1,105 @@ +require 'spec_helper' + +describe HealthCheckController do + let(:token) { current_application_settings.health_check_access_token } + let(:json_response) { JSON.parse(response.body) } + let(:xml_response) { Hash.from_xml(response.body)['hash'] } + + describe 'GET #index' do + context 'when services are up but NO access token' do + it 'returns a not found page' do + get :index + expect(response).to be_not_found + end + end + + context 'when services are up and an access token is provided' do + it 'supports passing the token in the header' do + request.headers['TOKEN'] = token + get :index + expect(response).to be_success + expect(response.content_type).to eq 'text/plain' + end + + it 'supports successful plaintest response' do + get :index, token: token + expect(response).to be_success + expect(response.content_type).to eq 'text/plain' + end + + it 'supports successful json response' do + get :index, token: token, format: :json + expect(response).to be_success + expect(response.content_type).to eq 'application/json' + expect(json_response['healthy']).to be true + end + + it 'supports successful xml response' do + get :index, token: token, format: :xml + expect(response).to be_success + expect(response.content_type).to eq 'application/xml' + expect(xml_response['healthy']).to be true + end + + it 'supports successful responses for specific checks' do + get :index, token: token, checks: 'email', format: :json + expect(response).to be_success + expect(response.content_type).to eq 'application/json' + expect(json_response['healthy']).to be true + end + end + + context 'when a service is down but NO access token' do + it 'returns a not found page' do + get :index + expect(response).to be_not_found + end + end + + context 'when a service is down and an access token is provided' do + before do + allow(HealthCheck::Utils).to receive(:process_checks).with('standard').and_return('The server is on fire') + allow(HealthCheck::Utils).to receive(:process_checks).with('email').and_return('Email is on fire') + end + + it 'supports passing the token in the header' do + request.headers['TOKEN'] = token + get :index + expect(response.status).to eq(500) + expect(response.content_type).to eq 'text/plain' + expect(response.body).to include('The server is on fire') + end + + it 'supports failure plaintest response' do + get :index, token: token + expect(response.status).to eq(500) + expect(response.content_type).to eq 'text/plain' + expect(response.body).to include('The server is on fire') + end + + it 'supports failure json response' do + get :index, token: token, format: :json + expect(response.status).to eq(500) + expect(response.content_type).to eq 'application/json' + expect(json_response['healthy']).to be false + expect(json_response['message']).to include('The server is on fire') + end + + it 'supports failure xml response' do + get :index, token: token, format: :xml + expect(response.status).to eq(500) + expect(response.content_type).to eq 'application/xml' + expect(xml_response['healthy']).to be false + expect(xml_response['message']).to include('The server is on fire') + end + + it 'supports failure responses for specific checks' do + get :index, token: token, checks: 'email', format: :json + expect(response.status).to eq(500) + expect(response.content_type).to eq 'application/json' + expect(json_response['healthy']).to be false + expect(json_response['message']).to include('Email is on fire') + end + end + end +end diff --git a/spec/features/admin/admin_health_check_spec.rb b/spec/features/admin/admin_health_check_spec.rb new file mode 100644 index 00000000000..4fde04b609b --- /dev/null +++ b/spec/features/admin/admin_health_check_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +feature "Admin Health Check", feature: true do + include WaitForAjax + + before do + login_as :admin + end + + describe '#show' do + before do + visit admin_health_check_path + end + + it { page.has_text? 'Health Check' } + it { page.has_text? 'Health information can be reteived' } + + it 'has a health check access token' do + token = current_application_settings.health_check_access_token + expect(page).to have_content("Access token is #{token}") + expect(page).to have_selector('#health-check-token', text: token) + end + + describe 'reload access token', js: true do + it 'changes the access token' do + orig_token = current_application_settings.health_check_access_token + click_button 'Reset health check access token' + wait_for_ajax + expect(find('#health-check-token').text).not_to eq orig_token + end + end + end + + context 'when services are up' do + before do + visit admin_health_check_path + end + + it 'shows healthy status' do + expect(page).to have_content('Current Status: Healthy') + end + end + + context 'when a service is down' do + before do + allow(HealthCheck::Utils).to receive(:process_checks).and_return('The server is on fire') + visit admin_health_check_path + end + + it 'shows unhealthy status' do + expect(page).to have_content('Current Status: Unhealthy') + expect(page).to have_content('The server is on fire') + end + end +end diff --git a/spec/routing/admin_routing_spec.rb b/spec/routing/admin_routing_spec.rb index cd16a8e6322..b5ed8584c8a 100644 --- a/spec/routing/admin_routing_spec.rb +++ b/spec/routing/admin_routing_spec.rb @@ -118,3 +118,10 @@ describe Admin::DashboardController, "routing" do expect(get("/admin")).to route_to('admin/dashboard#index') end end + +# admin_health_check GET /admin/health_check(.:format) admin/health_check#show +describe Admin::HealthCheckController, "routing" do + it "to #show" do + expect(get("/admin/health_check")).to route_to('admin/health_check#show') + end +end diff --git a/spec/routing/routing_spec.rb b/spec/routing/routing_spec.rb index 8530a2f31d5..de13c0db5d1 100644 --- a/spec/routing/routing_spec.rb +++ b/spec/routing/routing_spec.rb @@ -276,3 +276,13 @@ describe "Groups", "routing" do expect(get('/1')).to route_to('namespaces#show', id: '1') end end + +describe HealthCheckController, 'routing' do + it 'to #index' do + expect(get('/health_check')).to route_to('health_check#index') + end + + it 'also supports passing checks in the url' do + expect(get('/health_check/email')).to route_to('health_check#index', checks: 'email') + end +end