diff --git a/app/services/clusters/applications/base_helm_service.rb b/app/services/clusters/applications/base_helm_service.rb index 270a8eb24f4..e86ca8cf1d0 100644 --- a/app/services/clusters/applications/base_helm_service.rb +++ b/app/services/clusters/applications/base_helm_service.rb @@ -11,6 +11,25 @@ module Clusters protected + def log_error(error) + meta = { + exception: error.class.name, + error_code: error.respond_to?(:error_code) ? error.error_code : nil, + service: self.class.name, + app_id: app.id, + project_ids: app.cluster.project_ids, + group_ids: app.cluster.group_ids, + message: error.message + } + + logger.error(meta) + Gitlab::Sentry.track_acceptable_exception(error, extra: meta) + end + + def logger + @logger ||= Gitlab::Kubernetes::Logger.build + end + def cluster app.cluster end diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb index 6794580e1e8..21ec26ea233 100644 --- a/app/services/clusters/applications/check_installation_progress_service.rb +++ b/app/services/clusters/applications/check_installation_progress_service.rb @@ -15,8 +15,7 @@ module Clusters check_timeout end rescue Kubeclient::HttpError => e - Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}") - Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id }) + log_error(e) app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored? end diff --git a/app/services/clusters/applications/install_service.rb b/app/services/clusters/applications/install_service.rb index f4385748c43..5a65dc4ef59 100644 --- a/app/services/clusters/applications/install_service.rb +++ b/app/services/clusters/applications/install_service.rb @@ -13,12 +13,10 @@ module Clusters ClusterWaitForAppInstallationWorker.perform_in( ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) rescue Kubeclient::HttpError => e - Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}") - Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id }) + log_error(e) app.make_errored!("Kubernetes error: #{e.error_code}") rescue StandardError => e - Rails.logger.error "Can't start installation process: #{e.class.name} #{e.message}" - Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id }) + log_error(e) app.make_errored!("Can't start installation process.") end end diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 7e5a3eb9ccd..698f4caab3a 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec {"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"} ``` +## `kubernetes.log` + +Introduced in GitLab 11.6. This file lives in +`/var/log/gitlab/gitlab-rails/kubernetes.log` for Omnibus GitLab +packages or in `/home/git/gitlab/log/kubernetes.log` for +installations from source. + +It logs information related to the Kubernetes Integration including errors +during installing cluster applications on your GitLab managed Kubernetes +clusters. + +Each line contains a JSON line that can be ingested by Elasticsearch, Splunk, +etc. For example: + +```json +{"severity":"ERROR","time":"2018-11-23T15:14:54.652Z","exception":"Kubeclient::HttpError","error_code":401,"service":"Clusters::Applications::CheckInstallationProgressService","app_id":14,"project_ids":[1],"group_ids":[],"message":"Unauthorized"} +{"severity":"ERROR","time":"2018-11-23T15:42:11.647Z","exception":"Kubeclient::HttpError","error_code":null,"service":"Clusters::Applications::InstallService","app_id":2,"project_ids":[19],"group_ids":[],"message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)"} +``` + ## `githost.log` This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for diff --git a/doc/development/logging.md b/doc/development/logging.md index abd08c420da..5c1d96b9e0c 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -75,7 +75,7 @@ To create a new file: module Import class Logger < ::Gitlab::JsonLogger def self.file_name_noext - 'importer_json' + 'importer' end end end @@ -105,7 +105,7 @@ To create a new file: ```ruby # GOOD - logger.info("Unable to create project", project_id: project.id) + logger.info(message: "Unable to create project", project_id: project.id) ``` 1. Be sure to create a common base structure of your log messages. For example, @@ -118,13 +118,13 @@ To create a new file: ```ruby # BAD - logger.info("Import error", error: 1) - logger.info("Import error", error: "I/O failure") + logger.info(message: "Import error", error: 1) + logger.info(message: "Import error", error: "I/O failure") ``` ```ruby # GOOD - logger.info("Import error", error_code: 1, error: "I/O failure") + logger.info(message: "Import error", error_code: 1, error: "I/O failure") ``` ## Additional steps with new log files diff --git a/lib/gitlab/kubernetes/logger.rb b/lib/gitlab/kubernetes/logger.rb new file mode 100644 index 00000000000..5e59482419b --- /dev/null +++ b/lib/gitlab/kubernetes/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'kubernetes' + end + end + end +end diff --git a/spec/services/clusters/applications/check_installation_progress_service_spec.rb b/spec/services/clusters/applications/check_installation_progress_service_spec.rb index 9452a9e38fb..45b8ce94815 100644 --- a/spec/services/clusters/applications/check_installation_progress_service_spec.rb +++ b/spec/services/clusters/applications/check_installation_progress_service_spec.rb @@ -105,6 +105,12 @@ describe Clusters::Applications::CheckInstallationProgressService do expect(application).to be_errored expect(application.status_reason).to eq('Kubernetes error: 401') end + + it 'should log error' do + expect(service.send(:logger)).to receive(:error) + + service.execute + end end end end diff --git a/spec/services/clusters/applications/install_service_spec.rb b/spec/services/clusters/applications/install_service_spec.rb index 2f801d019fe..018d9822d3e 100644 --- a/spec/services/clusters/applications/install_service_spec.rb +++ b/spec/services/clusters/applications/install_service_spec.rb @@ -33,8 +33,9 @@ describe Clusters::Applications::InstallService do end context 'when k8s cluster communication fails' do + let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) } + before do - error = Kubeclient::HttpError.new(500, 'system failure', nil) expect(helm_client).to receive(:install).with(install_command).and_raise(error) end @@ -44,18 +45,81 @@ describe Clusters::Applications::InstallService do expect(application).to be_errored expect(application.status_reason).to match('Kubernetes error: 500') end + + it 'logs errors' do + expect(service.send(:logger)).to receive(:error).with( + { + exception: 'Kubeclient::HttpError', + message: 'system failure', + service: 'Clusters::Applications::InstallService', + app_id: application.id, + project_ids: application.cluster.project_ids, + group_ids: [], + error_code: 500 + } + ) + + expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with( + error, + extra: { + exception: 'Kubeclient::HttpError', + message: 'system failure', + service: 'Clusters::Applications::InstallService', + app_id: application.id, + project_ids: application.cluster.project_ids, + group_ids: [], + error_code: 500 + } + ) + + service.execute + end end - context 'when application cannot be persisted' do + context 'a non kubernetes error happens' do let(:application) { create(:clusters_applications_helm, :scheduled) } + let(:error) { StandardError.new("something bad happened") } + + before do + expect(application).to receive(:make_installing!).once.and_raise(error) + end it 'make the application errored' do - expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid) expect(helm_client).not_to receive(:install) service.execute expect(application).to be_errored + expect(application.status_reason).to eq("Can't start installation process.") + end + + it 'logs errors' do + expect(service.send(:logger)).to receive(:error).with( + { + exception: 'StandardError', + error_code: nil, + message: 'something bad happened', + service: 'Clusters::Applications::InstallService', + app_id: application.id, + project_ids: application.cluster.projects.pluck(:id), + group_ids: [] + } + ) + + expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with( + error, + extra: { + exception: 'StandardError', + error_code: nil, + message: 'something bad happened', + service: 'Clusters::Applications::InstallService', + app_id: application.id, + project_ids: application.cluster.projects.pluck(:id), + group_ids: [] + } + ) + + service.execute end end end