Merge branch 'json-logging-for-k8s' into 'master'

Json logging for k8s Integration

See merge request gitlab-org/gitlab-ce!23328

app/services/clusters/applications/base_helm_service.rb

@@ -11,6 +11,25 @@ module Clusters
 
       protected
 
+      def log_error(error)
+        meta = {
+          exception: error.class.name,
+          error_code: error.respond_to?(:error_code) ? error.error_code : nil,
+          service: self.class.name,
+          app_id: app.id,
+          project_ids: app.cluster.project_ids,
+          group_ids: app.cluster.group_ids,
+          message: error.message
+        }
+
+        logger.error(meta)
+        Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
+      end
+
       def cluster
         app.cluster
       end

app/services/clusters/applications/check_installation_progress_service.rb

@@ -15,8 +15,7 @@ module Clusters
           check_timeout
         end
       rescue Kubeclient::HttpError => e
-        Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
-        Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+        log_error(e)
         app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored?
       end
 

app/services/clusters/applications/install_service.rb

@@ -13,12 +13,10 @@ module Clusters
           ClusterWaitForAppInstallationWorker.perform_in(
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
         rescue Kubeclient::HttpError => e
-          Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
-          Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+          log_error(e)
           app.make_errored!("Kubernetes error: #{e.error_code}")
         rescue StandardError => e
-          Rails.logger.error "Can't start installation process: #{e.class.name} #{e.message}"
-          Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+          log_error(e)
           app.make_errored!("Can't start installation process.")
         end
       end

doc/administration/logs.md

@@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec
 {"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"}
 ```
 
+## `kubernetes.log`
+
+Introduced in GitLab 11.6. This file lives in
+`/var/log/gitlab/gitlab-rails/kubernetes.log` for Omnibus GitLab
+packages or in `/home/git/gitlab/log/kubernetes.log` for
+installations from source.
+
+It logs information related to the Kubernetes Integration including errors
+during installing cluster applications on your GitLab managed Kubernetes
+clusters.
+
+Each line contains a JSON line that can be ingested by Elasticsearch, Splunk,
+etc. For example:
+
+```json
+{"severity":"ERROR","time":"2018-11-23T15:14:54.652Z","exception":"Kubeclient::HttpError","error_code":401,"service":"Clusters::Applications::CheckInstallationProgressService","app_id":14,"project_ids":[1],"group_ids":[],"message":"Unauthorized"}
+{"severity":"ERROR","time":"2018-11-23T15:42:11.647Z","exception":"Kubeclient::HttpError","error_code":null,"service":"Clusters::Applications::InstallService","app_id":2,"project_ids":[19],"group_ids":[],"message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)"}
+```
+
 ## `githost.log`
 
 This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for

doc/development/logging.md

@@ -75,7 +75,7 @@ To create a new file:
           module Import
             class Logger < ::Gitlab::JsonLogger
               def self.file_name_noext
-                'importer_json'
+                'importer'
               end
             end
            end
@@ -105,7 +105,7 @@ To create a new file:
 
     ```ruby
     # GOOD
-    logger.info("Unable to create project", project_id: project.id)
+    logger.info(message: "Unable to create project", project_id: project.id)
     ```
 
 1. Be sure to create a common base structure of your log messages. For example,
@@ -118,13 +118,13 @@ To create a new file:
 
     ```ruby
     # BAD
-    logger.info("Import error", error: 1)
-    logger.info("Import error", error: "I/O failure")
+    logger.info(message: "Import error", error: 1)
+    logger.info(message: "Import error", error: "I/O failure")
     ```
 
     ```ruby
     # GOOD
-    logger.info("Import error", error_code: 1, error: "I/O failure")
+    logger.info(message: "Import error", error_code: 1, error: "I/O failure")
     ```
 
 ## Additional steps with new log files

lib/gitlab/kubernetes/logger.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class Logger < ::Gitlab::JsonLogger
+      def self.file_name_noext
+        'kubernetes'
+      end
+    end
+  end
+end

spec/services/clusters/applications/check_installation_progress_service_spec.rb

@@ -105,6 +105,12 @@ describe Clusters::Applications::CheckInstallationProgressService do
         expect(application).to be_errored
         expect(application.status_reason).to eq('Kubernetes error: 401')
       end
+
+      it 'should log error' do
+        expect(service.send(:logger)).to receive(:error)
+
+        service.execute
+      end
     end
   end
 end

spec/services/clusters/applications/install_service_spec.rb

@@ -33,8 +33,9 @@ describe Clusters::Applications::InstallService do
     end
 
     context 'when k8s cluster communication fails' do
+      let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
+
       before do
-        error = Kubeclient::HttpError.new(500, 'system failure', nil)
         expect(helm_client).to receive(:install).with(install_command).and_raise(error)
       end
 
@@ -44,18 +45,81 @@ describe Clusters::Applications::InstallService do
         expect(application).to be_errored
         expect(application.status_reason).to match('Kubernetes error: 500')
       end
+
+      it 'logs errors' do
+        expect(service.send(:logger)).to receive(:error).with(
+          {
+            exception: 'Kubeclient::HttpError',
+            message: 'system failure',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.project_ids,
+            group_ids: [],
+            error_code: 500
+          }
+        )
+
+        expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
+          error,
+          extra: {
+            exception: 'Kubeclient::HttpError',
+            message: 'system failure',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.project_ids,
+            group_ids: [],
+            error_code: 500
+          }
+        )
+
+        service.execute
+      end
     end
 
-    context 'when application cannot be persisted' do
+    context 'a non kubernetes error happens' do
       let(:application) { create(:clusters_applications_helm, :scheduled) }
+      let(:error) { StandardError.new("something bad happened") }
+
+      before do
+        expect(application).to receive(:make_installing!).once.and_raise(error)
+      end
 
       it 'make the application errored' do
-        expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid)
         expect(helm_client).not_to receive(:install)
 
         service.execute
 
         expect(application).to be_errored
+        expect(application.status_reason).to eq("Can't start installation process.")
+      end
+
+      it 'logs errors' do
+        expect(service.send(:logger)).to receive(:error).with(
+          {
+            exception: 'StandardError',
+            error_code: nil,
+            message: 'something bad happened',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.projects.pluck(:id),
+            group_ids: []
+          }
+        )
+
+        expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
+          error,
+          extra: {
+            exception: 'StandardError',
+            error_code: nil,
+            message: 'something bad happened',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.projects.pluck(:id),
+            group_ids: []
+          }
+        )
+
+        service.execute
       end
     end
   end