Commit 681d7139 authored by Stan Hu's avatar Stan Hu

Merge branch 'json-logging-for-k8s' into 'master'

Json logging for k8s Integration

See merge request gitlab-org/gitlab-ce!23328
parents 17acf40d 7e7fb6de
...@@ -11,6 +11,25 @@ module Clusters ...@@ -11,6 +11,25 @@ module Clusters
protected protected
def log_error(error)
meta = {
exception: error.class.name,
error_code: error.respond_to?(:error_code) ? error.error_code : nil,
service: self.class.name,
app_id: app.id,
project_ids: app.cluster.project_ids,
group_ids: app.cluster.group_ids,
message: error.message
}
logger.error(meta)
Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def cluster def cluster
app.cluster app.cluster
end end
......
...@@ -15,8 +15,7 @@ module Clusters ...@@ -15,8 +15,7 @@ module Clusters
check_timeout check_timeout
end end
rescue Kubeclient::HttpError => e rescue Kubeclient::HttpError => e
Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}") log_error(e)
Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored? app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored?
end end
......
...@@ -13,12 +13,10 @@ module Clusters ...@@ -13,12 +13,10 @@ module Clusters
ClusterWaitForAppInstallationWorker.perform_in( ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e rescue Kubeclient::HttpError => e
Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}") log_error(e)
Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
app.make_errored!("Kubernetes error: #{e.error_code}") app.make_errored!("Kubernetes error: #{e.error_code}")
rescue StandardError => e rescue StandardError => e
Rails.logger.error "Can't start installation process: #{e.class.name} #{e.message}" log_error(e)
Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
app.make_errored!("Can't start installation process.") app.make_errored!("Can't start installation process.")
end end
end end
......
...@@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec ...@@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec
{"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"} {"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"}
``` ```
## `kubernetes.log`
Introduced in GitLab 11.6. This file lives in
`/var/log/gitlab/gitlab-rails/kubernetes.log` for Omnibus GitLab
packages or in `/home/git/gitlab/log/kubernetes.log` for
installations from source.
It logs information related to the Kubernetes Integration including errors
during installing cluster applications on your GitLab managed Kubernetes
clusters.
Each line contains a JSON line that can be ingested by Elasticsearch, Splunk,
etc. For example:
```json
{"severity":"ERROR","time":"2018-11-23T15:14:54.652Z","exception":"Kubeclient::HttpError","error_code":401,"service":"Clusters::Applications::CheckInstallationProgressService","app_id":14,"project_ids":[1],"group_ids":[],"message":"Unauthorized"}
{"severity":"ERROR","time":"2018-11-23T15:42:11.647Z","exception":"Kubeclient::HttpError","error_code":null,"service":"Clusters::Applications::InstallService","app_id":2,"project_ids":[19],"group_ids":[],"message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)"}
```
## `githost.log` ## `githost.log`
This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for
......
...@@ -75,7 +75,7 @@ To create a new file: ...@@ -75,7 +75,7 @@ To create a new file:
module Import module Import
class Logger < ::Gitlab::JsonLogger class Logger < ::Gitlab::JsonLogger
def self.file_name_noext def self.file_name_noext
'importer_json' 'importer'
end end
end end
end end
...@@ -105,7 +105,7 @@ To create a new file: ...@@ -105,7 +105,7 @@ To create a new file:
```ruby ```ruby
# GOOD # GOOD
logger.info("Unable to create project", project_id: project.id) logger.info(message: "Unable to create project", project_id: project.id)
``` ```
1. Be sure to create a common base structure of your log messages. For example, 1. Be sure to create a common base structure of your log messages. For example,
...@@ -118,13 +118,13 @@ To create a new file: ...@@ -118,13 +118,13 @@ To create a new file:
```ruby ```ruby
# BAD # BAD
logger.info("Import error", error: 1) logger.info(message: "Import error", error: 1)
logger.info("Import error", error: "I/O failure") logger.info(message: "Import error", error: "I/O failure")
``` ```
```ruby ```ruby
# GOOD # GOOD
logger.info("Import error", error_code: 1, error: "I/O failure") logger.info(message: "Import error", error_code: 1, error: "I/O failure")
``` ```
## Additional steps with new log files ## Additional steps with new log files
......
# frozen_string_literal: true
module Gitlab
module Kubernetes
class Logger < ::Gitlab::JsonLogger
def self.file_name_noext
'kubernetes'
end
end
end
end
...@@ -105,6 +105,12 @@ describe Clusters::Applications::CheckInstallationProgressService do ...@@ -105,6 +105,12 @@ describe Clusters::Applications::CheckInstallationProgressService do
expect(application).to be_errored expect(application).to be_errored
expect(application.status_reason).to eq('Kubernetes error: 401') expect(application.status_reason).to eq('Kubernetes error: 401')
end end
it 'should log error' do
expect(service.send(:logger)).to receive(:error)
service.execute
end
end end
end end
end end
...@@ -33,8 +33,9 @@ describe Clusters::Applications::InstallService do ...@@ -33,8 +33,9 @@ describe Clusters::Applications::InstallService do
end end
context 'when k8s cluster communication fails' do context 'when k8s cluster communication fails' do
let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
before do before do
error = Kubeclient::HttpError.new(500, 'system failure', nil)
expect(helm_client).to receive(:install).with(install_command).and_raise(error) expect(helm_client).to receive(:install).with(install_command).and_raise(error)
end end
...@@ -44,18 +45,81 @@ describe Clusters::Applications::InstallService do ...@@ -44,18 +45,81 @@ describe Clusters::Applications::InstallService do
expect(application).to be_errored expect(application).to be_errored
expect(application.status_reason).to match('Kubernetes error: 500') expect(application.status_reason).to match('Kubernetes error: 500')
end end
it 'logs errors' do
expect(service.send(:logger)).to receive(:error).with(
{
exception: 'Kubeclient::HttpError',
message: 'system failure',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.project_ids,
group_ids: [],
error_code: 500
}
)
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
error,
extra: {
exception: 'Kubeclient::HttpError',
message: 'system failure',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.project_ids,
group_ids: [],
error_code: 500
}
)
service.execute
end
end end
context 'when application cannot be persisted' do context 'a non kubernetes error happens' do
let(:application) { create(:clusters_applications_helm, :scheduled) } let(:application) { create(:clusters_applications_helm, :scheduled) }
let(:error) { StandardError.new("something bad happened") }
before do
expect(application).to receive(:make_installing!).once.and_raise(error)
end
it 'make the application errored' do it 'make the application errored' do
expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid)
expect(helm_client).not_to receive(:install) expect(helm_client).not_to receive(:install)
service.execute service.execute
expect(application).to be_errored expect(application).to be_errored
expect(application.status_reason).to eq("Can't start installation process.")
end
it 'logs errors' do
expect(service.send(:logger)).to receive(:error).with(
{
exception: 'StandardError',
error_code: nil,
message: 'something bad happened',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.projects.pluck(:id),
group_ids: []
}
)
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
error,
extra: {
exception: 'StandardError',
error_code: nil,
message: 'something bad happened',
service: 'Clusters::Applications::InstallService',
app_id: application.id,
project_ids: application.cluster.projects.pluck(:id),
group_ids: []
}
)
service.execute
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment