From be34c53ed50392fa3342e17cfb33504e166805ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pascal=20Pfl=C3=BCger?= <pascal.pflueger@t-systems.com>
Date: Mon, 24 Jun 2024 10:39:46 +0200
Subject: [PATCH 1/7] add CODEOWNERS
---
.gitlab/CODEOWNERS | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS
index 8cc70c3..4b9b26a 100644
--- a/.gitlab/CODEOWNERS
+++ b/.gitlab/CODEOWNERS
@@ -1 +1 @@
-* @simpl/simpl-open/development/monitoring @n00bagqb
+* @simpl/simpl-open/development/monitoring @n00bagqb
\ No newline at end of file
--
GitLab
From 05f3dd6d54828b3db761dee27784b10c303daa02 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Tue, 19 Nov 2024 15:17:52 +0100
Subject: [PATCH 2/7] logstash and filebeat adjustments
---
charts/values/dev/observability/values.yaml | 170 ++++++++++----------
1 file changed, 81 insertions(+), 89 deletions(-)
diff --git a/charts/values/dev/observability/values.yaml b/charts/values/dev/observability/values.yaml
index 1114e68..962a0bb 100644
--- a/charts/values/dev/observability/values.yaml
+++ b/charts/values/dev/observability/values.yaml
@@ -110,17 +110,48 @@ logstash:
}
filter: |-
filter {
- if [kubernetes][container][name] == "ejbca-community-helm" {
- grok {
- match => {
- "message" => [
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
- ]
- }
- overwrite => [ "message" ]
+ ## removing ELK logs
+ if [kubernetes][container][name] == "filebeat" or [kubernetes][container][name] == "metricbeat" or [kubernetes][container][name] == "logstash" or [kubernetes][container][name] == "heartbeat" or [kubernetes][container][name] == "kibana" or [kubernetes][container][name] == "elasticsearch" {
+ drop { }
+ }
+
+ if [kubernetes][container][name] == "sd-creation-wizard-api" or [kubernetes][container][name] == "signer" or [kubernetes][container][name] == "sd-creation-wizard-api-validation" or [kubernetes][container][name] == "simpl-cloud-gateway" {
+ json {
+ source => "message"
+ skip_on_invalid_json => true
+ }
+ }
+
+ if [kubernetes][container][name] == "users-roles" {
+
+ json {
+ source => "message"
+ skip_on_invalid_json => true
+ }
+
+
+ ruby {
+ code => '
+ if event.get("[message]").is_a?(Hash)
+ event.set("is_json_message", true)
+ else
+ event.set("is_json_message", false)
+ end
+ '
}
+
+ if [is_json_message] {
+ if [message][httpStatus] { mutate { add_field => { "httpStatus" => "%{[message][httpStatus]}" } } }
+ if [message][msg] { mutate { add_field => { "msg" => "%{[message][msg]}" } } }
+ if [message][httpRequestSize] { mutate { add_field => { "httpRequestSize" => "%{[message][httpRequestSize]}" } } }
+ if [message][user] { mutate { add_field => { "user" => "%{[message][user]}" } } }
+ if [message][httpExecutionTime] { mutate { add_field => { "httpExecutionTime" => "%{[message][httpExecutionTime]}" } } }
+
+ mutate { remove_field => [ "[message]" ] }
+
+ }
}
+
if [kubernetes][container][name] == "keycloak" {
grok {
match => {
@@ -131,21 +162,7 @@ logstash:
overwrite => [ "message" ]
}
}
- if [kubernetes][container][name] == "onboarding" {
- grok {
- pattern_definitions => { "JAVA" => "[0-9A-Za-z\[\]\.\$]*" }
- match => {
- "message" => [
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}HTTP%{SPACE}%{WORD:http_method}%{SPACE}"%{DATA:uri}"',
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}',
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{DATA:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}%{GREEDYDATA:message}'
- ]
- }
- overwrite => [ "message" ]
- }
- }
+
if [kubernetes][container][name] == "postgresql" {
grok {
match => {
@@ -156,67 +173,17 @@ logstash:
overwrite => [ "message" ]
}
}
- if [kubernetes][container][name] == "vault" or [kubernetes][container][name] == "vault-agent-init" or [kubernetes][container][name] == "sidecar-injector" {
- grok {
- match => {
- "message" => [
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{DATA:handler}:%{SPACE}%{GREEDYDATA:message}'
-
-
- ]
- }
- overwrite => [ "message" ]
- }
- }
- if [kubernetes][container][name] == "simpl-cloud-gateway" or [kubernetes][container][name] == "users-roles" {
- grok {
- match => {
- "message" => [
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}'
- ]
- }
- overwrite => [ "message" ]
- }
- }
- if [kubernetes][container][name] == "neo4j" {
- grok {
- match => {
- "message" => [
- '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{GREEDYDATA:message}'
- ]
- }
- overwrite => [ "message" ]
- }
- }
- if [kubernetes][container][name] == "redis" {
- grok {
- match => {
- "message" => [
- '%{NUMBER:process_id}:%{WORD:process_type}%{SPACE}%{MONTHDAY:day}%{SPACE}%{MONTH:month}%{SPACE}%{YEAR:year}%{SPACE}%{TIME:time}\.%{INT:milliseconds}%{SPACE}\*%{SPACE}%{GREEDYDATA:message}'
- ]
- }
- overwrite => [ "message" ]
- add_field => {
- "timestamp" => "%{day} %{month} %{year} %{time}.%{milliseconds}"
- }
- }
-
- }
-
- if [fields][logtype] == "logs-sample-business" {
- grok {
- match => { "message" => '%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:origin}\|%{WORD:destination}\|%{WORD:business_operation}\|%{DATA:message_type}\|%{WORD:correlation_id}' }
- }
- }
-
- date {
- match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS" ]
- }
+ date {
+ match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"]
+ }
+ date {
+ match => [ "ts", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"]
+ }
}
output: |-
output {
- if [fields][logtype] == "logs-sample-business" {
+ if [kubernetes][container][name] == "simpl-cloud-gateway" {
elasticsearch {
hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
user => "${LOGSTASH_USER}"
@@ -388,34 +355,59 @@ filebeat4agents:
filebeat.autodiscover:
providers:
- type: kubernetes
+ # Filter logs only from the monitored namespace
+ namespace: "${MONITORED_NAMESPACE}"
templates:
+ # Condition for redis container in the monitored namespace
- condition:
- or:
- - equals:
- kubernetes.namespace: "${MONITORED_NAMESPACE}"
+ equals:
+ kubernetes.container.name: "redis"
config:
- type: container
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
multiline:
- type: pattern
- pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+ pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}'
negate: true
match: after
+ # Condition for json structured logs
- condition:
- equals:
- kubernetes.container.name: "redis"
+ or:
+ - equals:
+ kubernetes.container.name: "users-roles"
+ - equals:
+ kubernetes.container.name: "signer"
+ - equals:
+ kubernetes.container.name: "sd-creation-wizard-api"
+ - equals:
+ kubernetes.container.name: "sd-creation-wizard-api-validation"
+ - equals:
+ kubernetes.container.name: "simpl-cloud-gateway"
+ config:
+ - type: container
+ paths:
+ - /var/log/containers/*-${data.kubernetes.container.id}.log
+ # Condition for plain text logs
+ - condition:
+ or:
+ - equals:
+ kubernetes.container.name: "keycloak"
+ - equals:
+ kubernetes.container.name: "postgresql"
config:
- type: container
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
multiline:
- pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}'
+ type: pattern
+ pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
processors:
+ # Add cloud and host metadata
- add_cloud_metadata: {}
- add_host_metadata: {}
+
output: |
output.logstash:
hosts: ["${LOGSTASH_HOSTS}"]
--
GitLab
From 820320e839db494f0fb0f8883f49c3e24577b4bd Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Tue, 19 Nov 2024 15:39:06 +0100
Subject: [PATCH 3/7] version fix
---
charts/Chart.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/charts/Chart.yaml b/charts/Chart.yaml
index f21aa4e..c36e45d 100644
--- a/charts/Chart.yaml
+++ b/charts/Chart.yaml
@@ -1,6 +1,6 @@
name: eck-monitoring
-version: ${PROJECT_RELEASE_VERSION}
-appVersion: "${PROJECT_RELEASE_VERSION}"
-#version: 0.1.0
+#version: ${PROJECT_RELEASE_VERSION}
+#appVersion: "${PROJECT_RELEASE_VERSION}"
+version: 0.1.3
--
GitLab
From f2cb50a5abc0fc59cdc72bd53943b52314a78e19 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 20 Nov 2024 15:47:50 +0100
Subject: [PATCH 4/7] cpu request decreased
---
charts/values/dev/observability/values.yaml | 1 +
1 file changed, 1 insertion(+)
diff --git a/charts/values/dev/observability/values.yaml b/charts/values/dev/observability/values.yaml
index 962a0bb..ddf7828 100644
--- a/charts/values/dev/observability/values.yaml
+++ b/charts/values/dev/observability/values.yaml
@@ -28,6 +28,7 @@ elasticsearch:
resources:
requests:
memory: 4Gi
+ cpu: 300m
limits:
memory: 4Gi
cpu: "1"
--
GitLab
From 43f31b9d92bdc6fe5f757b69091efbfebf1113df Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 20 Nov 2024 16:27:44 +0100
Subject: [PATCH 5/7] kibana cert change
---
charts/templates/kibana.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/charts/templates/kibana.yaml b/charts/templates/kibana.yaml
index 1de015d..77a6bf0 100644
--- a/charts/templates/kibana.yaml
+++ b/charts/templates/kibana.yaml
@@ -74,7 +74,7 @@ spec:
http:
tls:
certificate:
- secretName: {{ .Release.Name }}-kibana-cert-secret
+ secretName: {{ .Release.Name }}-kibana-ssl
---
apiVersion: networking.k8s.io/v1
kind: Ingress
--
GitLab
From 49543543ef63e185f104fe495f2ac7fdbb63845b Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 21 Nov 2024 11:10:49 +0100
Subject: [PATCH 6/7] ready tu develop push
---
charts/Chart.yaml | 6 +++---
pipeline.variables.sh | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/charts/Chart.yaml b/charts/Chart.yaml
index c36e45d..dc19d20 100644
--- a/charts/Chart.yaml
+++ b/charts/Chart.yaml
@@ -1,6 +1,6 @@
name: eck-monitoring
-#version: ${PROJECT_RELEASE_VERSION}
-#appVersion: "${PROJECT_RELEASE_VERSION}"
-version: 0.1.3
+version: ${PROJECT_RELEASE_VERSION}
+appVersion: "${PROJECT_RELEASE_VERSION}"
+#version: 0.1.3
diff --git a/pipeline.variables.sh b/pipeline.variables.sh
index 3564d00..ecdeeed 100644
--- a/pipeline.variables.sh
+++ b/pipeline.variables.sh
@@ -1 +1 @@
-PROJECT_VERSION_NUMBER="0.1.2"
\ No newline at end of file
+PROJECT_VERSION_NUMBER="3"
\ No newline at end of file
--
GitLab
From 43098b5c941a0d200b16846b7ecb2206186750bd Mon Sep 17 00:00:00 2001
From: Albert Brzozowski <albert.brzozowski.external@atos.net>
Date: Thu, 21 Nov 2024 11:16:33 +0100
Subject: [PATCH 7/7] Fixes
---
charts/values/dev/observability/values.yaml | 2 +-
pipeline.variables.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/charts/values/dev/observability/values.yaml b/charts/values/dev/observability/values.yaml
index ddf7828..df345e9 100644
--- a/charts/values/dev/observability/values.yaml
+++ b/charts/values/dev/observability/values.yaml
@@ -37,7 +37,7 @@ kibana:
count: 1
image: docker.elastic.co/kibana/kibana
#Branch name to donwload dashboards
- dashboardsBranch: "develop"
+ dashboardsBranch: "main"
# Kibana's image tag, by default it equals to elasticVersion
imageTag: ""
# name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
diff --git a/pipeline.variables.sh b/pipeline.variables.sh
index ecdeeed..322e00a 100644
--- a/pipeline.variables.sh
+++ b/pipeline.variables.sh
@@ -1 +1 @@
-PROJECT_VERSION_NUMBER="3"
\ No newline at end of file
+PROJECT_VERSION_NUMBER="0.1.3"
\ No newline at end of file
--
GitLab