Develop

.gitlab/CODEOWNERS

-* @simpl/simpl-open/development/monitoring @n00bagqb
+* @simpl/simpl-open/development/monitoring @n00bagqb
\ No newline at end of file

charts/Chart.yaml

 name: eck-monitoring
 version: ${PROJECT_RELEASE_VERSION}
 appVersion: "${PROJECT_RELEASE_VERSION}"
-#version: 0.1.0
+#version: 0.1.3
 
 

charts/templates/kibana.yaml

@@ -74,7 +74,7 @@ spec:
   http:
     tls:
       certificate:
-        secretName: {{ .Release.Name }}-kibana-cert-secret
+        secretName: {{ .Release.Name }}-kibana-ssl
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress

charts/values/dev/observability/values.yaml

@@ -28,6 +28,7 @@ elasticsearch:
   resources:
     requests:
       memory: 4Gi
+      cpu: 300m
     limits:
       memory: 4Gi
       cpu: "1"
@@ -36,7 +37,7 @@ kibana:
   count: 1
   image: docker.elastic.co/kibana/kibana
   #Branch name to donwload dashboards
-  dashboardsBranch: "develop"
+  dashboardsBranch: "main"
   # Kibana's image tag, by default it equals to elasticVersion
   imageTag: ""
   # name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
@@ -110,17 +111,48 @@ logstash:
         }
       filter: |-
         filter {
-          if [kubernetes][container][name] == "ejbca-community-helm" {
-            grok {
-              match => { 
-                "message" => [
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}', 
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
-                  ] 
-              }
-              overwrite => [ "message" ]
+          ## removing ELK logs
+          if [kubernetes][container][name] == "filebeat" or [kubernetes][container][name] == "metricbeat" or [kubernetes][container][name] == "logstash" or [kubernetes][container][name] == "heartbeat"  or [kubernetes][container][name] == "kibana" or [kubernetes][container][name] == "elasticsearch" {
+            drop { }
+          }    
+          
+          if [kubernetes][container][name] == "sd-creation-wizard-api" or [kubernetes][container][name] == "signer" 	or [kubernetes][container][name] == "sd-creation-wizard-api-validation" or [kubernetes][container][name] == "simpl-cloud-gateway"  {
+            json {
+                    source => "message"
+                    skip_on_invalid_json => true
+                }
+          }
+              
+          if [kubernetes][container][name] == "users-roles" {
+
+            json {
+                    source => "message"
+                    skip_on_invalid_json => true
+                }
+
+
+            ruby {
+                code => '
+                    if event.get("[message]").is_a?(Hash)
+                        event.set("is_json_message", true)
+                    else
+                        event.set("is_json_message", false)
+                    end
+                '
             }
+              
+            if [is_json_message] {
+              if [message][httpStatus] { mutate { add_field => { "httpStatus" => "%{[message][httpStatus]}" } } }
+              if [message][msg] { mutate { add_field => { "msg" => "%{[message][msg]}" } } }
+              if [message][httpRequestSize] { mutate { add_field => { "httpRequestSize" => "%{[message][httpRequestSize]}" } } }
+              if [message][user] { mutate { add_field => { "user" => "%{[message][user]}" } } }
+              if [message][httpExecutionTime] { mutate { add_field => { "httpExecutionTime" => "%{[message][httpExecutionTime]}" } } }
+              
+              mutate { remove_field => [ "[message]" ] }
+              
+            } 
           }
+
           if [kubernetes][container][name] == "keycloak" {
             grok {
               match => { 
@@ -131,21 +163,7 @@ logstash:
               overwrite => [ "message" ]
             }
           }
-          if [kubernetes][container][name] == "onboarding" {
-            grok {
-              pattern_definitions => { "JAVA" => "[0-9A-Za-z\[\]\.\$]*" }
-              match => { 
-                "message" => [
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}', 
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}HTTP%{SPACE}%{WORD:http_method}%{SPACE}"%{DATA:uri}"',
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}',
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{DATA:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}%{GREEDYDATA:message}'
-                ] 
-              }
-              overwrite => [ "message" ]
-            }
-          }
+
           if [kubernetes][container][name] == "postgresql" {
             grok {
               match => { 
@@ -156,67 +174,17 @@ logstash:
               overwrite => [ "message" ]
             }
           }
-          if [kubernetes][container][name] == "vault" or [kubernetes][container][name] == "vault-agent-init" or [kubernetes][container][name] == "sidecar-injector" {
-            grok {
-              match => { 
-                "message" => [
-                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{DATA:handler}:%{SPACE}%{GREEDYDATA:message}' 
-                   
-
-                ]
-              }
-              overwrite => [ "message" ]
-            }
-          }
-          if [kubernetes][container][name] == "simpl-cloud-gateway" or [kubernetes][container][name] == "users-roles" {
-            grok {
-              match => { 
-                "message" => [
-                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}' 
-                ]
-              }
-              overwrite => [ "message" ]
-            }
-          }
-          if [kubernetes][container][name] == "neo4j" {
-            grok {
-              match => { 
-                "message" => [
-                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{GREEDYDATA:message}' 
-                ]
-              }
-              overwrite => [ "message" ]
-            }
-          }
-          if [kubernetes][container][name] == "redis" {  
-            grok {
-              match => { 
-                "message" => [
-                     '%{NUMBER:process_id}:%{WORD:process_type}%{SPACE}%{MONTHDAY:day}%{SPACE}%{MONTH:month}%{SPACE}%{YEAR:year}%{SPACE}%{TIME:time}\.%{INT:milliseconds}%{SPACE}\*%{SPACE}%{GREEDYDATA:message}'
-                  ]
-                }
-              overwrite => [ "message" ]
-              add_field => {
-                "timestamp" => "%{day} %{month} %{year} %{time}.%{milliseconds}"
-              }
-            }
             
-            
-          }
-
-           if [fields][logtype] == "logs-sample-business" {
-            grok {
-              match => { "message" => '%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:origin}\|%{WORD:destination}\|%{WORD:business_operation}\|%{DATA:message_type}\|%{WORD:correlation_id}' }
-            }
-          }  
-          
-            date {
-              match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS" ]
-            } 
+          date {
+            match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"]
+          } 
+          date {
+            match => [ "ts", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"]
+          } 
         }
       output: |-
         output {
-          if [fields][logtype] == "logs-sample-business" {
+          if [kubernetes][container][name] == "simpl-cloud-gateway" {
             elasticsearch {
             hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
             user => "${LOGSTASH_USER}"
@@ -388,34 +356,59 @@ filebeat4agents:
     filebeat.autodiscover:
       providers:
         - type: kubernetes
+          # Filter logs only from the monitored namespace
+          namespace: "${MONITORED_NAMESPACE}"
           templates:
+            # Condition for redis container in the monitored namespace
             - condition:
-                or: 
-                  - equals:
-                      kubernetes.namespace: "${MONITORED_NAMESPACE}"
+                equals:
+                  kubernetes.container.name: "redis"
               config:
                 - type: container
                   paths:
                     - /var/log/containers/*-${data.kubernetes.container.id}.log
                   multiline:
-                    type: pattern
-                    pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+                    pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}'
                     negate: true
                     match: after
+            # Condition for json structured logs
             - condition:
-                equals:
-                  kubernetes.container.name: "redis"  
+                or:
+                  - equals:
+                      kubernetes.container.name: "users-roles"
+                  - equals:
+                      kubernetes.container.name: "signer"
+                  - equals:
+                      kubernetes.container.name: "sd-creation-wizard-api"
+                  - equals:
+                      kubernetes.container.name: "sd-creation-wizard-api-validation"
+                  - equals:
+                      kubernetes.container.name: "simpl-cloud-gateway"
+              config:
+                - type: container
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+            # Condition for plain text logs
+            - condition:
+                or:
+                  - equals:
+                      kubernetes.container.name: "keycloak"
+                  - equals:
+                      kubernetes.container.name: "postgresql"
               config:
                 - type: container
                   paths:
                     - /var/log/containers/*-${data.kubernetes.container.id}.log
                   multiline:
-                    pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}'  
+                    type: pattern
+                    pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
                     negate: true
                     match: after
     processors:
+      # Add cloud and host metadata
       - add_cloud_metadata: {}
       - add_host_metadata: {}
+
   output: |
     output.logstash:
       hosts: ["${LOGSTASH_HOSTS}"]

pipeline.variables.sh

-PROJECT_VERSION_NUMBER="0.1.2"
\ No newline at end of file
+PROJECT_VERSION_NUMBER="0.1.3"
\ No newline at end of file