Merge branch 'feature/dashboards' into 'develop'

Feature/dashboards

See merge request !49

.gitlab/CODEOWNERS

+* @simpl/simpl-open/development/monitoring
\ No newline at end of file

charts/Chart.yaml

 name: eck-monitoring
-version: ${PROJECT_RELEASE_VERSION}
-appVersion: "${PROJECT_RELEASE_VERSION}"
-#version: 0.1.0
+#version: ${PROJECT_RELEASE_VERSION}
+#appVersion: "${PROJECT_RELEASE_VERSION}"
+version: 0.1.0
 
 

charts/kibana/dashboards/load_dashboards.sh

+#!/bin/bash
+
+#Wait 10 minutes ( 120 * 5 seconds delay ) for elasticsearch GREEN to load dashboards
+COUNT=120 
+
+#temp log file
+LOG_FILE="/mnt/dashboards/load_dashboard.tmp"
+
+for i in `seq 1 $COUNT`
+do
+        echo "Attempt no. $i to load dashoards."
+        curl -k -Ss  -u elastic:${ELASTIC_PASSWORD}  -X POST https://127.0.0.1:5601/api/saved_objects/_import?createNewCopies=false -H 'kbn-xsrf: true' --form file=@/mnt/dashboards/charts/kibana/dashboards/dashboards.ndjson > $LOG_FILE
+        if [[ `cat $LOG_FILE | awk -F"\"" '{print $2 $4}'` == "successCountsuccess" ]]
+        then
+                echo "Dashboards have been loaded successfully."
+                break
+        fi
+        sleep 5
+done

charts/kibana/scripts/load_objects.sh

+#!/bin/bash
+
+LOG_FILE="/mnt/ilm/load_ilm.tmp"
+
+#Loading ILMs
+
+echo "Starting loading objects..."
+for x in business-ilm technical-ilm
+do
+    i=1
+    echo "Starting loading ILM: $x"
+    while :
+    do
+        echo "Attempt no. $i to load ILM"
+        curl -k -Ss  -u elastic:${ELASTIC_PASSWORD}  -X PUT  https://elastic-elasticsearch-es-http:9200/_ilm/policy/$x  -H 'Content-Type: application/json' -H 'kbn-xsrf: true' -d @/usr/share/logstash/ilm/logstash-$x.json > $LOG_FILE
+        echo "Response:"
+        cat $LOG_FILE
+        echo -e "\n--------"
+        if [[ `cat $LOG_FILE | awk -F"\"" '{print $2}'` == "acknowledged" ]]
+        then
+                echo "ILM $x has been loaded successfully."
+                break
+        fi
+        i=`expr $i + 1`
+        sleep 5
+    done
+done
+
+#Loading templates
+
+for x in business-template technical-template
+do
+    i=1
+    echo "Starting template $x"
+    while :
+    do
+        echo "Attempt no. $i to load template"
+        curl -k -Ss  -u elastic:${ELASTIC_PASSWORD}  -X PUT  https://elastic-elasticsearch-es-http:9200/_index_template/$x  -H 'Content-Type: application/json' -H 'kbn-xsrf: true' -d @/mnt/ilm/charts/kibana/templates/$x-log.json > $LOG_FILE
+        echo "Response:"
+        cat $LOG_FILE
+        echo -e "\n--------"
+        if [[ `cat $LOG_FILE | awk -F"\"" '{print $2}'` == "acknowledged" ]]
+        then
+                echo "Teamplte $x has been loaded successfully."
+                break
+        fi
+        i=`expr $i + 1`
+        sleep 5
+    done
+done
+

charts/kibana/templates/business-template-log.json

+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "business-ilm"
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "timestamp": {
+            "type": "date"
+          },        
+        "origin": {
+          "type": "keyword"
+        },
+        "destination": {
+          "type": "keyword"
+        },
+        "business_operation": {
+          "type": "keyword"
+        },
+        "message_type": {
+          "type": "keyword"
+        },
+        "correlation_id": {
+          "type": "keyword"
+        }
+      }
+    }
+  },
+  "index_patterns": [
+    "business-logs"
+  ],
+  "data_stream": {
+    "hidden": false,
+    "allow_custom_routing": false
+  },
+  "composed_of": [],
+  "priority": 200
+}
\ No newline at end of file

charts/kibana/templates/technical-template-log.json

+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "technical-ilm"
+        }
+      }
+    },
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "timestamp": {
+            "type": "date"
+          },        
+        "logger": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "loglevel": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "message": {
+          "type": "match_only_text"
+        },
+        "method": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "msg": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "namespace": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "operation": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "path": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "pid": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "request_id": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "tags": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "thread": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "thread_name": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "ts": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "uri": {
+          "type": "keyword",
+          "ignore_above": 1024
+        },
+        "version": {
+          "type": "keyword",
+          "ignore_above": 1024
+        }
+      }
+    }
+  },
+  "index_patterns": [
+    "technical-logs"
+  ],
+  "data_stream": {
+    "hidden": false,
+    "allow_custom_routing": false
+  },
+  "composed_of": [],
+  "priority": 200
+}
\ No newline at end of file

charts/templates/elasticsearch.yaml

@@ -67,7 +67,7 @@ spec:
           {{- with .resources }}
           resources:
             {{- toYaml . | nindent 12 }}
-          {{- end }}                        
+          {{- end }}
           env:
             - name: ELASTICSEARCH_PASSWORD
               valueFrom:
@@ -202,15 +202,6 @@ metadata:
 spec:
   selfSigned: {}
 ---
-# apiVersion: rbac.authorization.k8s.io/v1
-# kind: Role
-# metadata:
-#   name: issuer-reader
-# rules:
-# - apiGroups: ["cert-manager.io"]
-#   resources: ["issuers"]
-#   verbs: ["get", "list", "watch"]
----
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:

charts/templates/filebeat-agents.yaml

@@ -150,4 +150,36 @@ roleRef:
   kind: ClusterRole
   name: filebeat4agents-role
   apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-ilm-configmap
+data: 
+  filebeat-bussines-ilm.json: |
+    {
+      "policy": {
+        "phases": {
+          "hot": {
+            "actions": {
+              "rollover": {
+                "max_age": "{{ .Values.heartbeat.ilm.hot.max_age }}",
+                "max_primary_shard_size": "{{ .Values.heartbeat.ilm.hot.max_primary_shard_size }}"
+              },
+              "set_priority": {
+                "priority": 100
+              }
+            },
+            "min_age": "0ms"
+          },
+          "delete": {
+            "min_age":  "{{ .Values.heartbeat.ilm.delete.min_age }}",
+            "actions": {
+              "delete": {}
+            }
+          }
+        }
+      }
+    }
+---
 

charts/templates/filebeat.yaml

@@ -15,19 +15,6 @@ spec:
         securityContext:
           runAsUser: 0  
           fsGroup: 1000  
-        initContainers:
-        - name: git-clone
-          image: alpine/git
-          args:
-            - clone
-            - --single-branch
-            - --branch
-            - feature/removed-roles # or the branch where your logs are
-            - https://code.europa.eu/simpl/simpl-open/development/monitoring/eck-monitoring.git
-            - /mnt/repo
-        volumeMounts:
-          - name: repo
-            mountPath: /mnt/repo
         containers:
         - name: filebeat
           command: ['sh', '-c', 'exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
@@ -39,8 +26,6 @@ spec:
             name: filebeat-certs
           - mountPath: /usr/share/filebeat/es-certs # used for monitoring
             name: es-certs
-          - mountPath: /mnt/repo
-            name: repo
           - mountPath: /usr/share/filebeat/logs/example.sh
             subPath: example.sh
             name: example-script
@@ -70,8 +55,6 @@ spec:
         - name: es-certs # used for monitoring
           secret:
             secretName: elastic-elasticsearch-http-cert-secret-internal
-        - name: repo
-          emptyDir: {}
         - name: example-script
           configMap:
             name: filebeat-example-script
@@ -174,7 +157,7 @@ data:
 
 
 
-      TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
+      TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%S")
       echo "$TIMESTAMP|$ORIGIN|$DESTINATION|$BUSINESS_OPERATION|$MESSAGE_TYPE|$CORRELATION_ID" >> "$LOG_FILE"
       count=$((count + 1))
     done

charts/templates/heartbeat.yaml

@@ -20,10 +20,56 @@ spec:
       id: ping-myhost
       name: My Host Ping
       hosts: ["elastic-kibana-kb-http.observability.svc"]
-      schedule: '*/5 * * * * * *'      
+      schedule: '*/5 * * * * * *'
+    setup.ilm.enabled: true
+    setup.ilm.policy_name: heartbeat-ilm
+    setup.ilm.policy_file: "/usr/share/heartbeat/ilm/heartbeat-ilm.json"
   deployment:
     replicas: 1
     podTemplate:
       spec:
         securityContext:
           runAsUser: 0
+        containers:
+        - name: heartbeat
+          volumeMounts:
+          - mountPath: /usr/share/heartbeat/ilm/heartbeat-ilm.json
+            name:  heartbeat-ilm-vol
+            subPath: heartbeat-ilm.json
+        volumes:
+        - name: heartbeat-ilm-vol
+          configMap:
+            name: heartbeat-ilm-configmap
+          defaultMode: 511          
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: heartbeat-ilm-configmap
+data: 
+  heartbeat-ilm.json: |
+    {
+      "policy": {
+        "phases": {
+          "hot": {
+            "actions": {
+              "rollover": {
+                "max_age": "{{ .Values.heartbeat.ilm.hot.max_age }}",
+                "max_primary_shard_size": "{{ .Values.heartbeat.ilm.hot.max_primary_shard_size }}"
+              },
+              "set_priority": {
+                "priority": 100
+              }
+            },
+            "min_age": "0ms"
+          },
+          "delete": {
+            "min_age":  "{{ .Values.heartbeat.ilm.delete.min_age }}",
+            "actions": {
+              "delete": {}
+            }
+          }
+        }
+      }
+    }
+---
\ No newline at end of file

charts/templates/kibana.yaml

@@ -14,8 +14,6 @@ spec:
   config:
     server.ssl.enabled: true
     elasticsearch.requestTimeout: 120000
-    #elasticsearch.ssl.verificationMode: certificate
-    #elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca.crt"]
     server.publicBaseUrl: "{{ template "kibana.dns.fullPath" . }}"
     {{- if and (.Values.kibana.ingressSubpath) (ne  "/" .Values.kibana.ingressSubpath) }}  
     server.basePath: {{ .Values.kibana.ingressSubpath }}
@@ -36,30 +34,48 @@ spec:
       securityContext:
         runAsUser: 1000
         fsGroup: 1000
+      initContainers:
+        - name: git-clone
+          image: alpine/git
+          args:
+            - clone
+            - --single-branch
+            - --branch
+            - {{ .Values.kibana.dashboardsBranch }}
+            - https://code.europa.eu/simpl/simpl-open/development/monitoring/eck-monitoring.git
+            - /mnt/dashboards
+          volumeMounts:
+            - name: repo
+              mountPath: /mnt/dashboards
       containers:
-        - name: kibana
-          imagePullPolicy: Always
-          {{- with .Values.kibana.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          #volumeMounts: 
-          #- name: es-certs
-          #  mountPath: /usr/share/kibana/config/certs     
-          #- name: lets-encrypt-ca
-          #  mountPath: /usr/share/kibana/config/certs-ca 
-          readinessProbe:
-            httpGet:
-              scheme: HTTPS
-              path: {{- with .Values.kibana.ingressSubpath }} {{ . }} {{- end }} 
-              port: 5601
-      #volumes:
-      #- name: es-certs
-      #  secret:
-      #    secretName:	elastic-elasticsearch-es-http-certs-internal
-      #- name: lets-encrypt-ca
-      #  secret: 
-      #    secretName: lets-encrypt-ca 
+      - name: kibana
+        imagePullPolicy: Always
+        {{- with .Values.kibana.resources }}
+        resources:
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            path: {{- with .Values.kibana.ingressSubpath }} {{ . }} {{- end }} 
+            port: 5601
+        env:
+          - name: ELASTIC_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Release.Name }}-elasticsearch-es-elastic-user 
+                key: elastic  
+        lifecycle:
+          postStart:
+            exec:
+              command: ["/bin/sh", "-c", "cd /mnt/dashboards/charts/kibana/dashboards/; chmod +x ./load_dashboards.sh; ./load_dashboards.sh > /mnt/dashboards/load_dashboard.log"]
+        volumeMounts:
+          - name: repo
+            mountPath: /mnt/dashboards
+      volumes:
+        - name: repo
+          emptyDir: {}
+
   http:
     tls:
       certificate:

charts/templates/logstash_beats.yaml

@@ -49,6 +49,39 @@ spec:
       labels:
         stack-namespace: {{ .Release.Namespace }} 
     spec:
+      securityContext:
+        runAsUser: 1000
+        fsGroup: 1000    
+      initContainers:
+      - name: git-clone
+        image: alpine/git
+        args:
+          - clone
+          - --single-branch
+          - --branch
+          - {{ .Values.kibana.dashboardsBranch }}            
+          - https://code.europa.eu/simpl/simpl-open/development/monitoring/eck-monitoring.git
+          - /mnt/ilm/
+        volumeMounts:
+          - name: repo
+            mountPath: /mnt/ilm/
+      - name: load-ilm
+        command:  ["/bin/sh", "-c", "cd /mnt/ilm/charts/kibana/scripts; chmod +x ./load_objects.sh; ./load_objects.sh 2>&1 "]
+        volumeMounts:
+          - name: repo
+            mountPath: /mnt/ilm/
+          - name:  logstash-business-ilm-vol
+            mountPath: /usr/share/logstash/ilm/logstash-business-ilm.json
+            subPath: logstash-business-ilm.json
+          - name:  logstash-technical-ilm-vol
+            mountPath: /usr/share/logstash/ilm/logstash-technical-ilm.json
+            subPath: logstash-technical-ilm.json            
+        env:
+          - name: ELASTIC_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Release.Name }}-elasticsearch-es-elastic-user 
+                key: elastic
       containers:
       - name: logstash
         {{- with .Values.logstash.resources }}
@@ -63,9 +96,17 @@ spec:
         {{- end }}
         - name: es-certs
           mountPath: /usr/share/logstash/config/certs
-        - mountPath: /usr/share/logstash/certs-logstash
-          name: certs-logstash
-        env:
+        - name: certs-logstash
+          mountPath: /usr/share/logstash/certs-logstash
+        - name: repo
+          mountPath: /mnt/ilm/
+        - name:  logstash-business-ilm-vol
+          mountPath: /usr/share/logstash/ilm/logstash-business-ilm.json
+          subPath: logstash-business-ilm.json
+        - name:  logstash-technical-ilm-vol
+          mountPath: /usr/share/logstash/ilm/logstash-technical-ilm.json
+          subPath: logstash-technical-ilm.json
+        env:      
           - name: LOGSTASH_USER
             valueFrom:
               secretKeyRef:
@@ -105,6 +146,16 @@ spec:
       - name: certs-logstash
         secret:
           secretName: logstash-secret-{{ .Values.logstash.beats.pipelines_group_name }}
+      - name: repo
+        emptyDir: {}
+      - name: logstash-business-ilm-vol
+        configMap:
+          name: logstash-business-ilm-configmap
+        defaultMode: 511
+      - name: logstash-technical-ilm-vol
+        configMap:
+          name: logstash-technical-ilm-configmap
+        defaultMode: 511
   pipelinesRef:
     secretName: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
 ---
@@ -187,4 +238,66 @@ spec:
     kind: Issuer
   privateKey:
     encoding: "PKCS8" 
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-business-ilm-configmap
+data: 
+  logstash-business-ilm.json: |
+    {
+      "policy": {
+        "phases": {
+          "hot": {
+            "actions": {
+              "rollover": {
+                "max_age": "{{ .Values.logstash.ilm.business.hot.max_age }}",
+                "max_primary_shard_size": "{{ .Values.logstash.ilm.business.hot.max_primary_shard_size }}"
+              },
+              "set_priority": {
+                "priority": 100
+              }
+            },
+            "min_age": "0ms"
+          },
+          "delete": {
+            "min_age":  "{{ .Values.logstash.ilm.business.delete.min_age }}",
+            "actions": {
+              "delete": {}
+            }
+          }
+        }
+      }
+    }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-technical-ilm-configmap
+data: 
+  logstash-technical-ilm.json: |
+    {
+      "policy": {
+        "phases": {
+          "hot": {
+            "actions": {
+              "rollover": {
+                "max_age": "{{ .Values.logstash.ilm.technical.hot.max_age }}",
+                "max_primary_shard_size": "{{ .Values.logstash.ilm.technical.hot.max_primary_shard_size }}"
+              },
+              "set_priority": {
+                "priority": 100
+              }
+            },
+            "min_age": "0ms"
+          },
+          "delete": {
+            "min_age":  "{{ .Values.logstash.ilm.technical.delete.min_age }}",
+            "actions": {
+              "delete": {}
+            }
+          }
+        }
+      }
+    }
 ---
\ No newline at end of file

charts/templates/metricbeat.yaml

@@ -68,6 +68,9 @@ spec:
         module: kubernetes
         ssl:
           verification_mode: none
+    setup.ilm.enabled: true
+    setup.ilm.policy_name: metricbeat-ilm
+    setup.ilm.policy_file: "/usr/share/metricbeat/ilm/metricbeat-ilm.json"
     processors:
     - add_cloud_metadata: {}
     - add_host_metadata: {}
@@ -98,6 +101,9 @@ spec:
             name: dockersock
           - mountPath: /hostfs/proc
             name: proc
+          - mountPath: /usr/share/metricbeat/ilm/metricbeat-ilm.json
+            name:  metricbeat-ilm-vol
+            subPath: metricbeat-ilm.json            
           env:
           - name: NODE_NAME
             valueFrom:
@@ -118,6 +124,10 @@ spec:
         - hostPath:
             path: /proc
           name: proc
+        - name: metricbeat-ilm-vol
+          configMap:
+            name: metricbeat-ilm-configmap
+          defaultMode: 511    
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -183,3 +193,34 @@ roleRef:
   name: metricbeat
   apiGroup: rbac.authorization.k8s.io
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metricbeat-ilm-configmap
+data: 
+  metricbeat-ilm.json: |
+    {
+      "policy": {
+        "phases": {
+          "hot": {
+            "actions": {
+              "rollover": {
+                "max_age": "{{ .Values.metricbeat.ilm.hot.max_age }}",
+                "max_primary_shard_size": "{{ .Values.metricbeat.ilm.hot.max_primary_shard_size }}"
+              },
+              "set_priority": {
+                "priority": 100
+              }
+            },
+            "min_age": "0ms"
+          },
+          "delete": {
+            "min_age":  "{{ .Values.metricbeat.ilm.delete.min_age }}",
+            "actions": {
+              "delete": {}
+            }
+          }
+        }
+      }
+    }
+---

charts/values/dev/observability/values.yaml

@@ -74,6 +74,8 @@ kibana:
   # set >0 to deploy kibana, 0 otherwise
   count: 1
   image: docker.elastic.co/kibana/kibana
+  #Branch name to donwload dashboards
+  dashboardsBranch: "feature/dashboards"
   # Kibana's image tag, by default it equals to elasticVersion
   imageTag: ""
   # name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
@@ -98,7 +100,20 @@ kibana:
 
 
 logstash:
-  count: 2
+  ilm:
+    business:
+      hot:
+        max_age: 30d
+        max_primary_shard_size: 1gb
+      delete:
+        min_age: 30d
+    technical:
+      hot:
+        max_age: 30d
+        max_primary_shard_size: 1gb
+      delete:
+        min_age: 30d
+  count: 1
   image: docker.elastic.co/logstash/logstash
   config: {}
   diskSpace: 3Gi
@@ -141,71 +156,110 @@ logstash:
         }
       filter: |-
         filter {
-          if [fields][logtype] == "logs-sample-onboarding" {
+          if [kubernetes][container][name] == "ejbca-community-helm" {
+            grok {
+              match => { 
+                "message" => [
+                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}', 
+                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
+                  ] 
+              }
+              overwrite => [ "message" ]
+            }
+          }
+          if [kubernetes][container][name] == "keycloak" {
             grok {
+              match => { 
+                "message" => [
+                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
+                  ]
+              }
+              overwrite => [ "message" ]
+            }
+          }
+          if [kubernetes][container][name] == "onboarding" {
+            grok {
+              pattern_definitions => { "JAVA" => "[0-9A-Za-z\[\]\.\$]*" }
               match => { 
                 "message" => [
                   '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}', 
                   '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
                   '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}HTTP%{SPACE}%{WORD:http_method}%{SPACE}"%{DATA:uri}"',
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}',
+                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}',
                   '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{DATA:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}%{GREEDYDATA:message}'
                 ] 
               }
               overwrite => [ "message" ]
             }
-          } 
-           if [fields][logtype] == "logs-sample-sdtooling" {
+          }
+          if [kubernetes][container][name] == "postgresql" {
             grok {
-              pattern_definitions => { "JAVA" => "[0-9A-Za-z\[\]\.\$]*" }
               match => { 
                 "message" => [
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}Method:%{SPACE}%{DATA:method}%{SPACE}\-%{SPACE}%{GREEDYDATA:message}', 
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}'
-                ] 
+                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:timezone}%{SPACE}\[%{NUMBER:pid}\]%{SPACE}%{WORD:log_level}:%{SPACE}%{GREEDYDATA:message}' 
+                  ]
               }
               overwrite => [ "message" ]
             }
-          } 
-           if [fields][logtype] == "logs-sample-catalogue" {
+          }
+          if [kubernetes][container][name] == "vault" or [kubernetes][container][name] == "vault-agent-init" or [kubernetes][container][name] == "sidecar-injector" {
             grok {
-              pattern_definitions => { "JAVA" => "[0-9A-Za-z\[\]\.\$]*" }
               match => { 
-                "message" => [ 
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:exec_thread}\]%{SPACE}\[%{DATA:request_id}\]%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}',
-                  '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:exec_thread}\]%{SPACE}%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}'
-                ] 
+                "message" => [
+                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{DATA:handler}:%{SPACE}%{GREEDYDATA:message}' 
+                   
+
+                ]
               }
               overwrite => [ "message" ]
             }
-          } 
-           if [fields][logtype] == "logs-sample-signer" {
-            json {
-              source => "message"
-            }
-          } 
-           if [fields][logtype] == "logs-sample-business" {
+          }
+          if [kubernetes][container][name] == "simpl-cloud-gateway" or [kubernetes][container][name] == "users-roles" {
             grok {
-              match => { "message" => '%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:origin}\|%{WORD:destination}\|%{WORD:business_operation}\|%{DATA:message_type}\|%{WORD:correlation_id}' }
+              match => { 
+                "message" => [
+                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}' 
+                ]
+              }
+              overwrite => [ "message" ]
             }
-          }  if [fields][logtype] == "logs-sample-wrapper" {
-              if [message] !~ "^\{" {
-                drop { }
+          }
+          if [kubernetes][container][name] == "neo4j" {
+            grok {
+              match => { 
+                "message" => [
+                    '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{GREEDYDATA:message}' 
+                ]
               }
-              json {
-                source => "message"
+              overwrite => [ "message" ]
+            }
+          }
+          if [kubernetes][container][name] == "redis" {  
+            grok {
+              match => { 
+                "message" => [
+                     '%{NUMBER:process_id}:%{WORD:process_type}%{SPACE}%{MONTHDAY:day}%{SPACE}%{MONTH:month}%{SPACE}%{YEAR:year}%{SPACE}%{TIME:time}\.%{INT:milliseconds}%{SPACE}\*%{SPACE}%{GREEDYDATA:message}'
+                  ]
+                }
+              overwrite => [ "message" ]
+              add_field => {
+                "timestamp" => "%{day} %{month} %{year} %{time}.%{milliseconds}"
               }
+            }
+            
+            
           }
-          if [ts] {
-            date {
-              match => [ "ts", "ISO8601" ]
+
+           if [fields][logtype] == "logs-sample-business" {
+            grok {
+              match => { "message" => '%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:origin}\|%{WORD:destination}\|%{WORD:business_operation}\|%{DATA:message_type}\|%{WORD:correlation_id}' }
             }
-          } else {
+          }  
+          
             date {
-              match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss" ]
+              match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS" ]
             } 
-          }
+          
         }
       output: |-
         output {
@@ -217,9 +271,9 @@ logstash:
             ssl_enabled => "true"
             ssl_verification_mode => "full"
             ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca.crt"
-            data_stream => "true"
-            data_stream_type => "logs"
-            data_stream_dataset => "business"
+            index => "business-logs"
+            template_name => "business-template"
+            action => "create"
             }
           }
           else if [fields][logtype] == "logs-sample-wrapper" {
@@ -230,24 +284,27 @@ logstash:
             ssl_enabled => "true"
             ssl_verification_mode => "full"
             ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca.crt"
-            data_stream => "true"
-            data_stream_type => "logs"
-            data_stream_dataset => "business"
-            }
-          }
-          else if [fields][logtype] == "agents" {
-            elasticsearch {
-            hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
-            user => "${LOGSTASH_USER}"
-            password => "${LOGSTASH_PASSWORD}"
-            ssl_enabled => "true"
-            ssl_verification_mode => "full"
-            ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca.crt"
-            data_stream => "true"
-            data_stream_type => "logs"
-            data_stream_dataset => "agents"
+            #data_stream => "true"
+            #data_stream_type => "logs"
+            #data_stream_dataset => "business"
+            index => "business-logs"
+            template_name => "business-template"
+            action => "create"            
             }
           }
+          # else if [fields][logtype] == "agents" {
+          #   elasticsearch {
+          #   hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+          #   user => "${LOGSTASH_USER}"
+          #   password => "${LOGSTASH_PASSWORD}"
+          #   ssl_enabled => "true"
+          #   ssl_verification_mode => "full"
+          #   ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca.crt"
+          #   data_stream => "true"
+          #   data_stream_type => "logs"
+          #   data_stream_dataset => "agents"
+          #   }
+          # }
           else {
             elasticsearch {
               hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
@@ -256,9 +313,12 @@ logstash:
               ssl_enabled => "true"
               ssl_verification_mode => "full"
               ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca.crt"
-              data_stream => "true"
-              data_stream_type => "logs"
-              data_stream_dataset => "technical"
+              #data_stream => "true"
+              #data_stream_type => "logs"
+              #data_stream_dataset => "technical"
+              index => "technical-logs"
+              template_name => "technical-template"
+              action => "create"
             }
           }
           #stdout { 
@@ -298,7 +358,7 @@ logstash:
 filebeat:
   image: docker.elastic.co/beats/filebeat
   config: {}
-  count: 0
+  count: 1
   # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
   imageTag: ""
   # Total number of the sample messages to generate. Provide negative number to generate infinitely
@@ -409,19 +469,37 @@ filebeat4agents:
             - condition:
                 or: 
                   - equals:
-                      kubernetes.namespace: "ingress-nginx"
+                      kubernetes.namespace: "authority01"
                   - equals:
-                      kubernetes.namespace: "kube-system"
+                      kubernetes.namespace: "dataprovider01"
+                  - equals:
+                      kubernetes.namespace: "consumer01"                      
               config:
                 - type: container
                   paths:
                     - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  multiline:
+                    type: pattern
+                    pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+                    negate: true
+                    match: after
+            - condition:
+                equals:
+                  kubernetes.container.name: "redis"  
+              config:
+                - type: container
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  multiline:
+                    pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}'  
+                    negate: true
+                    match: after
     processors:
       - add_cloud_metadata: {}
       - add_host_metadata: {}
-      - add_fields:
-          fields:
-            logtype: "agents"
+#      - add_fields:
+#          fields:
+#            logtype: "agents"
   output: |
     output.logstash:
       hosts: ["${LOGSTASH_HOSTS}"]
@@ -430,15 +508,14 @@ filebeat4agents:
       ssl.verification_mode: full
       ssl.certificate: "/usr/share/filebeat/certs/tls.crt"
       ssl.key: "/usr/share/filebeat/certs/tls.key"
-    # monitoring.enabled: "true"
-    # monitoring.elasticsearch:
-    #   hosts: ["${ELASTIC_ELASTICSEARCH_ES_HOSTS}"]
-    #   ssl.certificate_authorities: ["/usr/share/filebeat/es-certs/ca.crt"]
-    #   username: "${MONITORING_USER}"
-    #   password: "${MONITORING_PASSWORD}"
-
 
 metricbeat:
+  ilm:
+    hot:
+      max_age: 30d
+      max_primary_shard_size: 1gb
+    delete:
+      min_age: 30d
   resources:
     requests:
       memory: 500Mi
@@ -454,9 +531,21 @@ metricbeat:
       not:
         or:
           - equals:
-              kubernetes.namespace: observability
+              kubernetes.namespace: authority01
+          - equals:
+              kubernetes.namespace: dataprovider01
           - equals:
-              kubernetes.namespace: argo-cd
+              kubernetes.namespace: consumer01
           - equals:
               service.type: system
+
+
+heartbeat:
+  ilm:
+    hot:
+      max_age: 30d
+      max_primary_shard_size: 100mb
+    delete:
+      min_age: 30d
+      
     
\ No newline at end of file