Code development platform for open source projects from the European Union institutions :large_blue_circle: EU Login authentication by SMS has been phased out. To see alternatives please check here

Skip to content
Snippets Groups Projects
Commit 9cb84425 authored by Albert Brzozowski's avatar Albert Brzozowski
Browse files

Merge branch 'develop' into 'main'

Develop

See merge request !63
parents 123aab16 79091b12
Branches
No related tags found
3 merge requests!69Feature/merge develop into main,!67Feature/resolved conflicts,!63Develop
Pipeline #239450 failed
name: eck-monitoring name: eck-monitoring
version: ${PROJECT_RELEASE_VERSION} version: ${PROJECT_RELEASE_VERSION}
appVersion: "${PROJECT_RELEASE_VERSION}" appVersion: "${PROJECT_RELEASE_VERSION}"
#version: 0.1.0 #version: 0.1.3
...@@ -74,7 +74,7 @@ spec: ...@@ -74,7 +74,7 @@ spec:
http: http:
tls: tls:
certificate: certificate:
secretName: {{ .Release.Name }}-kibana-cert-secret secretName: {{ .Release.Name }}-kibana-ssl
--- ---
apiVersion: networking.k8s.io/v1 apiVersion: networking.k8s.io/v1
kind: Ingress kind: Ingress
......
...@@ -28,6 +28,7 @@ elasticsearch: ...@@ -28,6 +28,7 @@ elasticsearch:
resources: resources:
requests: requests:
memory: 4Gi memory: 4Gi
cpu: 300m
limits: limits:
memory: 4Gi memory: 4Gi
cpu: "1" cpu: "1"
...@@ -36,7 +37,7 @@ kibana: ...@@ -36,7 +37,7 @@ kibana:
count: 1 count: 1
image: docker.elastic.co/kibana/kibana image: docker.elastic.co/kibana/kibana
#Branch name to donwload dashboards #Branch name to donwload dashboards
dashboardsBranch: "develop" dashboardsBranch: "main"
# Kibana's image tag, by default it equals to elasticVersion # Kibana's image tag, by default it equals to elasticVersion
imageTag: "" imageTag: ""
# name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty. # name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
...@@ -110,113 +111,80 @@ logstash: ...@@ -110,113 +111,80 @@ logstash:
} }
filter: |- filter: |-
filter { filter {
if [kubernetes][container][name] == "ejbca-community-helm" { ## removing ELK logs
grok { if [kubernetes][container][name] == "filebeat" or [kubernetes][container][name] == "metricbeat" or [kubernetes][container][name] == "logstash" or [kubernetes][container][name] == "heartbeat" or [kubernetes][container][name] == "kibana" or [kubernetes][container][name] == "elasticsearch" {
match => { drop { }
"message" => [
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
]
}
overwrite => [ "message" ]
}
}
if [kubernetes][container][name] == "keycloak" {
grok {
match => {
"message" => [
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
]
}
overwrite => [ "message" ]
}
} }
if [kubernetes][container][name] == "onboarding" {
grok { if [kubernetes][container][name] == "sd-creation-wizard-api" or [kubernetes][container][name] == "signer" or [kubernetes][container][name] == "sd-creation-wizard-api-validation" or [kubernetes][container][name] == "simpl-cloud-gateway" {
pattern_definitions => { "JAVA" => "[0-9A-Za-z\[\]\.\$]*" } json {
match => { source => "message"
"message" => [ skip_on_invalid_json => true
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{PATH:path}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}',
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}HTTP%{SPACE}%{WORD:http_method}%{SPACE}"%{DATA:uri}"',
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVA:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}',
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{DATA:logger}%{SPACE}:%{SPACE}\[%{DATA:request_id}\]%{SPACE}%{GREEDYDATA:message}'
]
}
overwrite => [ "message" ]
}
}
if [kubernetes][container][name] == "postgresql" {
grok {
match => {
"message" => [
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:timezone}%{SPACE}\[%{NUMBER:pid}\]%{SPACE}%{WORD:log_level}:%{SPACE}%{GREEDYDATA:message}'
]
}
overwrite => [ "message" ]
} }
} }
if [kubernetes][container][name] == "vault" or [kubernetes][container][name] == "vault-agent-init" or [kubernetes][container][name] == "sidecar-injector" {
grok {
match => {
"message" => [
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}\[%{LOGLEVEL:loglevel}\]%{SPACE}%{DATA:handler}:%{SPACE}%{GREEDYDATA:message}'
if [kubernetes][container][name] == "users-roles" {
] json {
} source => "message"
overwrite => [ "message" ] skip_on_invalid_json => true
}
} }
if [kubernetes][container][name] == "simpl-cloud-gateway" or [kubernetes][container][name] == "users-roles" {
grok {
match => { ruby {
"message" => [ code => '
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{NUMBER:pid}%{SPACE}---%{SPACE}\[%{DATA:thread}\]%{SPACE}%{JAVACLASS:logger}%{SPACE}:%{SPACE}%{GREEDYDATA:message}' if event.get("[message]").is_a?(Hash)
] event.set("is_json_message", true)
else
event.set("is_json_message", false)
end
'
} }
overwrite => [ "message" ]
if [is_json_message] {
if [message][httpStatus] { mutate { add_field => { "httpStatus" => "%{[message][httpStatus]}" } } }
if [message][msg] { mutate { add_field => { "msg" => "%{[message][msg]}" } } }
if [message][httpRequestSize] { mutate { add_field => { "httpRequestSize" => "%{[message][httpRequestSize]}" } } }
if [message][user] { mutate { add_field => { "user" => "%{[message][user]}" } } }
if [message][httpExecutionTime] { mutate { add_field => { "httpExecutionTime" => "%{[message][httpExecutionTime]}" } } }
mutate { remove_field => [ "[message]" ] }
} }
} }
if [kubernetes][container][name] == "neo4j" {
if [kubernetes][container][name] == "keycloak" {
grok { grok {
match => { match => {
"message" => [ "message" => [
'%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}%{GREEDYDATA:message}' '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:loglevel}%{SPACE}\[%{JAVACLASS:logger}\]%{SPACE}\(%{DATA:thread}\)%{SPACE}%{GREEDYDATA:message}'
] ]
} }
overwrite => [ "message" ] overwrite => [ "message" ]
} }
} }
if [kubernetes][container][name] == "redis" {
if [kubernetes][container][name] == "postgresql" {
grok { grok {
match => { match => {
"message" => [ "message" => [
'%{NUMBER:process_id}:%{WORD:process_type}%{SPACE}%{MONTHDAY:day}%{SPACE}%{MONTH:month}%{SPACE}%{YEAR:year}%{SPACE}%{TIME:time}\.%{INT:milliseconds}%{SPACE}\*%{SPACE}%{GREEDYDATA:message}' '%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{WORD:timezone}%{SPACE}\[%{NUMBER:pid}\]%{SPACE}%{WORD:log_level}:%{SPACE}%{GREEDYDATA:message}'
] ]
} }
overwrite => [ "message" ] overwrite => [ "message" ]
add_field => {
"timestamp" => "%{day} %{month} %{year} %{time}.%{milliseconds}"
}
}
}
if [fields][logtype] == "logs-sample-business" {
grok {
match => { "message" => '%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:origin}\|%{WORD:destination}\|%{WORD:business_operation}\|%{DATA:message_type}\|%{WORD:correlation_id}' }
} }
} }
date { date {
match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"] match => [ "timestamp", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"]
} }
date {
match => [ "ts", "yyyy-MM-dd HH:mm:ss.SSS", "ISO8601", "yyyy-MM-dd HH:mm:ss", "dd MMM yyyy HH:mm:ss.SSS"]
}
} }
output: |- output: |-
output { output {
if [fields][logtype] == "logs-sample-business" { if [kubernetes][container][name] == "simpl-cloud-gateway" {
elasticsearch { elasticsearch {
hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ] hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
user => "${LOGSTASH_USER}" user => "${LOGSTASH_USER}"
...@@ -388,34 +356,59 @@ filebeat4agents: ...@@ -388,34 +356,59 @@ filebeat4agents:
filebeat.autodiscover: filebeat.autodiscover:
providers: providers:
- type: kubernetes - type: kubernetes
# Filter logs only from the monitored namespace
namespace: "${MONITORED_NAMESPACE}"
templates: templates:
# Condition for redis container in the monitored namespace
- condition: - condition:
or: equals:
- equals: kubernetes.container.name: "redis"
kubernetes.namespace: "${MONITORED_NAMESPACE}"
config: config:
- type: container - type: container
paths: paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log - /var/log/containers/*-${data.kubernetes.container.id}.log
multiline: multiline:
type: pattern pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}'
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true negate: true
match: after match: after
# Condition for json structured logs
- condition: - condition:
equals: or:
kubernetes.container.name: "redis" - equals:
kubernetes.container.name: "users-roles"
- equals:
kubernetes.container.name: "signer"
- equals:
kubernetes.container.name: "sd-creation-wizard-api"
- equals:
kubernetes.container.name: "sd-creation-wizard-api-validation"
- equals:
kubernetes.container.name: "simpl-cloud-gateway"
config:
- type: container
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
# Condition for plain text logs
- condition:
or:
- equals:
kubernetes.container.name: "keycloak"
- equals:
kubernetes.container.name: "postgresql"
config: config:
- type: container - type: container
paths: paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log - /var/log/containers/*-${data.kubernetes.container.id}.log
multiline: multiline:
pattern: '^\d+:\w+\s+\d{2}\s+\w{3}\s+\d{4}' type: pattern
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true negate: true
match: after match: after
processors: processors:
# Add cloud and host metadata
- add_cloud_metadata: {} - add_cloud_metadata: {}
- add_host_metadata: {} - add_host_metadata: {}
output: | output: |
output.logstash: output.logstash:
hosts: ["${LOGSTASH_HOSTS}"] hosts: ["${LOGSTASH_HOSTS}"]
......
PROJECT_VERSION_NUMBER="0.1.2" PROJECT_VERSION_NUMBER="0.1.3"
\ No newline at end of file \ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment