Code development platform for open source projects from the European Union institutions :large_blue_circle: EU Login authentication by SMS has been phased out. To see alternatives please check here

Skip to content
Snippets Groups Projects
Commit 49c91388 authored by Natalia Szakiel's avatar Natalia Szakiel
Browse files

Merge branch 'develop' into 'main'

Develop

See merge request !71
parents 36e730a5 f752786b
No related branches found
No related tags found
3 merge requests!75Feature/verion fix,!72Feature/logstash fix,!71Develop
Pipeline #245846 passed
# Changelog
All notable changes to this project will be documented in this file.
## [0.1.5] - 2024-12-13
### Added
- Changed internal issuer into cluster issuer to enable cross namespace communication between logstash and filebeat
- Added external Elasticsearch API
## [0.1.4] - 2024-11-29
### Added
- Added documentation in readme regarding Performance parameters
- Added documentation in readme regarding ILM
- Parsing onboarding technical and business logs
### Fixed
- Decreased cpu requestst
name: eck-monitoring
version: ${PROJECT_RELEASE_VERSION}
appVersion: "${PROJECT_RELEASE_VERSION}"
#version: 0.1.4
#version: 0.1.5
......@@ -24,7 +24,7 @@ elasticsearch.{{ .Values.namespaceTag }}.{{ .Values.domainSuffix }}
Logstash input dns
*/}}
{{- define "logstash.dns" -}}
logstash.{{ .Values.namespaceTag }}.{{ .Values.logstash.beats.pipelines_group_name }}.{{ .Values.domainSuffix }}
logstash.{{ .Values.logstash.beats.pipelines_group_name }}.{{ .Values.namespaceTag }}.{{ .Values.domainSuffix }}
{{- end -}}
{{/*
......
......@@ -110,6 +110,32 @@ spec:
port:
number: 9200
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ .Release.Name }}-elasticsearch-http-public
annotations:
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
acme.cert-manager.io/http01-edit-in-place: "true"
cert-manager.io/cluster-issuer: {{ .Values.clusterIssuer }}
spec:
ingressClassName: nginx
tls:
- hosts:
- {{ template "elasticsearch.dns" . }}
secretName: {{ .Release.Name }}-elasticsearch-ssl
rules:
- host: {{ template "elasticsearch.dns" . }}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: {{ .Release.Name }}-elasticsearch-es-http
port:
number: 9200
---
apiVersion: v1
kind: Secret
metadata:
......@@ -193,8 +219,9 @@ stringData:
- create_index
applications: []
---
{{- if eq .Values.namespaceTag .Values.mainNamespace }}
apiVersion: cert-manager.io/v1
kind: Issuer
kind: ClusterIssuer
metadata:
name: selfsigned-ca
spec:
......@@ -203,22 +230,24 @@ spec:
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: internal-ca
name: elk-ca
namespace: cert-manager
spec:
isCA: true
commonName: internal-ca
secretName: internal-ca
commonName: elk-ca
secretName: elk-ca
issuerRef:
name: selfsigned-ca
kind: Issuer
kind: ClusterIssuer
---
apiVersion: cert-manager.io/v1
kind: Issuer
kind: ClusterIssuer
metadata:
name: internal-issuer
name: elk-clusterissuer
spec:
ca:
secretName: internal-ca
secretName: elk-ca
{{- end }}
---
apiVersion: cert-manager.io/v1
kind: Certificate
......@@ -233,5 +262,5 @@ spec:
- {{ .Release.Name }}-elasticsearch-es-http.{{ .Release.Namespace }}
- {{ .Release.Name }}-elasticsearch-es-http.{{ .Release.Namespace }}.svc
issuerRef:
name: internal-issuer
kind: Issuer
name: elk-clusterissuer
kind: ClusterIssuer
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: filebeatagents
spec:
image: {{ .Values.filebeat4agents.image }}:{{ default .Values.elasticVersion .Values.filebeat4agents.imageTag }}
version: {{ .Values.elasticVersion }}
type: filebeat
daemonSet:
podTemplate:
spec:
serviceAccountName: filebeat4agents-account
automountServiceAccountToken: true
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true # Allows to provide richer host metadata
containers:
- name: filebeat
{{- with .Values.filebeat4agents.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
command: ['sh', '-c', 'exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
volumeMounts:
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
- mountPath: /usr/share/filebeat/filebeat.yml
subPath: filebeat.yml
name: config
- mountPath: /usr/share/filebeat/es-certs # used for monitoring
name: es-certs
- mountPath: /usr/share/filebeat/certs
name: filebeat-certs
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: ELASTIC_ELASTICSEARCH_ES_HOSTS
value: 'https://{{ .Release.Name }}-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
- name: LOGSTASH_HOSTS
value: 'logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Release.Namespace }}:5044'
- name: MONITORING_USER
valueFrom:
secretKeyRef:
name: user-monitoring-secret
key: username
- name: MONITORING_PASSWORD
valueFrom:
secretKeyRef:
name: user-monitoring-secret
key: password
- name: MONITORED_NAMESPACE
value: '{{ .Release.Namespace }}'
volumes:
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: config
secret:
secretName: filebeat4agents-config
defaultMode: 0555
- name: es-certs # used for monitoring
secret:
secretName: {{ .Release.Name }}-elasticsearch-http-cert-secret-internal
- name: filebeat-certs
secret:
secretName: filebeat4agents-certs-secret
---
apiVersion: v1
kind: Secret
metadata:
name: filebeat4agents-config
labels:
app: filebeat
type: Opaque
data:
filebeat.yml: |
{{ printf "%s\n%s" .Values.filebeat4agents.input (tpl .Values.filebeat4agents.output $ ) | b64enc | nindent 6 }}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: filebeat4agents-certificate
spec:
secretName: filebeat4agents-certs-secret
duration: {{ .Values.filebeat4agents.cert.duration }}
renewBefore: {{ .Values.filebeat4agents.cert.renewBefore }}
commonName: {{ template "filebeat4agents.dns" . }}
dnsNames:
- "{{ template "filebeat4agents.dns" . }}"
issuerRef:
name: internal-issuer
kind: Issuer
group: cert-manager.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat4agents-role-{{ .Release.Namespace }}
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
- apiGroups: ["apps"]
resources:
- replicasets
verbs:
- get
- list
- watch
- apiGroups: ["batch"]
resources:
- jobs
verbs:
- get
- list
- watch
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat4agents-account
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat4agents-rolebinding-{{ .Release.Namespace }}
subjects:
- kind: ServiceAccount
name: filebeat4agents-account
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: filebeat4agents-role-{{ .Release.Namespace }}
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-ilm-configmap
data:
filebeat-bussines-ilm.json: |
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_age": "{{ .Values.heartbeat.ilm.hot.max_age }}",
"max_primary_shard_size": "{{ .Values.heartbeat.ilm.hot.max_primary_shard_size }}"
},
"set_priority": {
"priority": 100
}
},
"min_age": "0ms"
},
"delete": {
"min_age": "{{ .Values.heartbeat.ilm.delete.min_age }}",
"actions": {
"delete": {}
}
}
}
}
}
---
......@@ -6,34 +6,46 @@ spec:
image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
version: {{ .Values.elasticVersion }}
type: filebeat
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
deployment:
replicas: {{ .Values.filebeat.count }}
daemonSet:
podTemplate:
spec:
securityContext:
runAsUser: 0
fsGroup: 1000
serviceAccountName: filebeat-account
automountServiceAccountToken: true
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true # Allows to provide richer host metadata
containers:
- name: filebeat
command: ['sh', '-c', 'exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
{{- with .Values.filebeat.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
command: ['sh', '-c', 'exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
volumeMounts:
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
- mountPath: /usr/share/filebeat/filebeat.yml
subPath: filebeat.yml
name: config
- mountPath: /usr/share/filebeat/certs
name: filebeat-certs
- mountPath: /usr/share/filebeat/es-certs # used for monitoring
name: es-certs
- mountPath: /usr/share/filebeat/logs/example.sh
subPath: example.sh
name: example-script
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: ELASTIC_ELASTICSEARCH_ES_HOSTS
value: 'https://{{ .Release.Name }}-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
- name: LOGSTASH_HOSTS
value: 'logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Release.Namespace }}:5044'
value: 'logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Values.mainNamespace }}.svc.cluster.local:5044'
- name: MONITORING_USER
valueFrom:
secretKeyRef:
......@@ -44,22 +56,26 @@ spec:
secretKeyRef:
name: user-monitoring-secret
key: password
- name: MONITORED_NAMESPACE
value: '{{ .Release.Namespace }}'
volumes:
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: config
secret:
secretName: filebeat-config
defaultMode: 0555
- name: filebeat-certs
secret:
secretName: filebeat-certs-secret
- name: es-certs # used for monitoring
secret:
secretName: {{ .Release.Name }}-elasticsearch-http-cert-secret-internal
- name: example-script
configMap:
name: filebeat-example-script
defaultMode: 0777
secretName: filebeat-certs-secret-{{ .Release.Namespace }}
---
apiVersion: v1
kind: Secret
......@@ -72,95 +88,97 @@ data:
filebeat.yml: |
{{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: {{ .Release.Namespace }}
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: RoleBinding
# metadata:
# name: filebeat-issuer
# roleRef:
# apiGroup: rbac.authorization.k8s.io
# kind: Role
# name: issuer-reader
# subjects:
# - kind: ServiceAccount
# name: filebeat
# namespace: {{ .Release.Namespace }}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: filebeat-certificate
name: filebeat-certificate-{{ .Release.Namespace }}
spec:
secretName: filebeat-certs-secret
secretName: filebeat-certs-secret-{{ .Release.Namespace }}
duration: {{ .Values.filebeat.cert.duration }}
renewBefore: {{ .Values.filebeat.cert.renewBefore }}
commonName: {{ template "filebeat.dns" . }}
dnsNames:
- "{{ template "filebeat.dns" . }}"
issuerRef:
name: internal-issuer
kind: Issuer
group: cert-manager.io
name: elk-clusterissuer
kind: ClusterIssuer
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat-role-{{ .Release.Namespace }}
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
- apiGroups: ["apps"]
resources:
- replicasets
verbs:
- get
- list
- watch
- apiGroups: ["batch"]
resources:
- jobs
verbs:
- get
- list
- watch
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat-account
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat-rolebinding-{{ .Release.Namespace }}
subjects:
- kind: ServiceAccount
name: filebeat-account
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: filebeat-role-{{ .Release.Namespace }}
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-example-script
name: filebeat-ilm-configmap
data:
example.sh: |
#!/bin/bash
# Function to generate a random string of given length
generate_random_string() {
local LENGTH=$1
tr -dc A-Za-z0-9 </dev/urandom | head -c $LENGTH
filebeat-bussines-ilm.json: |
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_age": "{{ .Values.heartbeat.ilm.hot.max_age }}",
"max_primary_shard_size": "{{ .Values.heartbeat.ilm.hot.max_primary_shard_size }}"
},
"set_priority": {
"priority": 100
}
# Initialize the count variable
count=0
# Path to the log file
LOG_FILE="/usr/share/filebeat/logs/example.log"
# Clear the log file
> "$LOG_FILE"
# Number of messages to generate
TOTAL_MESSAGES="{{ .Values.filebeat.totalMessages }}"
# Messages per minute
MESSAGES_PER_MINUTE="{{ .Values.filebeat.messagesPerMinute }}"
# Generate log entries
while [ "$count" -lt "$TOTAL_MESSAGES" ] || [ "$TOTAL_MESSAGES" -lt 0 ]
do
if [ "$MESSAGES_PER_MINUTE" -gt 0 ]; then
sleep_time=$((60 / MESSAGES_PER_MINUTE)) # Integer division
sleep $sleep_time
fi
ORIGINS=("originA" "originB" "originC" "originD" "originE")
ORIGIN=${ORIGINS[$RANDOM % ${#ORIGINS[@]}]}
DESTINATIONS=("destinationA" "destinationB" "destinationC" "destinationD" "destinationE")
DESTINATION=${DESTINATIONS[$RANDOM % ${#DESTINATIONS[@]}]}
BUSINESS_OPERATIONS=("operationA" "operationB" "operationC" "operationD" "operationE")
BUSINESS_OPERATION=${BUSINESS_OPERATIONS[$RANDOM % ${#BUSINESS_OPERATIONS[@]}]}
MESSAGE_TYPES=("request" "request ACK" "response" "response ACK")
MESSAGE_TYPE=${MESSAGE_TYPES[$RANDOM % ${#MESSAGE_TYPES[@]}]}
CORRELATION_ID=$(generate_random_string 20)
TIMESTAMP=$(date +"%Y-%m-%dT%H:%M:%S")
echo "$TIMESTAMP|$ORIGIN|$DESTINATION|$BUSINESS_OPERATION|$MESSAGE_TYPE|$CORRELATION_ID" >> "$LOG_FILE"
count=$((count + 1))
done
echo "Generated $count log entries and saved to $LOG_FILE"
},
"min_age": "0ms"
},
"delete": {
"min_age": "{{ .Values.heartbeat.ilm.delete.min_age }}",
"actions": {
"delete": {}
}
}
}
}
}
---
......@@ -236,10 +236,12 @@ spec:
- "{{$prefix}}{{$index_i}}.{{$concatUrl}}"
{{- end }}
- "logstash.{{ .Release.Namespace }}"
- "logstash.{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Values.namespaceTag }}.{{ .Values.domainSuffix }}"
- "logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Release.Namespace }}"
- "logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Release.Namespace }}.svc.cluster.local"
issuerRef:
name: internal-issuer
kind: Issuer
name: elk-clusterissuer
kind: ClusterIssuer
privateKey:
encoding: "PKCS8"
---
......
......@@ -2,6 +2,7 @@
elasticVersion: 8.15.1
namespaceTag: "test-namespace"
mainNamespace: observability
# This suffix will be used to create subdomain of following template:
# kibana.NAMESPACE.NAMESPACE_TAG.DOMAIN_SUFFIX
......@@ -210,7 +211,7 @@ logstash:
}
}
#stdout {
# codec => rubydebug
# codec => json
#}
}
syslog:
......@@ -244,88 +245,6 @@ logstash:
}
filebeat:
image: docker.elastic.co/beats/filebeat
count: 1
# name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
imageTag: ""
# Total number of the sample messages to generate. Provide negative number to generate infinitely
totalMessages: 604800
# Number of messages per minute. Provide negative number to generate messages without time limit.
messagesPerMinute: 30
cert:
duration: 2160h0m0s # 90d
renewBefore: 360h0m0s # 15d
# Filebeat configuration file - input
input: |
filebeat.inputs:
- type: filestream
paths:
- /mnt/repo/log_samples/onboarding/*.txt
fields:
logtype: logs-sample-onboarding
parsers:
- multiline:
type: pattern
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
- type: filestream
paths:
- /mnt/repo/log_samples/catalogue/signer/signer.txt
fields:
logtype: logs-sample-signer
- type: filestream
paths:
- /mnt/repo/log_samples/catalogue/sdtooling/sdtooling.txt
fields:
logtype: logs-sample-sdtooling
parsers:
- multiline:
type: pattern
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
- type: filestream
paths:
- /mnt/repo/log_samples/catalogue/*.txt
fields:
logtype: logs-sample-catalogue
parsers:
- multiline:
type: pattern
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
- type: filestream
paths:
- /usr/share/filebeat/logs/example.log
fields:
logtype: logs-sample-business
- type: filestream
paths:
- /mnt/repo/log_samples/wrapper/*.log
fields:
logtype: logs-sample-wrapper
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
output: |
output.logstash:
hosts: ["${LOGSTASH_HOSTS}"]
ssl.enabled: true
ssl.certificate_authorities: ["/usr/share/filebeat/es-certs/ca.crt"]
ssl.verification_mode: full
ssl.certificate: "/usr/share/filebeat/certs/tls.crt"
ssl.key: "/usr/share/filebeat/certs/tls.key"
monitoring.enabled: "true"
monitoring.elasticsearch:
hosts: ["${ELASTIC_ELASTICSEARCH_ES_HOSTS}"]
ssl.certificate_authorities: ["/usr/share/filebeat/es-certs/ca.crt"]
username: "${MONITORING_USER}"
password: "${MONITORING_PASSWORD}"
filebeat4agents:
image: docker.elastic.co/beats/filebeat
imageTag: ""
resources:
......@@ -395,7 +314,7 @@ filebeat4agents:
output.logstash:
hosts: ["${LOGSTASH_HOSTS}"]
ssl.enabled: true
ssl.certificate_authorities: ["/usr/share/filebeat/es-certs/ca.crt"]
ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca.crt"]
ssl.verification_mode: full
ssl.certificate: "/usr/share/filebeat/certs/tls.crt"
ssl.key: "/usr/share/filebeat/certs/tls.key"
......
elasticsearch:
count: 2
resources:
requests:
memory: 4Gi
limits:
memory: 4Gi
cpu: "1"
count: 0
kibana:
count: 1
filebeat:
count: 0
logstash:
count_beats: 1
count_beats: 0
count_syslog: 0
PROJECT_VERSION_NUMBER="0.1.4"
PROJECT_VERSION_NUMBER="0.1.5"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment