diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index f8c3c1749cd6a6ee359d2c4e62c8919cfd5562de..8e886ba1c1642747b48094582ac116a9a4f25aa4 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -20,3 +20,21 @@ Elasticsearch api dns
elasticsearch.{{ default .Release.Namespace .Values.namespaceTag }}.{{ .Values.domainSuffix }}
{{- end -}}
+{{/*
+Logstash input dns
+*/}}
+{{- define "logstash.dns" -}}
+logstash.{{ .Values.logstash.pipelines_group_name }}.{{ default .Release.Namespace .Values.namespaceTag }}.{{ .Values.domainSuffix }}
+{{- end -}}
+
+{{/*
+Logstash input dns for many ingressRouteTCPs
+*/}}
+{{- define "logstash.dns.array" -}}
+{{ $concatUrl := ( printf ".%s" (include "logstash.dns" .)) }}
+{{ $urlPrefix := (default "l" .Values.logstash.urlPrefix) }}
+{{ $maxRange := (.Values.logstash.count |int ) }}
+{{- range $index := until $maxRange -}}
+ {{- $urlPrefix}}{{$index }}{{ $concatUrl }}{{if lt $index (sub $maxRange 1) }},{{end}}
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/templates/elasticsearch.yaml b/templates/elasticsearch.yaml
index 555a7223fda8fe45058e4a1ba19ad948f9f580cb..a3395c1c8f58168e6c0857a69faa9ed7c36058b4 100644
--- a/templates/elasticsearch.yaml
+++ b/templates/elasticsearch.yaml
@@ -5,6 +5,11 @@ metadata:
spec:
version: {{ .Values.elasticVersion }}
image: {{ .Values.elasticsearch.image }}:{{ default .Values.elasticVersion .Values.elasticsearch.imageTag }}
+ auth:
+ roles:
+ - secretName: logstash-writer-role-secret
+ fileRealm:
+ - secretName: logstash-writer-secret
nodeSets:
{{- range .Values.elasticsearch.nodeSets }}
- name: {{ .name }}
@@ -92,6 +97,32 @@ spec:
port:
number: 9200
---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-writer-secret
+type: kubernetes.io/basic-auth
+data:
+ username: {{ "logstash-writer" | b64enc }}
+ {{- if .Release.IsInstall }}
+ password: {{ randAlphaNum 20 | b64enc }}
+ {{ else }}
+ password: {{ index (lookup "v1" "Secret" .Release.Namespace "logstash-writer-secret").data "password" }}
+ {{ end }}
+ roles: {{ "logstash-writer-role" | b64enc }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+ name: logstash-writer-role-secret
+stringData:
+ roles.yml: |-
+ logstash-writer-role:
+ cluster: ["manage_index_templates", "monitor", "manage_ilm"]
+ indices:
+ - names: [ '*' ]
+ privileges: ["read","write","create","create_index","manage","manage_ilm"]
+---
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b4245309311dafa6fbbce88795bc733254da8bdb
--- /dev/null
+++ b/templates/filebeat.yaml
@@ -0,0 +1,93 @@
+apiVersion: beat.k8s.elastic.co/v1beta1
+kind: Beat
+metadata:
+ name: filebeat
+ labels:
+ app: filebeat
+spec:
+ image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
+ version: {{ .Values.elasticVersion }}
+ type: filebeat
+ elasticsearchRef:
+ name: {{ .Release.Name }}-elasticsearch
+ #clusterName: {{ .Release.Name }}-elasticsearch
+ daemonSet:
+ podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
+ spec:
+ securityContext:
+ runAsUser: 0 # Running as root user
+ fsGroup: 1000 # Filebeat user ID
+ containers:
+ - name: filebeat
+ command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
+ volumeMounts:
+ - mountPath: /usr/share/filebeat/filebeat.yml
+ subPath: filebeat.yml
+ name: config
+ - mountPath: /usr/share/filebeat/logs/example.sh
+ subPath: example.sh
+ name: example-script
+ volumes:
+ - name: config
+ secret:
+ secretName: filebeat-config
+ defaultMode: 0555
+ - name: example-script
+ configMap:
+ name: filebeat-example-script
+ defaultMode: 0777
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: filebeat-config
+ labels:
+ app: filebeat
+type: Opaque
+data:
+ filebeat.yml: |
+ {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: filebeat-example-script
+data:
+ example.sh: |
+ #!/bin/bash
+ count=0
+ > /usr/share/filebeat/logs/example.log
+ while [ "$count" -lt {{ .Values.filebeat.totalMessages }} ] || [ {{ .Values.filebeat.totalMessages }} -lt 0 ]
+ do
+ {{- if gt (.Values.filebeat.messagesPerMinute | int ) 0 }}
+ sleep {{ divf 60 .Values.filebeat.messagesPerMinute }}
+ {{- end }}
+ case $((RANDOM % 3)) in
+ (0) status=INFO
+ ;;
+ (1) status=WARN
+ ;;
+ (2) status=ERROR
+ ;;
+ esac
+ case $((RANDOM % 4)) in
+ (0) data=\
+ 'Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.'
+ ;;
+ (1) data=\
+ 'It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using "Content here, content here", making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for alorem ipsum will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose injected humour and the like.'
+ ;;
+ (2) data=\
+ 'Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source.'
+ ;;
+ (3) data=\
+ 'Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" The Extremes of Good and Evil by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32'
+ ;;
+ esac
+ echo "2022-04-01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)).000 | $count | $status | $data" >> /usr/share/filebeat/logs/example.log
+ count=$((count + 1))
+ done
+---
\ No newline at end of file
diff --git a/templates/logstash_beats.yaml b/templates/logstash_beats.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ff650212de719686b1d18f53e20b443e47639e09
--- /dev/null
+++ b/templates/logstash_beats.yaml
@@ -0,0 +1,100 @@
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+ name: logstash-beats
+ labels:
+ app: logstash
+spec:
+ image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
+ version: {{ .Values.elasticVersion }}
+ count: {{ .Values.logstash.count }}
+ elasticsearchRefs:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
+ services:
+{{- range $index := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+ - name: beats-{{$index}}
+ service:
+ spec:
+ ports:
+ - port: 5044
+ name: {{ $.Values.logstash.beats.pipelines_group_name }}
+ protocol: TCP
+ selector:
+ statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
+{{- end}}
+ config:
+ {{- with .Values.logstash.config }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
+ spec:
+ containers:
+ - name: logstash
+ {{- with .Values.logstash.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ volumeMounts:
+ {{- range .Values.logstash.beats.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
+ subPath: {{ .name -}}.config
+ {{- end }}
+ env:
+ - name: LOGSTASH_USER
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: username
+ - name: LOGSTASH_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: password
+ - name: ELASTIC_ELASTICSEARCH_ES_HOSTS
+ value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
+ volumes:
+ {{- range .Values.logstash.beats.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ configMap:
+ name: logstash-{{- $.Values.logstash.beats.pipelines_group_name -}}-{{- .name -}}-config
+ defaultMode: 511
+ {{- end }}
+ pipelinesRef:
+ secretName: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
+data:
+ pipelines.yml: |
+ {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+---
+{{- range .Values.logstash.beats.pipelines }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logstash-{{ $.Values.logstash.beats.pipelines_group_name }}-{{ .name }}-config
+data:
+ {{ .name }}.config: |
+ {{ tpl .input $ | nindent 6 }}
+ {{ tpl .filter $ | nindent 6 }}
+ {{ tpl .output $ | nindent 6 }}
+---
+{{- end }}
diff --git a/templates/logstash_syslog.yaml b/templates/logstash_syslog.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7de24c98e630e650868f880234550dedc96ce11c
--- /dev/null
+++ b/templates/logstash_syslog.yaml
@@ -0,0 +1,100 @@
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+ name: logstash-syslog
+ labels:
+ app: logstash
+spec:
+ image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
+ version: {{ .Values.elasticVersion }}
+ count: {{ .Values.logstash.count }}
+ elasticsearchRefs:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
+ services:
+{{- range $index := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+ - name: syslog-{{$index}}
+ service:
+ spec:
+ ports:
+ - port: 514
+ name: {{ $.Values.logstash.syslog.pipelines_group_name }}
+ protocol: TCP
+ selector:
+ statefulset.kubernetes.io/pod-name: logstash-syslog-ls-{{$index}}
+{{- end}}
+ config:
+ {{- with .Values.logstash.config }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
+ spec:
+ containers:
+ - name: logstash
+ {{- with .Values.logstash.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ volumeMounts:
+ {{- range .Values.logstash.syslog.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
+ subPath: {{ .name -}}.config
+ {{- end }}
+ env:
+ - name: LOGSTASH_USER
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: username
+ - name: LOGSTASH_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: password
+ - name: ELASTIC_ELASTICSEARCH_ES_HOSTS
+ value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
+ volumes:
+ {{- range .Values.logstash.syslog.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ configMap:
+ name: logstash-{{- $.Values.logstash.syslog.pipelines_group_name -}}-{{- .name -}}-config
+ defaultMode: 511
+ {{- end }}
+ pipelinesRef:
+ secretName: logstash-{{ .Values.logstash.syslog.pipelines_group_name }}-pipelines-yml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-{{ .Values.logstash.syslog.pipelines_group_name }}-pipelines-yml
+data:
+ pipelines.yml: |
+ {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+---
+{{- range .Values.logstash.syslog.pipelines }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logstash-{{ $.Values.logstash.syslog.pipelines_group_name }}-{{ .name }}-config
+data:
+ {{ .name }}.config: |
+ {{ tpl .input $ | nindent 6 }}
+ {{ tpl .filter $ | nindent 6 }}
+ {{ tpl .output $ | nindent 6 }}
+---
+{{- end }}
diff --git a/values.yaml b/values.yaml
index c1df6519ffffcdc2db166a8c39591b9be62cd591..5513fb28b8e3ea32a29473abba07c48af70b7bb8 100644
--- a/values.yaml
+++ b/values.yaml
@@ -1,5 +1,5 @@
# version of all elastic applications
-elasticVersion: 8.11.1
+elasticVersion: 8.14.1
# Name of aws environment
envTag: "dev"
@@ -65,6 +65,12 @@ elasticsearch:
# config:
# node.roles: ["master"]
config: {}
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ cpu: "1"
kibana:
# set >0 to deploy kibana, 0 otherwise
@@ -75,7 +81,6 @@ kibana:
# name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
elasticsearchReleaseName: ""
# Kibana server will listen on that subpath
- ingressSubpath: "/celk"
cert:
duration: 2160h0m0s # 90d
renewBefore: 360h0m0s # 15d
@@ -94,4 +99,127 @@ kibana:
env: []
+logstash:
+ count: 2
+ image: docker.elastic.co/logstash/logstash
+ config: {}
+ diskSpace: 3Gi
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ storageClassName: csi-cinder-high-speed
+ imageTag: ""
+ env:
+ LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ pipelines_yml_config: |-
+ - pipeline.id: main
+ path.config: "/app/elastic/logstash/config/pipelines/*.config"
+ pipeline.workers: 1
+ pipeline.batch.size: 125
+
+ workers: 1
+ batch:
+ size: 125
+ beats:
+ pipelines_group_name: "beats"
+ pipelines:
+ - name: "beats-pipeline"
+ input: |-
+ input {
+ beats {
+ port => 5044
+ }
+ }
+ filter: |-
+ filter {
+ grok {
+ match => { "message" => "%{TIMESTAMP_ISO8601:logdatetime} \| %{DATA:thread} \| %{LOGLEVEL:loglevel} \| %{GREEDYDATA:details}" }
+ }
+ }
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ ssl => true
+ ssl_certificate_verification => false
+ data_stream => "true"
+ data_stream_type => "logs"
+ data_stream_dataset => "filebeat"
+
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
+ syslog:
+ pipelines_group_name: "syslog"
+ pipelines:
+ - name: "syslog-pipeline"
+ input: |-
+ input {
+ syslog {
+ port => 514
+ }
+ }
+ filter: |-
+ filter {
+ }
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ ssl => true
+ ssl_certificate_verification => false
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
+
+filebeat:
+ image: docker.elastic.co/beats/filebeat
+ config: {}
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ imageTag: ""
+ # Total number of the sample messages to generate. Provide negative number to generate infinitely
+ totalMessages: 604800
+
+ # Number of messages per minute. Provide negative number to generate messages without time limit.
+ messagesPerMinute: 30
+
+ # Filebeat configuration file - input
+ input: |
+ filebeat.inputs:
+ - type: log
+ enabled: true
+ paths:
+ - /usr/share/filebeat/logs/example.log
+ multiline.pattern: (^([0-9]{4}((\-[0-9]{2}){2})))
+ multiline.negate: true
+ multiline.match: after
+ fields:
+ logtype: simpl-test-tool
+ filebeat.config.modules:
+ path: ${path.config}/modules.d/*.yml
+ reload.enabled: false
+ # processors:
+ # - add_kubernetes_metadata:
+ # in_cluster: true
+ # logging.level: debug
+ name: "test"
+ output: |
+ output.logstash:
+ hosts: ["logstash-beats-ls-beats-0.observability.svc:5044"]
+
+