From 68fd5d2e72b6431e0f7c8d17a93da900a0d589b6 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 11:19:30 +0200
Subject: [PATCH 01/61] added logstash
---
templates/_helpers.tpl | 18 ++++
templates/logstash.yaml | 198 ++++++++++++++++++++++++++++++++++++++++
values.yaml | 92 ++++++++++++++++++-
3 files changed, 307 insertions(+), 1 deletion(-)
create mode 100644 templates/logstash.yaml
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index f8c3c17..8e886ba 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -20,3 +20,21 @@ Elasticsearch api dns
elasticsearch.{{ default .Release.Namespace .Values.namespaceTag }}.{{ .Values.domainSuffix }}
{{- end -}}
+{{/*
+Logstash input dns
+*/}}
+{{- define "logstash.dns" -}}
+logstash.{{ .Values.logstash.pipelines_group_name }}.{{ default .Release.Namespace .Values.namespaceTag }}.{{ .Values.domainSuffix }}
+{{- end -}}
+
+{{/*
+Logstash input dns for many ingressRouteTCPs
+*/}}
+{{- define "logstash.dns.array" -}}
+{{ $concatUrl := ( printf ".%s" (include "logstash.dns" .)) }}
+{{ $urlPrefix := (default "l" .Values.logstash.urlPrefix) }}
+{{ $maxRange := (.Values.logstash.count |int ) }}
+{{- range $index := until $maxRange -}}
+ {{- $urlPrefix}}{{$index }}{{ $concatUrl }}{{if lt $index (sub $maxRange 1) }},{{end}}
+{{- end -}}
+{{- end -}}
\ No newline at end of file
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
new file mode 100644
index 0000000..046dffe
--- /dev/null
+++ b/templates/logstash.yaml
@@ -0,0 +1,198 @@
+{{- if gt (.Values.logstash.count | int) 0 }}
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+ name: logstash-{{ .Values.logstash.pipelines_group_name }}
+spec:
+ image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
+ version: {{ .Values.elasticVersion }}
+ count: {{ .Values.logstash.count }}
+ elasticsearchRefs:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
+ services:
+{{- range $index := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+ - name: beats-{{$index}}
+ {{- if $.Values.logstash.enableTLS }}
+ tls:
+ certificate:
+ secretName: logstash-secret-{{ $.Values.logstash.pipelines_group_name }}
+ {{- end }}
+ service:
+ spec:
+ ports:
+ - port: 5044
+ name: {{ $.Values.logstash.pipelines_group_name }}
+ protocol: TCP
+ selector:
+ statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
+{{- end}}
+ podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
+ spec:
+ imagePullpolicy: Always
+ {{- with $.Values.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ volumes:
+ {{- range .Values.logstash.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ configMap:
+ name: logstash-{{- $.Values.logstash.pipelines_group_name -}}-{{- .name -}}-config
+ defaultMode: 511
+ {{- end }}
+ {{- if .Values.logstash.enableTLS }}
+ - name: certs-logstash
+ secret:
+ secretName: logstash-secret-{{ .Values.logstash.pipelines_group_name }}
+ {{- end}}
+ containers:
+ - name: logstash
+ # command: ["/bin/bash", "-c", "sleep inf"]
+ {{- with .Values.logstash.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ volumeMounts:
+ {{- range .Values.logstash.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
+ subPath: {{ .name -}}.config
+ {{- end }}
+ {{- if .Values.logstash.enableTLS }}
+ - mountPath: /usr/share/logstash/certs-logstash
+ name: certs-logstash
+ {{- end }}
+ secureSettings:
+ - secretName: {{ .Release.Namespace }}-logstash-{{ .Values.logstash.pipelines_group_name }}-{{ .Release.Namespace }}-{{ .Release.Name }}-elasticsearch-logstash-user
+ env:
+ - name: LOGSTASH_USER
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: username
+ - name: LOGSTASH_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: password
+ {{- with .Values.logstash.env }}
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ config:
+ {{- with .Values.logstash.config }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ pipelinesRef:
+ secretName: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+data:
+ pipelines.yml: |
+ {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+---
+{{- range .Values.logstash.pipelines }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logstash-{{ $.Values.logstash.pipelines_group_name }}-{{ .name }}-config
+data:
+ {{ .name }}.config: |
+ {{ tpl .input $ | nindent 6 }}
+ {{ tpl .filter $ | nindent 6 }}
+ {{ tpl .output $ | nindent 6 }}
+---
+{{- end }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: logstash-api-{{ .Values.logstash.pipelines_group_name }}
+ annotations:
+ kubernetes.io/ingress.class: nginx
+ nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+ acme.cert-manager.io/http01-edit-in-place: "true"
+ cert-manager.io/cluster-issuer: dev-staging
+spec:
+ {{- if .Values.logstash.enableTLS }}
+ tls:
+ - hosts:
+ - {{ template "logstash.dns" . }}
+ secretName: logstash-secret-{{ .Values.logstash.pipelines_group_name }}
+ {{- end }}
+ rules:
+ - host: {{ template "logstash.dns" . }}
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: logstash-{{ .Values.logstash.pipelines_group_name }}-ls-api
+ port:
+ number: 9600
+{{ $concatUrl := (include "logstash.dns" .) }}
+{{ $prefix := (default "l" .Values.logstash.urlPrefix) }}
+{{- range $index_i := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRouteTCP
+metadata:
+ name: logstash-route-{{ $.Values.logstash.pipelines_group_name }}-{{$index_i}}
+spec:
+ entryPoints:
+ - logstash
+ routes:
+ - match: "HostSNI(`{{- if $.Values.logstash.enableTLS -}}{{$prefix}}{{$index_i}}.{{$concatUrl}}{{- else -}}*{{- end -}}`)"
+ services:
+ - name: logstash-{{ $.Values.logstash.pipelines_group_name }}-ls-{{ $.Values.logstash.pipelines_group_name }}-{{$index_i}}
+ port: 5044
+ {{- if $.Values.logstash.enableTLS }}
+ tls:
+ passthrough: true
+ secretName: logstash-secret-{{ $.Values.logstash.pipelines_group_name }}
+ {{- end }}
+{{- end }}
+---
+{{- if .Values.logstash.enableTLS }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ name: logstash-{{ .Values.logstash.pipelines_group_name }}
+spec:
+ dnsNames:
+ - "{{ template "logstash.dns" . }}"
+{{- range $index_i := until (.Values.logstash.count |int ) }}
+ - "{{$prefix}}{{$index_i}}.{{$concatUrl}}"
+{{- end }}
+ - "logstash.{{ .Release.Namespace }}"
+ secretName: logstash-secret-{{ .Values.logstash.pipelines_group_name }}
+ duration: {{ .Values.logstash.cert.duration }}
+ renewBefore: {{ .Values.logstash.cert.renewBefore }}
+ issuerRef:
+ name: dev-staging
+ kind: ClusterIssuer
+ privateKey:
+ encoding: "PKCS8"
+---
+{{- end }}
+
+{{- end }}
diff --git a/values.yaml b/values.yaml
index c1df651..fd3c51f 100644
--- a/values.yaml
+++ b/values.yaml
@@ -65,6 +65,12 @@ elasticsearch:
# config:
# node.roles: ["master"]
config: {}
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ cpu: "1"
kibana:
# set >0 to deploy kibana, 0 otherwise
@@ -75,7 +81,6 @@ kibana:
# name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
elasticsearchReleaseName: ""
# Kibana server will listen on that subpath
- ingressSubpath: "/celk"
cert:
duration: 2160h0m0s # 90d
renewBefore: 360h0m0s # 15d
@@ -94,4 +99,89 @@ kibana:
env: []
+logstash:
+ # set >0 to deploy logstash , 0 otherwise
+ count: 2
+ urlPrefix: ""
+ image: docker.elastic.co/logstash/logstash
+ # Logstash's image tag, by default it equals to elasticVersion
+ imageTag: ""
+ jvm_options_xms: 1g
+ jvm_options_xmx: 1g
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ cert:
+ duration: 2160h0m0s # 90d
+ renewBefore: 360h0m0s # 15d
+ # Set to false when there is logstash already deployed in the same namespace
+ createConfigMaps: true
+ # Indicates if logstash should be connected to elasticsearch
+ isElasticsearchDeployed: true
+ custompatternsactive: false
+ dropwizardactive: false
+ # persistent volume claim size
+ diskSpace: 1Gi
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ storageClassName: csi-cinder-high-speed
+
+ # Here you can specify extra logstash.yml config e.g
+ # config:
+ # xpack.monitoring.enabled: "true"
+ config: {}
+
+ #Environment variables to set in logstash pod
+ #Usage from cli:
+ # --set "logstash.env[0].name=VARIABLE_NAME" --set "logstash.env[0].value=VARIABLE_VALUE"
+ env: []
+
+ # Use TLS in ingress for logstash input.
+ enableTLS: false
+
+ # Content of pipelines.yml configuration file.
+ pipelines_yml_config: |-
+ - pipeline.id: main
+ path.config: "/app/elastic/logstash/config/pipelines/*.config"
+ pipeline.workers: 1
+ pipeline.batch.size: 125
+
+ pipelines_group_name: "beats"
+ workers: 1
+ batch:
+ size: 125
+ pipelines:
+ # Name of pipeline. This does not affect any logstash functionality,
+ # but creates unique, human-readable names of k8s resources
+ - name: "input"
+ # Input of logstash pipeline
+ # The default one is an input for beats
+ input: |-
+ input {
+ beats {
+ port => 5044
+ }
+ }
+ filter: |-
+ filter {}
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ #index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ #ssl_enabled => "true"
+ ssl_verification_mode => "none"
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
+
+
+
+
--
GitLab
From b50d958ee75d757ab23dfbdaa1e06d7ffd177112 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 13:28:30 +0200
Subject: [PATCH 02/61] added logstash
---
templates/logstash.yaml | 47 +----------------------------------------
1 file changed, 1 insertion(+), 46 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 046dffe..57e421d 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -148,51 +148,6 @@ spec:
name: logstash-{{ .Values.logstash.pipelines_group_name }}-ls-api
port:
number: 9600
-{{ $concatUrl := (include "logstash.dns" .) }}
-{{ $prefix := (default "l" .Values.logstash.urlPrefix) }}
-{{- range $index_i := until (.Values.logstash.count |int ) -}}
-{{- printf "\n"}}
----
-apiVersion: traefik.containo.us/v1alpha1
-kind: IngressRouteTCP
-metadata:
- name: logstash-route-{{ $.Values.logstash.pipelines_group_name }}-{{$index_i}}
-spec:
- entryPoints:
- - logstash
- routes:
- - match: "HostSNI(`{{- if $.Values.logstash.enableTLS -}}{{$prefix}}{{$index_i}}.{{$concatUrl}}{{- else -}}*{{- end -}}`)"
- services:
- - name: logstash-{{ $.Values.logstash.pipelines_group_name }}-ls-{{ $.Values.logstash.pipelines_group_name }}-{{$index_i}}
- port: 5044
- {{- if $.Values.logstash.enableTLS }}
- tls:
- passthrough: true
- secretName: logstash-secret-{{ $.Values.logstash.pipelines_group_name }}
- {{- end }}
-{{- end }}
----
-{{- if .Values.logstash.enableTLS }}
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
- name: logstash-{{ .Values.logstash.pipelines_group_name }}
-spec:
- dnsNames:
- - "{{ template "logstash.dns" . }}"
-{{- range $index_i := until (.Values.logstash.count |int ) }}
- - "{{$prefix}}{{$index_i}}.{{$concatUrl}}"
-{{- end }}
- - "logstash.{{ .Release.Namespace }}"
- secretName: logstash-secret-{{ .Values.logstash.pipelines_group_name }}
- duration: {{ .Values.logstash.cert.duration }}
- renewBefore: {{ .Values.logstash.cert.renewBefore }}
- issuerRef:
- name: dev-staging
- kind: ClusterIssuer
- privateKey:
- encoding: "PKCS8"
----
-{{- end }}
+
{{- end }}
--
GitLab
From 987d66403e1b200c50b836c4e1ab6239f9637ffb Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 14:34:15 +0200
Subject: [PATCH 03/61] added logstash
---
values.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/values.yaml b/values.yaml
index fd3c51f..424e9eb 100644
--- a/values.yaml
+++ b/values.yaml
@@ -101,7 +101,7 @@ kibana:
logstash:
# set >0 to deploy logstash , 0 otherwise
- count: 2
+ count: 1
urlPrefix: ""
image: docker.elastic.co/logstash/logstash
# Logstash's image tag, by default it equals to elasticVersion
@@ -138,7 +138,7 @@ logstash:
env: []
# Use TLS in ingress for logstash input.
- enableTLS: false
+ enableTLS: true
# Content of pipelines.yml configuration file.
pipelines_yml_config: |-
--
GitLab
From 9941970264b1fb994e252e0e671231d8feda871c Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 14:49:09 +0200
Subject: [PATCH 04/61] added logstash
---
templates/elasticsearch.yaml | 29 +++++++++++++++++++++++++++++
values.yaml | 16 ++++++++--------
2 files changed, 37 insertions(+), 8 deletions(-)
diff --git a/templates/elasticsearch.yaml b/templates/elasticsearch.yaml
index 555a722..83c0d24 100644
--- a/templates/elasticsearch.yaml
+++ b/templates/elasticsearch.yaml
@@ -5,6 +5,9 @@ metadata:
spec:
version: {{ .Values.elasticVersion }}
image: {{ .Values.elasticsearch.image }}:{{ default .Values.elasticVersion .Values.elasticsearch.imageTag }}
+ auth:
+ fileRealm:
+ - secretName: logstash-writer-secret
nodeSets:
{{- range .Values.elasticsearch.nodeSets }}
- name: {{ .name }}
@@ -92,6 +95,32 @@ spec:
port:
number: 9200
---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-writer-secret
+type: kubernetes.io/basic-auth
+data:
+ username: {{ "logstash-writer" | b64enc }}
+ {{- if .Release.IsInstall }}
+ password: {{ randAlphaNum 20 | b64enc }}
+ {{ else }}
+ password: {{ index (lookup "v1" "Secret" .Release.Namespace "logstash-writer-secret").data "password" }}
+ {{ end }}
+ roles: {{ "logstash-writer-role" | b64enc }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+ name: logstash-writer-role-secret
+stringData:
+ roles.yml: |-
+ logstash-writer-role:
+ cluster: ["manage_index_templates", "monitor", "manage_ilm"]
+ indices:
+ - names: [ '*' ]
+ privileges: ["read","write","create","create_index","manage","manage_ilm"]
+---
diff --git a/values.yaml b/values.yaml
index 424e9eb..3f2207f 100644
--- a/values.yaml
+++ b/values.yaml
@@ -168,14 +168,14 @@ logstash:
output: |-
output {
- elasticsearch {
- hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- #index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
- user => "${LOGSTASH_USER}"
- password => "${LOGSTASH_PASSWORD}"
- #ssl_enabled => "true"
- ssl_verification_mode => "none"
- }
+ #elasticsearch {
+ # hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ # index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ # user => "${LOGSTASH_USER}"
+ # password => "${LOGSTASH_PASSWORD}"
+ # #ssl_enabled => "true"
+ # ssl_verification_mode => "none"
+ #}
stdout {
codec => rubydebug
}
--
GitLab
From 2c0ecfa5f38db0526ab6b5b44330c5ce25834b0e Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 15:38:25 +0200
Subject: [PATCH 05/61] added logstash
---
templates/logstash.yaml | 29 +----------------------------
values.yaml | 2 +-
2 files changed, 2 insertions(+), 29 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 57e421d..1a2d6bb 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -121,33 +121,6 @@ data:
{{ tpl .output $ | nindent 6 }}
---
{{- end }}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
- name: logstash-api-{{ .Values.logstash.pipelines_group_name }}
- annotations:
- kubernetes.io/ingress.class: nginx
- nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
- acme.cert-manager.io/http01-edit-in-place: "true"
- cert-manager.io/cluster-issuer: dev-staging
-spec:
- {{- if .Values.logstash.enableTLS }}
- tls:
- - hosts:
- - {{ template "logstash.dns" . }}
- secretName: logstash-secret-{{ .Values.logstash.pipelines_group_name }}
- {{- end }}
- rules:
- - host: {{ template "logstash.dns" . }}
- http:
- paths:
- - path: /
- pathType: Prefix
- backend:
- service:
- name: logstash-{{ .Values.logstash.pipelines_group_name }}-ls-api
- port:
- number: 9600
-
+---
{{- end }}
diff --git a/values.yaml b/values.yaml
index 3f2207f..0b3ebb7 100644
--- a/values.yaml
+++ b/values.yaml
@@ -138,7 +138,7 @@ logstash:
env: []
# Use TLS in ingress for logstash input.
- enableTLS: true
+ enableTLS: false
# Content of pipelines.yml configuration file.
pipelines_yml_config: |-
--
GitLab
From feebeac593369be0b09ad699d71433518172ab24 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 15:52:10 +0200
Subject: [PATCH 06/61] added logstash
---
templates/logstash.yaml | 155 ++++++++++++----------------------------
1 file changed, 44 insertions(+), 111 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 1a2d6bb..ec396ba 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -1,126 +1,59 @@
-{{- if gt (.Values.logstash.count | int) 0 }}
----
apiVersion: logstash.k8s.elastic.co/v1alpha1
kind: Logstash
metadata:
- name: logstash-{{ .Values.logstash.pipelines_group_name }}
+ name: logstash
spec:
- image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
version: {{ .Values.elasticVersion }}
- count: {{ .Values.logstash.count }}
- elasticsearchRefs:
- - name: {{ .Release.Name }}-elasticsearch
- clusterName: {{ .Release.Name }}-elasticsearch
- volumeClaimTemplates:
- - metadata:
- name: logstash-data
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: {{ .Values.logstash.diskSpace }}
- storageClassName: {{ .Values.logstash.storageClassName }}
- services:
-{{- range $index := until (.Values.logstash.count |int ) -}}
-{{- printf "\n"}}
- - name: beats-{{$index}}
- {{- if $.Values.logstash.enableTLS }}
- tls:
- certificate:
- secretName: logstash-secret-{{ $.Values.logstash.pipelines_group_name }}
- {{- end }}
- service:
- spec:
- ports:
- - port: 5044
- name: {{ $.Values.logstash.pipelines_group_name }}
- protocol: TCP
- selector:
- statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
-{{- end}}
+ count: 1
+ elasticsearchRef:
+ name: "elasticsearch"
podTemplate:
- metadata:
- labels:
- stack-namespace: {{ .Release.Namespace }}
spec:
- imagePullpolicy: Always
- {{- with $.Values.imagePullSecrets }}
- imagePullSecrets:
- {{- toYaml . | nindent 10 }}
- {{- end }}
- volumes:
- {{- range .Values.logstash.pipelines }}
- - name: pipeline-config-{{- .name }}
- configMap:
- name: logstash-{{- $.Values.logstash.pipelines_group_name -}}-{{- .name -}}-config
- defaultMode: 511
- {{- end }}
- {{- if .Values.logstash.enableTLS }}
- - name: certs-logstash
- secret:
- secretName: logstash-secret-{{ .Values.logstash.pipelines_group_name }}
- {{- end}}
containers:
- name: logstash
- # command: ["/bin/bash", "-c", "sleep inf"]
- {{- with .Values.logstash.resources }}
- resources:
- {{- toYaml . | nindent 10 }}
- {{- end }}
volumeMounts:
- {{- range .Values.logstash.pipelines }}
- - name: pipeline-config-{{- .name }}
- mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
- subPath: {{ .name -}}.config
- {{- end }}
- {{- if .Values.logstash.enableTLS }}
- - mountPath: /usr/share/logstash/certs-logstash
- name: certs-logstash
- {{- end }}
- secureSettings:
- - secretName: {{ .Release.Namespace }}-logstash-{{ .Values.logstash.pipelines_group_name }}-{{ .Release.Namespace }}-{{ .Release.Name }}-elasticsearch-logstash-user
- env:
- - name: LOGSTASH_USER
- valueFrom:
- secretKeyRef:
- name: logstash-writer-secret
- key: username
- - name: LOGSTASH_PASSWORD
- valueFrom:
- secretKeyRef:
- name: logstash-writer-secret
- key: password
- {{- with .Values.logstash.env }}
- {{- toYaml . | nindent 10 }}
- {{- end }}
- config:
- {{- with .Values.logstash.config }}
- {{- toYaml . | nindent 4 }}
- {{- end }}
- pipelinesRef:
- secretName: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
-data:
- pipelines.yml: |
- {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+ - name: logstash-pipeline
+ mountPath: /usr/share/logstash/pipeline/logstash.conf
+ subPath: logstash.conf
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-pipeline
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: 1Gi
+ config:
+ logstash.yml: |
+ http.host: "0.0.0.0"
+ xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch-es-http:9200" ]
+ pipelines:
+ - pipeline.id: main
+ path.config: /usr/share/logstash/pipeline/logstash.conf
+ volumes:
+ - name: logstash-pipeline
+ configMap:
+ name: logstash-pipeline
---
-{{- range .Values.logstash.pipelines }}
apiVersion: v1
kind: ConfigMap
metadata:
- name: logstash-{{ $.Values.logstash.pipelines_group_name }}-{{ .name }}-config
+ name: logstash-pipeline
+ namespace: default
+ labels:
+ app: logstash
data:
- {{ .name }}.config: |
- {{ tpl .input $ | nindent 6 }}
- {{ tpl .filter $ | nindent 6 }}
- {{ tpl .output $ | nindent 6 }}
----
-{{- end }}
-
----
-{{- end }}
+ logstash.conf: |
+ input {
+ beats {
+ port => 5044
+ }
+ }
+ output {
+ elasticsearch {
+ hosts => ["http://elasticsearch-es-http:9200"]
+ index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
+ user => "elastic"
+ password => "test"
+ }
+ }
\ No newline at end of file
--
GitLab
From 7568a22e857670dd8ee954cfbdc089093c93837b Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:02:15 +0200
Subject: [PATCH 07/61] added logstash
---
templates/logstash.yaml | 59 ---------------------------
values.yaml | 89 ++++++-----------------------------------
2 files changed, 13 insertions(+), 135 deletions(-)
delete mode 100644 templates/logstash.yaml
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
deleted file mode 100644
index ec396ba..0000000
--- a/templates/logstash.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-apiVersion: logstash.k8s.elastic.co/v1alpha1
-kind: Logstash
-metadata:
- name: logstash
-spec:
- version: {{ .Values.elasticVersion }}
- count: 1
- elasticsearchRef:
- name: "elasticsearch"
- podTemplate:
- spec:
- containers:
- - name: logstash
- volumeMounts:
- - name: logstash-pipeline
- mountPath: /usr/share/logstash/pipeline/logstash.conf
- subPath: logstash.conf
- volumeClaimTemplates:
- - metadata:
- name: logstash-pipeline
- spec:
- accessModes: [ "ReadWriteOnce" ]
- resources:
- requests:
- storage: 1Gi
- config:
- logstash.yml: |
- http.host: "0.0.0.0"
- xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch-es-http:9200" ]
- pipelines:
- - pipeline.id: main
- path.config: /usr/share/logstash/pipeline/logstash.conf
- volumes:
- - name: logstash-pipeline
- configMap:
- name: logstash-pipeline
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: logstash-pipeline
- namespace: default
- labels:
- app: logstash
-data:
- logstash.conf: |
- input {
- beats {
- port => 5044
- }
- }
- output {
- elasticsearch {
- hosts => ["http://elasticsearch-es-http:9200"]
- index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
- user => "elastic"
- password => "test"
- }
- }
\ No newline at end of file
diff --git a/values.yaml b/values.yaml
index 0b3ebb7..e926608 100644
--- a/values.yaml
+++ b/values.yaml
@@ -100,88 +100,25 @@ kibana:
logstash:
- # set >0 to deploy logstash , 0 otherwise
- count: 1
- urlPrefix: ""
- image: docker.elastic.co/logstash/logstash
- # Logstash's image tag, by default it equals to elasticVersion
- imageTag: ""
- jvm_options_xms: 1g
- jvm_options_xmx: 1g
- resources:
- requests:
- memory: 4Gi
- limits:
- memory: 4Gi
- cert:
- duration: 2160h0m0s # 90d
- renewBefore: 360h0m0s # 15d
- # Set to false when there is logstash already deployed in the same namespace
- createConfigMaps: true
- # Indicates if logstash should be connected to elasticsearch
- isElasticsearchDeployed: true
- custompatternsactive: false
- dropwizardactive: false
- # persistent volume claim size
- diskSpace: 1Gi
- # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
- storageClassName: csi-cinder-high-speed
-
- # Here you can specify extra logstash.yml config e.g
- # config:
- # xpack.monitoring.enabled: "true"
- config: {}
-
- #Environment variables to set in logstash pod
- #Usage from cli:
- # --set "logstash.env[0].name=VARIABLE_NAME" --set "logstash.env[0].value=VARIABLE_VALUE"
- env: []
-
- # Use TLS in ingress for logstash input.
- enableTLS: false
-
- # Content of pipelines.yml configuration file.
- pipelines_yml_config: |-
- - pipeline.id: main
- path.config: "/app/elastic/logstash/config/pipelines/*.config"
- pipeline.workers: 1
- pipeline.batch.size: 125
-
- pipelines_group_name: "beats"
- workers: 1
- batch:
- size: 125
- pipelines:
- # Name of pipeline. This does not affect any logstash functionality,
- # but creates unique, human-readable names of k8s resources
- - name: "input"
- # Input of logstash pipeline
- # The default one is an input for beats
- input: |-
+ enabled: true
+ replicas: 1
+ imageTag: "8.0.0"
+ env:
+ LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+ pipeline:
+ logstash.conf: |
input {
beats {
port => 5044
}
}
- filter: |-
- filter {}
-
- output: |-
output {
- #elasticsearch {
- # hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- # index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
- # user => "${LOGSTASH_USER}"
- # password => "${LOGSTASH_PASSWORD}"
- # #ssl_enabled => "true"
- # ssl_verification_mode => "none"
- #}
- stdout {
- codec => rubydebug
+ elasticsearch {
+ hosts => ["http://elasticsearch-es-http:9200"]
+ index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
+ user => "elastic"
+ password => "your_password"
}
- }
-
-
-
+ }
--
GitLab
From 593b81a7196150b7b8df8ebc420129ca467cd719 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:26:44 +0200
Subject: [PATCH 08/61] added logstash
---
templates/logstash.yaml | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
create mode 100644 templates/logstash.yaml
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
new file mode 100644
index 0000000..fabd980
--- /dev/null
+++ b/templates/logstash.yaml
@@ -0,0 +1,30 @@
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Logstash
+metadata:
+ name: {{ include "logstash.fullname" . }}
+ labels:
+ app: logstash
+spec:
+ version: "{{ .Values.logstash.imageTag }}"
+ count: {{ .Values.logstash.replicas }}
+ config:
+ input {
+ beats {
+ port => 5044
+ }
+ }
+ output {
+ elasticsearch {
+ hosts => ["http://elasticsearch-es-http:9200"]
+ index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
+ user => "elastic"
+ password => "your_password"
+ }
+ }
+ podTemplate:
+ spec:
+ containers:
+ - name: logstash
+ env:
+ - name: LS_JAVA_OPTS
+ value: "{{ .Values.logstash.env.LS_JAVA_OPTS }}"
\ No newline at end of file
--
GitLab
From 624bd8b0eb4750e06888885ddbf26ca35e20a0b6 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:30:36 +0200
Subject: [PATCH 09/61] added logstash
---
templates/logstash.yaml | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index fabd980..122b652 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -1,12 +1,15 @@
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Logstash
metadata:
- name: {{ include "logstash.fullname" . }}
+ name: logstash-beats
labels:
app: logstash
spec:
version: "{{ .Values.logstash.imageTag }}"
count: {{ .Values.logstash.replicas }}
+ elasticsearchRefs:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
config:
input {
beats {
--
GitLab
From 08c94c40400e77d4b8fbec3fc0a0bfd990831b43 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:34:28 +0200
Subject: [PATCH 10/61] added logstash
---
values.yaml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/values.yaml b/values.yaml
index e926608..09542c7 100644
--- a/values.yaml
+++ b/values.yaml
@@ -102,7 +102,9 @@ kibana:
logstash:
enabled: true
replicas: 1
- imageTag: "8.0.0"
+ image: docker.elastic.co/logstash/logstash
+
+ imageTag: ""
env:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
pipeline:
--
GitLab
From 61615500995d93d09d49fae5e9035694594a4391 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:35:52 +0200
Subject: [PATCH 11/61] added logstash
---
templates/logstash.yaml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 122b652..362fc81 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -1,11 +1,12 @@
-apiVersion: elasticsearch.k8s.elastic.co/v1
+apiVersion: logstash.k8s.elastic.co/v1alpha1
kind: Logstash
metadata:
name: logstash-beats
labels:
app: logstash
spec:
- version: "{{ .Values.logstash.imageTag }}"
+ image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
+ version: {{ .Values.elasticVersion }}
count: {{ .Values.logstash.replicas }}
elasticsearchRefs:
- name: {{ .Release.Name }}-elasticsearch
--
GitLab
From 75fe63d592a072730f9cf99a9712d15739f8dae4 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:44:42 +0200
Subject: [PATCH 12/61] added logstash
---
templates/logstash.yaml | 37 +++++++++++++++++++------------------
values.yaml | 1 +
2 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 362fc81..ee40008 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -11,24 +11,25 @@ spec:
elasticsearchRefs:
- name: {{ .Release.Name }}-elasticsearch
clusterName: {{ .Release.Name }}-elasticsearch
- config:
- input {
- beats {
- port => 5044
- }
- }
- output {
- elasticsearch {
- hosts => ["http://elasticsearch-es-http:9200"]
- index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
- user => "elastic"
- password => "your_password"
- }
- }
+ config:
+ {{- with .Values.logstash.config }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
podTemplate:
spec:
containers:
- - name: logstash
- env:
- - name: LS_JAVA_OPTS
- value: "{{ .Values.logstash.env.LS_JAVA_OPTS }}"
\ No newline at end of file
+ - name: logstash
+ volumeMounts:
+ - name: logstash-pipeline
+ mountPath: /usr/share/logstash/pipeline/logstash.conf
+ subPath: logstash.conf
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-pipeline
+ pipelines:
+ - pipeline.id: main
+ path.config: /usr/share/logstash/pipeline/logstash.conf
+ volumes:
+ - name: logstash-pipeline
+ configMap:
+ name: logstash-pipeline
\ No newline at end of file
diff --git a/values.yaml b/values.yaml
index 09542c7..03eae1c 100644
--- a/values.yaml
+++ b/values.yaml
@@ -103,6 +103,7 @@ logstash:
enabled: true
replicas: 1
image: docker.elastic.co/logstash/logstash
+ config: {}
imageTag: ""
env:
--
GitLab
From 50e1dcf10e510c2b4b52463244db1ed33e745ab9 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 16:49:37 +0200
Subject: [PATCH 13/61] added logstash
---
templates/logstash.yaml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index ee40008..0f991de 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -23,13 +23,13 @@ spec:
- name: logstash-pipeline
mountPath: /usr/share/logstash/pipeline/logstash.conf
subPath: logstash.conf
+ volumes:
+ - name: logstash-pipeline
+ configMap:
+ name: logstash-pipeline
volumeClaimTemplates:
- metadata:
name: logstash-pipeline
pipelines:
- pipeline.id: main
path.config: /usr/share/logstash/pipeline/logstash.conf
- volumes:
- - name: logstash-pipeline
- configMap:
- name: logstash-pipeline
\ No newline at end of file
--
GitLab
From ef43cb730d2b6222e4e348680cb97cea6db3470d Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 17:03:16 +0200
Subject: [PATCH 14/61] added logstash
---
values.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/values.yaml b/values.yaml
index 03eae1c..a202614 100644
--- a/values.yaml
+++ b/values.yaml
@@ -1,5 +1,5 @@
# version of all elastic applications
-elasticVersion: 8.11.1
+elasticVersion: 8.14.1
# Name of aws environment
envTag: "dev"
--
GitLab
From d2ce4563e0d5b441da993b41e512869b8be3976f Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 17:12:57 +0200
Subject: [PATCH 15/61] added logstash
---
values.yaml | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/values.yaml b/values.yaml
index a202614..a7021ac 100644
--- a/values.yaml
+++ b/values.yaml
@@ -116,11 +116,14 @@ logstash:
}
}
output {
- elasticsearch {
- hosts => ["http://elasticsearch-es-http:9200"]
- index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
- user => "elastic"
- password => "your_password"
+ #elasticsearch {
+ # hosts => ["http://elasticsearch-es-http:9200"]
+ # index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
+ # user => "elastic"
+ # password => "your_password"
+ #}
+ stdout {
+ codec => rubydebug
}
}
--
GitLab
From 895e2a6869522c77979742d25f91811a35575ceb Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 17:25:41 +0200
Subject: [PATCH 16/61] added logstash
---
templates/logstash.yaml | 13 +++++++++++++
values.yaml | 12 ++++++------
2 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 0f991de..22563be 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -23,6 +23,19 @@ spec:
- name: logstash-pipeline
mountPath: /usr/share/logstash/pipeline/logstash.conf
subPath: logstash.conf
+ env:
+ - name: LOGSTASH_USER
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: username
+ - name: LOGSTASH_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: password
+ - name: ELASTIC_ELASTICSEARCH_ES_HOSTS
+ value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
volumes:
- name: logstash-pipeline
configMap:
diff --git a/values.yaml b/values.yaml
index a7021ac..d259af6 100644
--- a/values.yaml
+++ b/values.yaml
@@ -116,12 +116,12 @@ logstash:
}
}
output {
- #elasticsearch {
- # hosts => ["http://elasticsearch-es-http:9200"]
- # index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
- # user => "elastic"
- # password => "your_password"
- #}
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ }
stdout {
codec => rubydebug
}
--
GitLab
From 182c51d0d2a8a3a6514a1581d5017b41578b6a72 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 17:34:30 +0200
Subject: [PATCH 17/61] added logstash
---
templates/logstash.yaml | 13 +++++++++++--
values.yaml | 4 +++-
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 22563be..b564aaa 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -41,8 +41,17 @@ spec:
configMap:
name: logstash-pipeline
volumeClaimTemplates:
- - metadata:
- name: logstash-pipeline
+ - metadata:
+ name: logstash-data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
+ - metadata:
+ name: logstash-pipeline
pipelines:
- pipeline.id: main
path.config: /usr/share/logstash/pipeline/logstash.conf
diff --git a/values.yaml b/values.yaml
index d259af6..d70a380 100644
--- a/values.yaml
+++ b/values.yaml
@@ -104,7 +104,9 @@ logstash:
replicas: 1
image: docker.elastic.co/logstash/logstash
config: {}
-
+ diskSpace: 3Gi
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ storageClassName: csi-cinder-high-speed
imageTag: ""
env:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
--
GitLab
From 07709f4095882af661cdefc736634b88e8cc82cc Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 17:41:41 +0200
Subject: [PATCH 18/61] added logstash
---
templates/logstash.yaml | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index b564aaa..5d39417 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -52,6 +52,13 @@ spec:
storageClassName: {{ .Values.logstash.storageClassName }}
- metadata:
name: logstash-pipeline
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
pipelines:
- pipeline.id: main
path.config: /usr/share/logstash/pipeline/logstash.conf
--
GitLab
From 1a880e4485d62a8bf7e117a95cfb07763ef218f9 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 17:56:35 +0200
Subject: [PATCH 19/61] added logstash
---
templates/logstash.yaml | 90 ++++++++++++++++++++++++++++-------------
values.yaml | 27 +++++++++++--
2 files changed, 86 insertions(+), 31 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 5d39417..2b79eee 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -11,18 +11,50 @@ spec:
elasticsearchRefs:
- name: {{ .Release.Name }}-elasticsearch
clusterName: {{ .Release.Name }}-elasticsearch
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
+ services:
+{{- range $index := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+ - name: beats-{{$index}}
+ service:
+ spec:
+ ports:
+ - port: 5044
+ name: {{ $.Values.logstash.pipelines_group_name }}
+ protocol: TCP
+ selector:
+ statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
+{{- end}}
config:
{{- with .Values.logstash.config }}
{{- toYaml . | nindent 4 }}
{{- end }}
podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
spec:
containers:
- name: logstash
+ {{- with .Values.logstash.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
volumeMounts:
- - name: logstash-pipeline
- mountPath: /usr/share/logstash/pipeline/logstash.conf
- subPath: logstash.conf
+ {{- range .Values.logstash.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
+ subPath: {{ .name -}}.config
+ {{- end }}
env:
- name: LOGSTASH_USER
valueFrom:
@@ -37,28 +69,32 @@ spec:
- name: ELASTIC_ELASTICSEARCH_ES_HOSTS
value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
volumes:
- - name: logstash-pipeline
+ {{- range .Values.logstash.pipelines }}
+ - name: pipeline-config-{{- .name }}
configMap:
- name: logstash-pipeline
- volumeClaimTemplates:
- - metadata:
- name: logstash-data
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: {{ .Values.logstash.diskSpace }}
- storageClassName: {{ .Values.logstash.storageClassName }}
- - metadata:
- name: logstash-pipeline
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: {{ .Values.logstash.diskSpace }}
- storageClassName: {{ .Values.logstash.storageClassName }}
- pipelines:
- - pipeline.id: main
- path.config: /usr/share/logstash/pipeline/logstash.conf
+ name: logstash-{{- $.Values.logstash.pipelines_group_name -}}-{{- .name -}}-config
+ defaultMode: 511
+ {{- end }}
+ pipelinesRef:
+ secretName: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+data:
+ pipelines.yml: |
+ {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+---
+{{- range .Values.logstash.pipelines }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logstash-{{ $.Values.logstash.pipelines_group_name }}-{{ .name }}-config
+data:
+ {{ .name }}.config: |
+ {{ tpl .input $ | nindent 6 }}
+ {{ tpl .filter $ | nindent 6 }}
+ {{ tpl .output $ | nindent 6 }}
+---
+{{- end }}
diff --git a/values.yaml b/values.yaml
index d70a380..8d040d3 100644
--- a/values.yaml
+++ b/values.yaml
@@ -100,8 +100,7 @@ kibana:
logstash:
- enabled: true
- replicas: 1
+ count: 2
image: docker.elastic.co/logstash/logstash
config: {}
diskSpace: 3Gi
@@ -110,13 +109,33 @@ logstash:
imageTag: ""
env:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
- pipeline:
- logstash.conf: |
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ pipelines_yml_config: |-
+ - pipeline.id: main
+ path.config: "/app/elastic/logstash/config/pipelines/*.config"
+ pipeline.workers: 1
+ pipeline.batch.size: 125
+
+ pipelines_group_name: "beats"
+ workers: 1
+ batch:
+ size: 125
+ pipelines:
+ - name: "beats-pipeline"
+ input: |-
input {
beats {
port => 5044
}
}
+ filter: |-
+ filter {}
+
+ output: |-
output {
elasticsearch {
hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
--
GitLab
From 45932ff5008e9e62a2b397a1fb644d359194f0ad Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 18:03:33 +0200
Subject: [PATCH 20/61] added logstash
---
values.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/values.yaml b/values.yaml
index 8d040d3..c4d566c 100644
--- a/values.yaml
+++ b/values.yaml
@@ -100,7 +100,7 @@ kibana:
logstash:
- count: 2
+ count: 1
image: docker.elastic.co/logstash/logstash
config: {}
diskSpace: 3Gi
@@ -134,7 +134,7 @@ logstash:
}
filter: |-
filter {}
-
+
output: |-
output {
elasticsearch {
--
GitLab
From 8928edd9066018fc84e0a73f97aebb37cf9414a9 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 18:25:59 +0200
Subject: [PATCH 21/61] added logstash
---
templates/logstash.yaml | 16 +++++++++-------
values.yaml | 26 ++++++++++++++++++++++++--
2 files changed, 33 insertions(+), 9 deletions(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index 2b79eee..b8c4e25 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -21,19 +21,21 @@ spec:
requests:
storage: {{ .Values.logstash.diskSpace }}
storageClassName: {{ .Values.logstash.storageClassName }}
- services:
-{{- range $index := until (.Values.logstash.count |int ) -}}
-{{- printf "\n"}}
- - name: beats-{{$index}}
+{{- range .Values.logstash.pipelines }}
+ services:
+ - name: {{ .Values.logstash.pipelines_group_name }}
service:
spec:
ports:
- port: 5044
- name: {{ $.Values.logstash.pipelines_group_name }}
+ name: {{ .name }}
+ protocol: TCP
+ - port: 514
+ name: {{ .name }}
protocol: TCP
selector:
- statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
-{{- end}}
+ statefulset.kubernetes.io/pod-name: logstash-beats-ls-0
+ {{- end }}
config:
{{- with .Values.logstash.config }}
{{- toYaml . | nindent 4 }}
diff --git a/values.yaml b/values.yaml
index c4d566c..09503e2 100644
--- a/values.yaml
+++ b/values.yaml
@@ -100,7 +100,7 @@ kibana:
logstash:
- count: 1
+ count: 2
image: docker.elastic.co/logstash/logstash
config: {}
diskSpace: 3Gi
@@ -120,7 +120,7 @@ logstash:
pipeline.workers: 1
pipeline.batch.size: 125
- pipelines_group_name: "beats"
+ pipelines_group_name: "pipelines"
workers: 1
batch:
size: 125
@@ -147,5 +147,27 @@ logstash:
codec => rubydebug
}
}
+ - name: "syslog-pipeline"
+ input: |-
+ input {
+ syslog {
+ port => 514
+ }
+ }
+ filter: |-
+ filter {}
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
--
GitLab
From 81b91dfc90ce7ec8fd3949ab7f6b8b1aad7dd553 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 18:29:31 +0200
Subject: [PATCH 22/61] added logstash
---
templates/logstash.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/logstash.yaml b/templates/logstash.yaml
index b8c4e25..7de6400 100644
--- a/templates/logstash.yaml
+++ b/templates/logstash.yaml
@@ -23,7 +23,7 @@ spec:
storageClassName: {{ .Values.logstash.storageClassName }}
{{- range .Values.logstash.pipelines }}
services:
- - name: {{ .Values.logstash.pipelines_group_name }}
+ - name: {{ $.Values.logstash.pipelines_group_name }}
service:
spec:
ports:
--
GitLab
From 78e01c11bd26d7cabb9810a98eacd2945844752b Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 19:24:55 +0200
Subject: [PATCH 23/61] added logstash
---
templates/logstash_beats.yaml | 100 ++++++++++
.../logstash.yaml => tmp/logstash_syslog.yaml | 0
tmp/values.yaml | 173 ++++++++++++++++++
values.yaml | 25 +--
4 files changed, 274 insertions(+), 24 deletions(-)
create mode 100644 templates/logstash_beats.yaml
rename templates/logstash.yaml => tmp/logstash_syslog.yaml (100%)
create mode 100644 tmp/values.yaml
diff --git a/templates/logstash_beats.yaml b/templates/logstash_beats.yaml
new file mode 100644
index 0000000..2b79eee
--- /dev/null
+++ b/templates/logstash_beats.yaml
@@ -0,0 +1,100 @@
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+ name: logstash-beats
+ labels:
+ app: logstash
+spec:
+ image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
+ version: {{ .Values.elasticVersion }}
+ count: {{ .Values.logstash.replicas }}
+ elasticsearchRefs:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
+ volumeClaimTemplates:
+ - metadata:
+ name: logstash-data
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.logstash.diskSpace }}
+ storageClassName: {{ .Values.logstash.storageClassName }}
+ services:
+{{- range $index := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+ - name: beats-{{$index}}
+ service:
+ spec:
+ ports:
+ - port: 5044
+ name: {{ $.Values.logstash.pipelines_group_name }}
+ protocol: TCP
+ selector:
+ statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
+{{- end}}
+ config:
+ {{- with .Values.logstash.config }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
+ spec:
+ containers:
+ - name: logstash
+ {{- with .Values.logstash.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{- end }}
+ volumeMounts:
+ {{- range .Values.logstash.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
+ subPath: {{ .name -}}.config
+ {{- end }}
+ env:
+ - name: LOGSTASH_USER
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: username
+ - name: LOGSTASH_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: logstash-writer-secret
+ key: password
+ - name: ELASTIC_ELASTICSEARCH_ES_HOSTS
+ value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
+ volumes:
+ {{- range .Values.logstash.pipelines }}
+ - name: pipeline-config-{{- .name }}
+ configMap:
+ name: logstash-{{- $.Values.logstash.pipelines_group_name -}}-{{- .name -}}-config
+ defaultMode: 511
+ {{- end }}
+ pipelinesRef:
+ secretName: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+data:
+ pipelines.yml: |
+ {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+---
+{{- range .Values.logstash.pipelines }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logstash-{{ $.Values.logstash.pipelines_group_name }}-{{ .name }}-config
+data:
+ {{ .name }}.config: |
+ {{ tpl .input $ | nindent 6 }}
+ {{ tpl .filter $ | nindent 6 }}
+ {{ tpl .output $ | nindent 6 }}
+---
+{{- end }}
diff --git a/templates/logstash.yaml b/tmp/logstash_syslog.yaml
similarity index 100%
rename from templates/logstash.yaml
rename to tmp/logstash_syslog.yaml
diff --git a/tmp/values.yaml b/tmp/values.yaml
new file mode 100644
index 0000000..09503e2
--- /dev/null
+++ b/tmp/values.yaml
@@ -0,0 +1,173 @@
+# version of all elastic applications
+elasticVersion: 8.14.1
+
+# Name of aws environment
+envTag: "dev"
+
+# This suffix will be used to create subdomain of following template:
+# kibana.NAMESPACE_TAG.DOMAIN_SUFFIX
+domainSuffix: "dev"
+
+# This value is used to create unique dns for each deployment. By default it equals to namespace.
+namespaceTag: ""
+
+
+# Spread pods evenly between subnets
+createTopologySpreadConstraints: true
+
+# Force pods to be scheduled evenly between separete AZ, this means if all nodes in AZ will be full pod won't be scheduled.
+# Turning this off will still create pods in seperate AZ, but k8s scheduler won't be able to respect topologySpreadConstrains, will schedule pods in the same AZ.
+forceDeployInSeperateAZ: true
+
+
+# Open port 443 for each component on the Traefik level.
+openStandardHttpsPorts: true
+
+# imagePullSecrets defines the secrets with credentials to docker registry, for example nexus.
+imagePullSecrets: []
+
+# Precise on which nodes should pods be deployed if label is available - if not, it will go on random nodes.
+affinity:
+ purpose: simpl
+
+
+
+
+
+
+elasticsearch:
+
+ image: docker.elastic.co/elasticsearch/elasticsearch
+ # Elasticsearch's image tag, by default it equals to elasticVersion
+ ccr_remote_ca_file: ccr-remote-ca.pem
+ imageTag: ""
+ jvm_options_xms: 1g
+ jvm_options_xmx: 1g
+ cert:
+ duration: 2160h0m0s # 90d
+ renewBefore: 360h0m0s # 15d
+ #Environment variables to set in kibana pod
+ #Usage from cli:
+ # --set "elasticsearch.env[0].name=VARIABLE_NAME" --set "elasticsearch.env[0].value=VARIABLE_VALUE"
+ env: []
+
+
+ # here you can specify several node groups (e.g. master, data groups)
+ nodeSets:
+ - name: "node"
+ # number of elasticsearch nodes in each group
+ count: 3
+ # persistent volume claim size
+ diskSpace: 3Gi
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ storageClassName: csi-cinder-high-speed
+ # here you can specify elasticsearch config e.g
+ # config:
+ # node.roles: ["master"]
+ config: {}
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ cpu: "1"
+
+kibana:
+ # set >0 to deploy kibana, 0 otherwise
+ count: 2
+ image: docker.elastic.co/kibana/kibana
+ # Kibana's image tag, by default it equals to elasticVersion
+ imageTag: ""
+ # name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
+ elasticsearchReleaseName: ""
+ # Kibana server will listen on that subpath
+ cert:
+ duration: 2160h0m0s # 90d
+ renewBefore: 360h0m0s # 15d
+ # Additional kibana's config according to this link: https://www.elastic.co/guide/en/kibana/current/settings.html
+ config:
+ xpack.reporting.roles.enabled: false
+ resources:
+ requests:
+ memory: 1Gi
+ limits:
+ memory: 1Gi
+
+ #Environment variables to set in kibana pod
+ #Usage from cli:
+ # --set "kibana.env[0].name=VARIABLE_NAME" --set "kibana.env[0].value=VARIABLE_VALUE"
+ env: []
+
+
+logstash:
+ count: 2
+ image: docker.elastic.co/logstash/logstash
+ config: {}
+ diskSpace: 3Gi
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ storageClassName: csi-cinder-high-speed
+ imageTag: ""
+ env:
+ LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+ resources:
+ requests:
+ memory: 4Gi
+ limits:
+ memory: 4Gi
+ pipelines_yml_config: |-
+ - pipeline.id: main
+ path.config: "/app/elastic/logstash/config/pipelines/*.config"
+ pipeline.workers: 1
+ pipeline.batch.size: 125
+
+ pipelines_group_name: "pipelines"
+ workers: 1
+ batch:
+ size: 125
+ pipelines:
+ - name: "beats-pipeline"
+ input: |-
+ input {
+ beats {
+ port => 5044
+ }
+ }
+ filter: |-
+ filter {}
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
+ - name: "syslog-pipeline"
+ input: |-
+ input {
+ syslog {
+ port => 514
+ }
+ }
+ filter: |-
+ filter {}
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
+
+
diff --git a/values.yaml b/values.yaml
index 09503e2..825a6d9 100644
--- a/values.yaml
+++ b/values.yaml
@@ -147,27 +147,4 @@ logstash:
codec => rubydebug
}
}
- - name: "syslog-pipeline"
- input: |-
- input {
- syslog {
- port => 514
- }
- }
- filter: |-
- filter {}
-
- output: |-
- output {
- elasticsearch {
- hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
- user => "${LOGSTASH_USER}"
- password => "${LOGSTASH_PASSWORD}"
- }
- stdout {
- codec => rubydebug
- }
- }
-
-
+
--
GitLab
From a5561ab28f4f24e868baae560e90484b6220fea1 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 19:56:22 +0200
Subject: [PATCH 24/61] added logstash
---
templates/logstash_beats.yaml | 18 +++---
{tmp => templates}/logstash_syslog.yaml | 34 ++++++------
values.yaml | 73 +++++++++++++++++--------
3 files changed, 75 insertions(+), 50 deletions(-)
rename {tmp => templates}/logstash_syslog.yaml (72%)
diff --git a/templates/logstash_beats.yaml b/templates/logstash_beats.yaml
index 2b79eee..20fda18 100644
--- a/templates/logstash_beats.yaml
+++ b/templates/logstash_beats.yaml
@@ -7,7 +7,7 @@ metadata:
spec:
image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
version: {{ .Values.elasticVersion }}
- count: {{ .Values.logstash.replicas }}
+ count: {{ .Values.logstash.count }}
elasticsearchRefs:
- name: {{ .Release.Name }}-elasticsearch
clusterName: {{ .Release.Name }}-elasticsearch
@@ -29,7 +29,7 @@ spec:
spec:
ports:
- port: 5044
- name: {{ $.Values.logstash.pipelines_group_name }}
+ name: {{ $.Values.logstash.beats.pipelines_group_name }}
protocol: TCP
selector:
statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
@@ -50,7 +50,7 @@ spec:
{{- toYaml . | nindent 10 }}
{{- end }}
volumeMounts:
- {{- range .Values.logstash.pipelines }}
+ {{- range .Values.logstash.beats.pipelines }}
- name: pipeline-config-{{- .name }}
mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
subPath: {{ .name -}}.config
@@ -69,28 +69,28 @@ spec:
- name: ELASTIC_ELASTICSEARCH_ES_HOSTS
value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
volumes:
- {{- range .Values.logstash.pipelines }}
+ {{- range .Values.logstash.beats.pipelines }}
- name: pipeline-config-{{- .name }}
configMap:
- name: logstash-{{- $.Values.logstash.pipelines_group_name -}}-{{- .name -}}-config
+ name: logstash-{{- $.Values.logstash.beats.pipelines_group_name -}}-{{- .name -}}-config
defaultMode: 511
{{- end }}
pipelinesRef:
- secretName: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+ secretName: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
---
apiVersion: v1
kind: Secret
metadata:
- name: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+ name: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
data:
pipelines.yml: |
- {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
+ {{ tpl .Values.logstash.beats.pipelines_yml_config $ | nindent 6 | b64enc }}
---
{{- range .Values.logstash.pipelines }}
apiVersion: v1
kind: ConfigMap
metadata:
- name: logstash-{{ $.Values.logstash.pipelines_group_name }}-{{ .name }}-config
+ name: logstash-{{ $.Values.logstash.beats.pipelines_group_name }}-{{ .name }}-config
data:
{{ .name }}.config: |
{{ tpl .input $ | nindent 6 }}
diff --git a/tmp/logstash_syslog.yaml b/templates/logstash_syslog.yaml
similarity index 72%
rename from tmp/logstash_syslog.yaml
rename to templates/logstash_syslog.yaml
index 7de6400..ab91b6a 100644
--- a/tmp/logstash_syslog.yaml
+++ b/templates/logstash_syslog.yaml
@@ -1,13 +1,13 @@
apiVersion: logstash.k8s.elastic.co/v1alpha1
kind: Logstash
metadata:
- name: logstash-beats
+ name: logstash-syslog
labels:
app: logstash
spec:
image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion .Values.logstash.imageTag }}
version: {{ .Values.elasticVersion }}
- count: {{ .Values.logstash.replicas }}
+ count: {{ .Values.logstash.count }}
elasticsearchRefs:
- name: {{ .Release.Name }}-elasticsearch
clusterName: {{ .Release.Name }}-elasticsearch
@@ -21,21 +21,19 @@ spec:
requests:
storage: {{ .Values.logstash.diskSpace }}
storageClassName: {{ .Values.logstash.storageClassName }}
-{{- range .Values.logstash.pipelines }}
- services:
- - name: {{ $.Values.logstash.pipelines_group_name }}
+ services:
+{{- range $index := until (.Values.logstash.count |int ) -}}
+{{- printf "\n"}}
+ - name: syslog-{{$index}}
service:
spec:
ports:
- port: 5044
- name: {{ .name }}
- protocol: TCP
- - port: 514
- name: {{ .name }}
+ name: {{ $.Values.logstash.syslog.pipelines_group_name }}
protocol: TCP
selector:
- statefulset.kubernetes.io/pod-name: logstash-beats-ls-0
- {{- end }}
+ statefulset.kubernetes.io/pod-name: logstash-syslog-ls-{{$index}}
+{{- end}}
config:
{{- with .Values.logstash.config }}
{{- toYaml . | nindent 4 }}
@@ -52,7 +50,7 @@ spec:
{{- toYaml . | nindent 10 }}
{{- end }}
volumeMounts:
- {{- range .Values.logstash.pipelines }}
+ {{- range .Values.logstash.syslog.pipelines }}
- name: pipeline-config-{{- .name }}
mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
subPath: {{ .name -}}.config
@@ -71,28 +69,28 @@ spec:
- name: ELASTIC_ELASTICSEARCH_ES_HOSTS
value: 'https://elastic-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
volumes:
- {{- range .Values.logstash.pipelines }}
+ {{- range .Values.logstash.syslog.pipelines }}
- name: pipeline-config-{{- .name }}
configMap:
- name: logstash-{{- $.Values.logstash.pipelines_group_name -}}-{{- .name -}}-config
+ name: logstash-{{- $.Values.logstash.syslog.pipelines_group_name -}}-{{- .name -}}-config
defaultMode: 511
{{- end }}
pipelinesRef:
- secretName: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+ secretName: logstash-{{ .Values.logstash.syslog.pipelines_group_name }}-pipelines-yml
---
apiVersion: v1
kind: Secret
metadata:
- name: logstash-{{ .Values.logstash.pipelines_group_name }}-pipelines-yml
+ name: logstash-{{ .Values.logstash.syslog.pipelines_group_name }}-pipelines-yml
data:
pipelines.yml: |
{{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
---
-{{- range .Values.logstash.pipelines }}
+{{- range .Values.logstash.syslog.pipelines }}
apiVersion: v1
kind: ConfigMap
metadata:
- name: logstash-{{ $.Values.logstash.pipelines_group_name }}-{{ .name }}-config
+ name: logstash-{{ $.Values.logstash.syslog.pipelines_group_name }}-{{ .name }}-config
data:
{{ .name }}.config: |
{{ tpl .input $ | nindent 6 }}
diff --git a/values.yaml b/values.yaml
index 825a6d9..3226fb6 100644
--- a/values.yaml
+++ b/values.yaml
@@ -119,32 +119,59 @@ logstash:
path.config: "/app/elastic/logstash/config/pipelines/*.config"
pipeline.workers: 1
pipeline.batch.size: 125
-
- pipelines_group_name: "pipelines"
+
workers: 1
batch:
size: 125
- pipelines:
- - name: "beats-pipeline"
- input: |-
- input {
- beats {
- port => 5044
+ beats:
+ pipelines_group_name: "pipelines"
+ pipelines:
+ - name: "beats-pipeline"
+ input: |-
+ input {
+ beats {
+ port => 5044
+ }
}
- }
- filter: |-
- filter {}
-
- output: |-
- output {
- elasticsearch {
- hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
- user => "${LOGSTASH_USER}"
- password => "${LOGSTASH_PASSWORD}"
+ filter: |-
+ filter {}
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ }
+ stdout {
+ codec => rubydebug
+ }
}
- stdout {
- codec => rubydebug
+ syslog:
+ pipelines_group_name: "pipelines"
+ pipelines:
+ - name: "syslog-pipeline"
+ input: |-
+ input {
+ syslog {
+ port => 514
+ }
}
- }
-
+ filter: |-
+ filter {}
+
+ output: |-
+ output {
+ elasticsearch {
+ hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ user => "${LOGSTASH_USER}"
+ password => "${LOGSTASH_PASSWORD}"
+ }
+ stdout {
+ codec => rubydebug
+ }
+ }
+
+
--
GitLab
From 0fd0a81aac8326f8917a402c592688d3f5a750d9 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 19:58:37 +0200
Subject: [PATCH 25/61] added logstash
---
templates/logstash_beats.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/logstash_beats.yaml b/templates/logstash_beats.yaml
index 20fda18..3763000 100644
--- a/templates/logstash_beats.yaml
+++ b/templates/logstash_beats.yaml
@@ -84,7 +84,7 @@ metadata:
name: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
data:
pipelines.yml: |
- {{ tpl .Values.logstash.beats.pipelines_yml_config $ | nindent 6 | b64enc }}
+ {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
---
{{- range .Values.logstash.pipelines }}
apiVersion: v1
--
GitLab
From 127b1375e3d777868a51607fe67e8c754c3e3cd3 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 20:01:03 +0200
Subject: [PATCH 26/61] 2 logstashed added
---
values.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/values.yaml b/values.yaml
index 3226fb6..a199878 100644
--- a/values.yaml
+++ b/values.yaml
@@ -124,7 +124,7 @@ logstash:
batch:
size: 125
beats:
- pipelines_group_name: "pipelines"
+ pipelines_group_name: "beats-pipelines"
pipelines:
- name: "beats-pipeline"
input: |-
@@ -149,7 +149,7 @@ logstash:
}
}
syslog:
- pipelines_group_name: "pipelines"
+ pipelines_group_name: "syslog-pipelines"
pipelines:
- name: "syslog-pipeline"
input: |-
--
GitLab
From 9c5bd8e747e89f8a64c3196dd505776167d3f729 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 20:11:59 +0200
Subject: [PATCH 27/61] 2 logstashed added
---
templates/logstash_syslog.yaml | 2 +-
values.yaml | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/templates/logstash_syslog.yaml b/templates/logstash_syslog.yaml
index ab91b6a..7de24c9 100644
--- a/templates/logstash_syslog.yaml
+++ b/templates/logstash_syslog.yaml
@@ -28,7 +28,7 @@ spec:
service:
spec:
ports:
- - port: 5044
+ - port: 514
name: {{ $.Values.logstash.syslog.pipelines_group_name }}
protocol: TCP
selector:
diff --git a/values.yaml b/values.yaml
index a199878..566494a 100644
--- a/values.yaml
+++ b/values.yaml
@@ -124,7 +124,7 @@ logstash:
batch:
size: 125
beats:
- pipelines_group_name: "beats-pipelines"
+ pipelines_group_name: "beats"
pipelines:
- name: "beats-pipeline"
input: |-
@@ -149,7 +149,7 @@ logstash:
}
}
syslog:
- pipelines_group_name: "syslog-pipelines"
+ pipelines_group_name: "syslog"
pipelines:
- name: "syslog-pipeline"
input: |-
--
GitLab
From bc749d371ac11ad9eca40df242b04e20857e13f3 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 20:22:59 +0200
Subject: [PATCH 28/61] 2 logstashed added
---
templates/logstash_beats.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/logstash_beats.yaml b/templates/logstash_beats.yaml
index 3763000..ff65021 100644
--- a/templates/logstash_beats.yaml
+++ b/templates/logstash_beats.yaml
@@ -86,7 +86,7 @@ data:
pipelines.yml: |
{{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
---
-{{- range .Values.logstash.pipelines }}
+{{- range .Values.logstash.beats.pipelines }}
apiVersion: v1
kind: ConfigMap
metadata:
--
GitLab
From 6500de3b9118e6b1d4c24c9540461bc75a64679c Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Wed, 26 Jun 2024 20:27:48 +0200
Subject: [PATCH 29/61] 2 logstashed added without ssl
---
tmp/values.yaml | 173 ------------------------------------------------
1 file changed, 173 deletions(-)
delete mode 100644 tmp/values.yaml
diff --git a/tmp/values.yaml b/tmp/values.yaml
deleted file mode 100644
index 09503e2..0000000
--- a/tmp/values.yaml
+++ /dev/null
@@ -1,173 +0,0 @@
-# version of all elastic applications
-elasticVersion: 8.14.1
-
-# Name of aws environment
-envTag: "dev"
-
-# This suffix will be used to create subdomain of following template:
-# kibana.NAMESPACE_TAG.DOMAIN_SUFFIX
-domainSuffix: "dev"
-
-# This value is used to create unique dns for each deployment. By default it equals to namespace.
-namespaceTag: ""
-
-
-# Spread pods evenly between subnets
-createTopologySpreadConstraints: true
-
-# Force pods to be scheduled evenly between separete AZ, this means if all nodes in AZ will be full pod won't be scheduled.
-# Turning this off will still create pods in seperate AZ, but k8s scheduler won't be able to respect topologySpreadConstrains, will schedule pods in the same AZ.
-forceDeployInSeperateAZ: true
-
-
-# Open port 443 for each component on the Traefik level.
-openStandardHttpsPorts: true
-
-# imagePullSecrets defines the secrets with credentials to docker registry, for example nexus.
-imagePullSecrets: []
-
-# Precise on which nodes should pods be deployed if label is available - if not, it will go on random nodes.
-affinity:
- purpose: simpl
-
-
-
-
-
-
-elasticsearch:
-
- image: docker.elastic.co/elasticsearch/elasticsearch
- # Elasticsearch's image tag, by default it equals to elasticVersion
- ccr_remote_ca_file: ccr-remote-ca.pem
- imageTag: ""
- jvm_options_xms: 1g
- jvm_options_xmx: 1g
- cert:
- duration: 2160h0m0s # 90d
- renewBefore: 360h0m0s # 15d
- #Environment variables to set in kibana pod
- #Usage from cli:
- # --set "elasticsearch.env[0].name=VARIABLE_NAME" --set "elasticsearch.env[0].value=VARIABLE_VALUE"
- env: []
-
-
- # here you can specify several node groups (e.g. master, data groups)
- nodeSets:
- - name: "node"
- # number of elasticsearch nodes in each group
- count: 3
- # persistent volume claim size
- diskSpace: 3Gi
- # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
- storageClassName: csi-cinder-high-speed
- # here you can specify elasticsearch config e.g
- # config:
- # node.roles: ["master"]
- config: {}
- resources:
- requests:
- memory: 4Gi
- limits:
- memory: 4Gi
- cpu: "1"
-
-kibana:
- # set >0 to deploy kibana, 0 otherwise
- count: 2
- image: docker.elastic.co/kibana/kibana
- # Kibana's image tag, by default it equals to elasticVersion
- imageTag: ""
- # name of helm release where elasticsearch is installed. If you install kibana together with elasticsearch, leave it empty.
- elasticsearchReleaseName: ""
- # Kibana server will listen on that subpath
- cert:
- duration: 2160h0m0s # 90d
- renewBefore: 360h0m0s # 15d
- # Additional kibana's config according to this link: https://www.elastic.co/guide/en/kibana/current/settings.html
- config:
- xpack.reporting.roles.enabled: false
- resources:
- requests:
- memory: 1Gi
- limits:
- memory: 1Gi
-
- #Environment variables to set in kibana pod
- #Usage from cli:
- # --set "kibana.env[0].name=VARIABLE_NAME" --set "kibana.env[0].value=VARIABLE_VALUE"
- env: []
-
-
-logstash:
- count: 2
- image: docker.elastic.co/logstash/logstash
- config: {}
- diskSpace: 3Gi
- # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
- storageClassName: csi-cinder-high-speed
- imageTag: ""
- env:
- LS_JAVA_OPTS: "-Xmx256m -Xms256m"
- resources:
- requests:
- memory: 4Gi
- limits:
- memory: 4Gi
- pipelines_yml_config: |-
- - pipeline.id: main
- path.config: "/app/elastic/logstash/config/pipelines/*.config"
- pipeline.workers: 1
- pipeline.batch.size: 125
-
- pipelines_group_name: "pipelines"
- workers: 1
- batch:
- size: 125
- pipelines:
- - name: "beats-pipeline"
- input: |-
- input {
- beats {
- port => 5044
- }
- }
- filter: |-
- filter {}
-
- output: |-
- output {
- elasticsearch {
- hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
- user => "${LOGSTASH_USER}"
- password => "${LOGSTASH_PASSWORD}"
- }
- stdout {
- codec => rubydebug
- }
- }
- - name: "syslog-pipeline"
- input: |-
- input {
- syslog {
- port => 514
- }
- }
- filter: |-
- filter {}
-
- output: |-
- output {
- elasticsearch {
- hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
- user => "${LOGSTASH_USER}"
- password => "${LOGSTASH_PASSWORD}"
- }
- stdout {
- codec => rubydebug
- }
- }
-
-
--
GitLab
From a646bd1f66ff5e17adbaf2dac109c05b7b8b98f3 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:13:15 +0200
Subject: [PATCH 30/61] added filebeat
---
templates/filebeat.yaml | 100 ++++++++++++++++++++++++++++++++++++++++
values.yaml | 39 ++++++++++++++++
2 files changed, 139 insertions(+)
create mode 100644 templates/filebeat.yaml
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
new file mode 100644
index 0000000..e824c84
--- /dev/null
+++ b/templates/filebeat.yaml
@@ -0,0 +1,100 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: filebeat-config
+ labels:
+ app: filebeat
+type: Opaque
+data:
+ filebeat.yml: |
+ {{ printf "%s\n%s" .Values.input (tpl .Values.output $ ) | b64enc | nindent 6 | }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: filebeat-example-script
+data:
+ example.sh: |
+ #!/bin/bash
+ count=0
+ > /usr/share/filebeat/logs/example.log
+ while [ "$count" -lt {{ .Values.totalMessages }} ] || [ {{ .Values.totalMessages }} -lt 0 ]
+ do
+ {{- if gt (.Values.messagesPerMinute | int ) 0 }}
+ sleep {{ divf 60 .Values.messagesPerMinute }}
+ {{- end }}
+ case $((RANDOM % 3)) in
+ (0) status=INFO
+ ;;
+ (1) status=WARN
+ ;;
+ (2) status=ERROR
+ ;;
+ esac
+ case $((RANDOM % 4)) in
+ (0) data=\
+ 'Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.'
+ ;;
+ (1) data=\
+ 'It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using "Content here, content here", making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for alorem ipsum will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose injected humour and the like.'
+ ;;
+ (2) data=\
+ 'Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source.'
+ ;;
+ (3) data=\
+ 'Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" The Extremes of Good and Evil by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32'
+ ;;
+ esac
+ echo "2022.04.01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)) | $count | $status | $data" >> /usr/share/filebeat/logs/example.log
+ count=$((count + 1))
+ done
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: filebeat
+ labels:
+ app: filebeat
+spec:
+ # This is number of nodes that we want to run
+ replicas: 1
+ selector:
+ matchLabels:
+ app: filebeat
+ template:
+ metadata:
+ labels:
+ app: filebeat
+ spec:
+ securityContext:
+ runAsNonRoot: true
+ #fsGroup: 0
+ terminationGracePeriodSeconds: 300
+ containers:
+ - name: filebeat
+ image: {{ .Values.image }}:{{ .Values.imageTag }}
+ imagePullPolicy: Always
+ command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
+ volumeMounts:
+ - mountPath: /usr/share/filebeat/filebeat.yml
+ subPath: filebeat.yml
+ name: config
+ - mountPath: /usr/share/filebeat/logs/example.sh
+ subPath: example.sh
+ name: example-script
+ - mountPath: /usr/share/filebeat/certs
+ name: certs
+ volumes:
+ - name: config
+ secret:
+ secretName: filebeat-config
+ defaultMode: 0555
+ - name: example-script
+ configMap:
+ name: filebeat-example-script
+ defaultMode: 0777
+ - name: certs
+ secret:
+ secretName: {{ .Release.Name }}-filebeat-cert-secret
+{{- end }}
+---
\ No newline at end of file
diff --git a/values.yaml b/values.yaml
index 566494a..dc0d716 100644
--- a/values.yaml
+++ b/values.yaml
@@ -174,4 +174,43 @@ logstash:
}
}
+filebeat:
+ image: docker.elastic.co/beats/filebeat
+ config: {}
+ # name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
+ imageTag: ""
+ # Total number of the sample messages to generate. Provide negative number to generate infinitely
+ totalMessages: 604800
+
+ # Number of messages per minute. Provide negative number to generate messages without time limit.
+ messagesPerMinute: 30
+
+ # Filebeat configuration file - input
+ input: |-
+ filebeat.inputs:
+ - type: log
+ enabled: true
+ paths:
+ - /usr/share/filebeat/logs/example.log
+ multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
+ multiline.negate: true
+ multiline.match: after
+ fields:
+ logtype: ccneb-test-tool
+ filebeat.config.modules:
+ path: ${path.config}/modules.d/*.yml
+ reload.enabled: false
+ # processors:
+ # - add_kubernetes_metadata:
+ # in_cluster: true
+ # logging.level: debug
+ name: "test"
+
+ # # Filebeat configuration file - output
+ output: |-
+ output.logstash:
+ hosts: ["logstash.observability.svc.cluster.local:5044"]
+ ssl.enabled: false
+
+
--
GitLab
From b05dc0421fb65d8a44a9098dda8d9062f1ff8e60 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:16:31 +0200
Subject: [PATCH 31/61] added filebeat
---
templates/filebeat.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index e824c84..1e7e604 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -96,5 +96,4 @@ spec:
- name: certs
secret:
secretName: {{ .Release.Name }}-filebeat-cert-secret
-{{- end }}
---
\ No newline at end of file
--
GitLab
From a590aa47901add597e24c5267e04ba6c5491f694 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:20:20 +0200
Subject: [PATCH 32/61] added filebeat
---
templates/filebeat.yaml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 1e7e604..4c3f12d 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -7,7 +7,7 @@ metadata:
type: Opaque
data:
filebeat.yml: |
- {{ printf "%s\n%s" .Values.input (tpl .Values.output $ ) | b64enc | nindent 6 | }}
+ {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 | }}
---
apiVersion: v1
kind: ConfigMap
@@ -18,10 +18,10 @@ data:
#!/bin/bash
count=0
> /usr/share/filebeat/logs/example.log
- while [ "$count" -lt {{ .Values.totalMessages }} ] || [ {{ .Values.totalMessages }} -lt 0 ]
+ while [ "$count" -lt {{ .Values.filebeat.totalMessages }} ] || [ {{ .Values.filebeat.totalMessages }} -lt 0 ]
do
- {{- if gt (.Values.messagesPerMinute | int ) 0 }}
- sleep {{ divf 60 .Values.messagesPerMinute }}
+ {{- if gt (.Values.filebeat.messagesPerMinute | int ) 0 }}
+ sleep {{ divf 60 .Values.filebeat.messagesPerMinute }}
{{- end }}
case $((RANDOM % 3)) in
(0) status=INFO
@@ -72,7 +72,7 @@ spec:
terminationGracePeriodSeconds: 300
containers:
- name: filebeat
- image: {{ .Values.image }}:{{ .Values.imageTag }}
+ image: {{ .Values.filebeat.image }}:{{ .Values.filebeat.imageTag }}
imagePullPolicy: Always
command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
volumeMounts:
--
GitLab
From 7330177bd224596755241942d3404510a9937a6d Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:27:14 +0200
Subject: [PATCH 33/61] added filebeat
---
values.yaml | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/values.yaml b/values.yaml
index dc0d716..b61fcc1 100644
--- a/values.yaml
+++ b/values.yaml
@@ -186,7 +186,7 @@ filebeat:
messagesPerMinute: 30
# Filebeat configuration file - input
- input: |-
+ input: |
filebeat.inputs:
- type: log
enabled: true
@@ -207,10 +207,9 @@ filebeat:
name: "test"
# # Filebeat configuration file - output
- output: |-
+ output: |
output.logstash:
hosts: ["logstash.observability.svc.cluster.local:5044"]
- ssl.enabled: false
--
GitLab
From 89a8cee668e345bf5ba991804b915298054444f7 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:35:06 +0200
Subject: [PATCH 34/61] added filebeat
---
templates/filebeat.yaml | 2 +-
values.yaml | 5 +----
2 files changed, 2 insertions(+), 5 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 4c3f12d..563451a 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -7,7 +7,7 @@ metadata:
type: Opaque
data:
filebeat.yml: |
- {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 | }}
+ {{ tpl .Values.filebeat.filebeat_config $ | nindent 6 | b64enc }}
---
apiVersion: v1
kind: ConfigMap
diff --git a/values.yaml b/values.yaml
index b61fcc1..867e0db 100644
--- a/values.yaml
+++ b/values.yaml
@@ -186,7 +186,7 @@ filebeat:
messagesPerMinute: 30
# Filebeat configuration file - input
- input: |
+ filebeat_config: |-
filebeat.inputs:
- type: log
enabled: true
@@ -205,9 +205,6 @@ filebeat:
# in_cluster: true
# logging.level: debug
name: "test"
-
- # # Filebeat configuration file - output
- output: |
output.logstash:
hosts: ["logstash.observability.svc.cluster.local:5044"]
--
GitLab
From 3b560fb7af762d71d25f1a4604116219749d496d Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:41:50 +0200
Subject: [PATCH 35/61] added filebeat
---
templates/filebeat.yaml | 2 +-
values.yaml | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 563451a..a5c5417 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -7,7 +7,7 @@ metadata:
type: Opaque
data:
filebeat.yml: |
- {{ tpl .Values.filebeat.filebeat_config $ | nindent 6 | b64enc }}
+ {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
---
apiVersion: v1
kind: ConfigMap
diff --git a/values.yaml b/values.yaml
index 867e0db..7038f69 100644
--- a/values.yaml
+++ b/values.yaml
@@ -186,7 +186,7 @@ filebeat:
messagesPerMinute: 30
# Filebeat configuration file - input
- filebeat_config: |-
+ input: |
filebeat.inputs:
- type: log
enabled: true
@@ -205,6 +205,7 @@ filebeat:
# in_cluster: true
# logging.level: debug
name: "test"
+ output: |
output.logstash:
hosts: ["logstash.observability.svc.cluster.local:5044"]
--
GitLab
From 0da4634294770f9cd6d786a81ccdc96194c18eb5 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:44:16 +0200
Subject: [PATCH 36/61] added filebeat
---
templates/filebeat.yaml | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index a5c5417..0e1025f 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -1,15 +1,4 @@
apiVersion: v1
-kind: Secret
-metadata:
- name: filebeat-config
- labels:
- app: filebeat
-type: Opaque
-data:
- filebeat.yml: |
- {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
----
-apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-example-script
@@ -96,4 +85,14 @@ spec:
- name: certs
secret:
secretName: {{ .Release.Name }}-filebeat-cert-secret
----
\ No newline at end of file
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: filebeat-config
+ labels:
+ app: filebeat
+type: Opaque
+data:
+ filebeat.yml: |
+ {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
\ No newline at end of file
--
GitLab
From 64887bf9f98b1eb3e35c978ea32e5b63d93240ab Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:52:05 +0200
Subject: [PATCH 37/61] added filebeat
---
templates/filebeat.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 0e1025f..bf7b748 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -61,7 +61,7 @@ spec:
terminationGracePeriodSeconds: 300
containers:
- name: filebeat
- image: {{ .Values.filebeat.image }}:{{ .Values.filebeat.imageTag }}
+ image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
imagePullPolicy: Always
command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
volumeMounts:
--
GitLab
From efedcb58ea388a753e798e8215dc685102982a95 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 11:53:52 +0200
Subject: [PATCH 38/61] added filebeat
---
templates/filebeat.yaml | 5 -----
1 file changed, 5 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index bf7b748..d552a3d 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -71,8 +71,6 @@ spec:
- mountPath: /usr/share/filebeat/logs/example.sh
subPath: example.sh
name: example-script
- - mountPath: /usr/share/filebeat/certs
- name: certs
volumes:
- name: config
secret:
@@ -82,9 +80,6 @@ spec:
configMap:
name: filebeat-example-script
defaultMode: 0777
- - name: certs
- secret:
- secretName: {{ .Release.Name }}-filebeat-cert-secret
---
apiVersion: v1
kind: Secret
--
GitLab
From 7b4eeb2a2644da0308ccc490ef84f0f08e7d4ee3 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 12:35:04 +0200
Subject: [PATCH 39/61] added filebeat
---
templates/elasticsearch.yaml | 2 ++
values.yaml | 3 +++
2 files changed, 5 insertions(+)
diff --git a/templates/elasticsearch.yaml b/templates/elasticsearch.yaml
index 83c0d24..a3395c1 100644
--- a/templates/elasticsearch.yaml
+++ b/templates/elasticsearch.yaml
@@ -6,6 +6,8 @@ spec:
version: {{ .Values.elasticVersion }}
image: {{ .Values.elasticsearch.image }}:{{ default .Values.elasticVersion .Values.elasticsearch.imageTag }}
auth:
+ roles:
+ - secretName: logstash-writer-role-secret
fileRealm:
- secretName: logstash-writer-secret
nodeSets:
diff --git a/values.yaml b/values.yaml
index 7038f69..bb4941f 100644
--- a/values.yaml
+++ b/values.yaml
@@ -143,6 +143,9 @@ logstash:
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
user => "${LOGSTASH_USER}"
password => "${LOGSTASH_PASSWORD}"
+ ssl => true
+ ssl_certificate_verification => false
+
}
stdout {
codec => rubydebug
--
GitLab
From 5e275afaca31b733f70f43c3197695c401a0c3af Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 12:36:09 +0200
Subject: [PATCH 40/61] added filebeat
---
values.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/values.yaml b/values.yaml
index bb4941f..c3f23f5 100644
--- a/values.yaml
+++ b/values.yaml
@@ -210,7 +210,7 @@ filebeat:
name: "test"
output: |
output.logstash:
- hosts: ["logstash.observability.svc.cluster.local:5044"]
+ hosts: ["logstash-beats-ls-0.observability.svc:5044"]
--
GitLab
From 470a299e1f50f12bb7f345fb96eb5599c68a8078 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 12:51:42 +0200
Subject: [PATCH 41/61] added filebeat working version
---
values.yaml | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/values.yaml b/values.yaml
index c3f23f5..8b3706e 100644
--- a/values.yaml
+++ b/values.yaml
@@ -140,7 +140,7 @@ logstash:
output {
elasticsearch {
hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}"
user => "${LOGSTASH_USER}"
password => "${LOGSTASH_PASSWORD}"
ssl => true
@@ -168,9 +168,11 @@ logstash:
output {
elasticsearch {
hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+ index => "%{[@metadata][beat]}-%{[@metadata][version]}"
user => "${LOGSTASH_USER}"
password => "${LOGSTASH_PASSWORD}"
+ ssl => true
+ ssl_certificate_verification => false
}
stdout {
codec => rubydebug
@@ -210,7 +212,7 @@ filebeat:
name: "test"
output: |
output.logstash:
- hosts: ["logstash-beats-ls-0.observability.svc:5044"]
+ hosts: ["logstash-beats-ls-beats-0.observability.svc:5044"]
--
GitLab
From a3ad7496cab9009decc6e2c0048b33848202dda7 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 12:58:46 +0200
Subject: [PATCH 42/61] added filebeat eck
---
templates/filebeat.yaml | 124 +++++++++++++++++++++++-----------------
1 file changed, 70 insertions(+), 54 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index d552a3d..8c8c9a0 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -1,3 +1,72 @@
+apiVersion: beat.k8s.elastic.co/v1
+kind: Beat
+metadata:
+ name: filebeat
+ labels:
+ app: filebeat
+spec:
+ image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
+ version: {{ .Values.elasticVersion }}
+ elasticsearchRefs:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
+ config:
+ filebeat.yml: |
+ filebeat.inputs:
+ - type: log
+ enabled: true
+ paths:
+ - /usr/share/filebeat/logs/example.log
+ multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
+ multiline.negate: true
+ multiline.match: after
+ fields:
+ logtype: ccneb-test-tool
+ filebeat.config.modules:
+ path: ${path.config}/modules.d/*.yml
+ reload.enabled: false
+ name: "test"
+ output.logstash:
+ hosts: ["logstash-beats-ls-beats-0.observability.svc:5044"]
+ daemonSet:
+ podTemplate:
+ metadata:
+ labels:
+ stack-namespace: {{ .Release.Namespace }}
+ spec:
+ containers:
+ - name: filebeat
+ command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
+ volumeMounts:
+ - mountPath: /usr/share/filebeat/filebeat.yml
+ subPath: filebeat.yml
+ name: config
+ - mountPath: /usr/share/filebeat/logs/example.sh
+ subPath: example.sh
+ name: example-script
+ - mountPath: /usr/share/filebeat/certs
+ name: certs
+ volumes:
+ - name: config
+ secret:
+ secretName: filebeat-config
+ defaultMode: 0555
+ - name: example-script
+ configMap:
+ name: filebeat-example-script
+ defaultMode: 0777
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: filebeat-config
+ labels:
+ app: filebeat
+type: Opaque
+data:
+ filebeat.yml: |
+ {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
+---
apiVersion: v1
kind: ConfigMap
metadata:
@@ -37,57 +106,4 @@ data:
echo "2022.04.01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)) | $count | $status | $data" >> /usr/share/filebeat/logs/example.log
count=$((count + 1))
done
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: filebeat
- labels:
- app: filebeat
-spec:
- # This is number of nodes that we want to run
- replicas: 1
- selector:
- matchLabels:
- app: filebeat
- template:
- metadata:
- labels:
- app: filebeat
- spec:
- securityContext:
- runAsNonRoot: true
- #fsGroup: 0
- terminationGracePeriodSeconds: 300
- containers:
- - name: filebeat
- image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
- imagePullPolicy: Always
- command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
- volumeMounts:
- - mountPath: /usr/share/filebeat/filebeat.yml
- subPath: filebeat.yml
- name: config
- - mountPath: /usr/share/filebeat/logs/example.sh
- subPath: example.sh
- name: example-script
- volumes:
- - name: config
- secret:
- secretName: filebeat-config
- defaultMode: 0555
- - name: example-script
- configMap:
- name: filebeat-example-script
- defaultMode: 0777
----
-apiVersion: v1
-kind: Secret
-metadata:
- name: filebeat-config
- labels:
- app: filebeat
-type: Opaque
-data:
- filebeat.yml: |
- {{ printf "%s\n%s" .Values.filebeat.input (tpl .Values.filebeat.output $ ) | b64enc | nindent 6 }}
\ No newline at end of file
+---
\ No newline at end of file
--
GitLab
From f969595cbb08b5da3e91adfb6aa3722927fe3259 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:01:27 +0200
Subject: [PATCH 43/61] added filebeat eck
---
templates/filebeat.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 8c8c9a0..e843e78 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -1,4 +1,4 @@
-apiVersion: beat.k8s.elastic.co/v1
+apiVersion: beats.beat.k8s.elastic.co
kind: Beat
metadata:
name: filebeat
--
GitLab
From 2a29d6db7e10f60eae0baf3d3f918f580bde930b Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:04:35 +0200
Subject: [PATCH 44/61] added filebeat eck
---
templates/filebeat.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index e843e78..d13d8e2 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -1,4 +1,4 @@
-apiVersion: beats.beat.k8s.elastic.co
+apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: filebeat
--
GitLab
From e45b76a2bd7800e9feb2aa2f7bdb18204a7e60d3 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:06:03 +0200
Subject: [PATCH 45/61] added filebeat eck
---
templates/filebeat.yaml | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index d13d8e2..f97ac88 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -7,9 +7,10 @@ metadata:
spec:
image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
version: {{ .Values.elasticVersion }}
- elasticsearchRefs:
- - name: {{ .Release.Name }}-elasticsearch
- clusterName: {{ .Release.Name }}-elasticsearch
+ type: filebeat
+ #elasticsearchRefs:
+ #- name: {{ .Release.Name }}-elasticsearch
+ # clusterName: {{ .Release.Name }}-elasticsearch
config:
filebeat.yml: |
filebeat.inputs:
--
GitLab
From d2fa1308ef043b597355b548982569dcf404550b Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:09:46 +0200
Subject: [PATCH 46/61] added filebeat eck
---
templates/filebeat.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index f97ac88..5df7395 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -8,9 +8,9 @@ spec:
image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
version: {{ .Values.elasticVersion }}
type: filebeat
- #elasticsearchRefs:
- #- name: {{ .Release.Name }}-elasticsearch
- # clusterName: {{ .Release.Name }}-elasticsearch
+ elasticsearchRef:
+ - name: {{ .Release.Name }}-elasticsearch
+ clusterName: {{ .Release.Name }}-elasticsearch
config:
filebeat.yml: |
filebeat.inputs:
--
GitLab
From 040e4d81f660be23a5fc92a19dcfe6c62b745a3c Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:11:55 +0200
Subject: [PATCH 47/61] added filebeat eck
---
templates/filebeat.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 5df7395..b3ff869 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -9,7 +9,7 @@ spec:
version: {{ .Values.elasticVersion }}
type: filebeat
elasticsearchRef:
- - name: {{ .Release.Name }}-elasticsearch
+ name: {{ .Release.Name }}-elasticsearch
clusterName: {{ .Release.Name }}-elasticsearch
config:
filebeat.yml: |
--
GitLab
From 712d11eb88473c61b8ee52739e5a48389f56f03c Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:16:05 +0200
Subject: [PATCH 48/61] added filebeat eck
---
templates/filebeat.yaml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index b3ff869..dc81ac8 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -10,7 +10,7 @@ spec:
type: filebeat
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
- clusterName: {{ .Release.Name }}-elasticsearch
+ #clusterName: {{ .Release.Name }}-elasticsearch
config:
filebeat.yml: |
filebeat.inputs:
@@ -45,8 +45,6 @@ spec:
- mountPath: /usr/share/filebeat/logs/example.sh
subPath: example.sh
name: example-script
- - mountPath: /usr/share/filebeat/certs
- name: certs
volumes:
- name: config
secret:
--
GitLab
From 713a54d98713ccf001cec547e065c697fecec017 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:20:15 +0200
Subject: [PATCH 49/61] added filebeat eck
---
templates/filebeat.yaml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index dc81ac8..c1b7509 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -35,6 +35,9 @@ spec:
labels:
stack-namespace: {{ .Release.Namespace }}
spec:
+ securityContext:
+ runAsUser: 0 # Running as root user
+ fsGroup: 1000 # Filebeat user ID
containers:
- name: filebeat
command: ['sh', '-c',' exec /usr/share/filebeat/logs/example.sh & exec /usr/share/filebeat/filebeat -e -c /usr/share/filebeat/filebeat.yml']
--
GitLab
From dad33881d59df5f42d241476c541d6cf8e1f9360 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:26:44 +0200
Subject: [PATCH 50/61] added filebeat eck
---
templates/filebeat.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index c1b7509..60246be 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -22,7 +22,7 @@ spec:
multiline.negate: true
multiline.match: after
fields:
- logtype: ccneb-test-tool
+ logtype: simpl-test-tool
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
--
GitLab
From 4010b37aa4c1891b3eb9cc5530fe82dcaf55a7a2 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:27:58 +0200
Subject: [PATCH 51/61] added filebeat eck
---
templates/filebeat.yaml | 1 +
values.yaml | 1 +
2 files changed, 2 insertions(+)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 60246be..8da1eb2 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -8,6 +8,7 @@ spec:
image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
version: {{ .Values.elasticVersion }}
type: filebeat
+ count: {{ .Values.filebeat.count }}
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
#clusterName: {{ .Release.Name }}-elasticsearch
diff --git a/values.yaml b/values.yaml
index 8b3706e..5a04a67 100644
--- a/values.yaml
+++ b/values.yaml
@@ -182,6 +182,7 @@ logstash:
filebeat:
image: docker.elastic.co/beats/filebeat
config: {}
+ count: 1
# name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
imageTag: ""
# Total number of the sample messages to generate. Provide negative number to generate infinitely
--
GitLab
From 6bbbcda9afdb5f84ac1cd789494235fd2870d902 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 13:56:08 +0200
Subject: [PATCH 52/61] added filebeat eck
---
values.yaml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/values.yaml b/values.yaml
index 5a04a67..046f7f8 100644
--- a/values.yaml
+++ b/values.yaml
@@ -140,11 +140,13 @@ logstash:
output {
elasticsearch {
hosts => [ "${ELASTIC_ELASTICSEARCH_ES_HOSTS}" ]
- index => "%{[@metadata][beat]}-%{[@metadata][version]}"
user => "${LOGSTASH_USER}"
password => "${LOGSTASH_PASSWORD}"
ssl => true
ssl_certificate_verification => false
+ data_stream => "true"
+ data_stream_type => "logs"
+ data_stream_dataset => "filebeat"
}
stdout {
--
GitLab
From 78b35541f25870812b6564ce4167b4ad498edf5f Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 14:06:10 +0200
Subject: [PATCH 53/61] added filebeat eck
---
templates/filebeat.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 8da1eb2..60246be 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -8,7 +8,6 @@ spec:
image: {{ .Values.filebeat.image }}:{{ default .Values.elasticVersion .Values.filebeat.imageTag }}
version: {{ .Values.elasticVersion }}
type: filebeat
- count: {{ .Values.filebeat.count }}
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
#clusterName: {{ .Release.Name }}-elasticsearch
--
GitLab
From a261d118371b71b3b74d2c87740ac58bf7f4b1d3 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 14:08:12 +0200
Subject: [PATCH 54/61] added filebeat eck
---
values.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/values.yaml b/values.yaml
index 046f7f8..1282d7b 100644
--- a/values.yaml
+++ b/values.yaml
@@ -184,7 +184,6 @@ logstash:
filebeat:
image: docker.elastic.co/beats/filebeat
config: {}
- count: 1
# name of StorageClass that will be used to create VolumeClaims. (StorageClass must exist)
imageTag: ""
# Total number of the sample messages to generate. Provide negative number to generate infinitely
--
GitLab
From b049313882d7c518a9fbc582ea1396eb1ca8289d Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 14:54:46 +0200
Subject: [PATCH 55/61] added filebeat eck
---
templates/filebeat.yaml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 60246be..e534b5d 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -17,7 +17,7 @@ spec:
- type: log
enabled: true
paths:
- - /usr/share/filebeat/logs/example.log
+ - /usr/share/filebeat/logs/example2.log
multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
multiline.negate: true
multiline.match: after
@@ -77,7 +77,7 @@ data:
example.sh: |
#!/bin/bash
count=0
- > /usr/share/filebeat/logs/example.log
+ > /usr/share/filebeat/logs/example2.log
while [ "$count" -lt {{ .Values.filebeat.totalMessages }} ] || [ {{ .Values.filebeat.totalMessages }} -lt 0 ]
do
{{- if gt (.Values.filebeat.messagesPerMinute | int ) 0 }}
@@ -105,7 +105,7 @@ data:
'Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" The Extremes of Good and Evil by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32'
;;
esac
- echo "2022.04.01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)) | $count | $status | $data" >> /usr/share/filebeat/logs/example.log
+ echo "2022.04.01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)) | $count | $status | $data" >> /usr/share/filebeat/logs/example2.log
count=$((count + 1))
done
---
\ No newline at end of file
--
GitLab
From 5630173281521faa8fded845b7f76f3b41ad3cfa Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Thu, 27 Jun 2024 15:09:53 +0200
Subject: [PATCH 56/61] added filebeat eck
---
values.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/values.yaml b/values.yaml
index 1282d7b..97641d0 100644
--- a/values.yaml
+++ b/values.yaml
@@ -198,12 +198,12 @@ filebeat:
- type: log
enabled: true
paths:
- - /usr/share/filebeat/logs/example.log
+ - /usr/share/filebeat/logs/example2.log
multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
multiline.negate: true
multiline.match: after
fields:
- logtype: ccneb-test-tool
+ logtype: simpl-test-tool
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
--
GitLab
From bb3c481d519b9aaa51a47312c45b4c96f8bac055 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Fri, 28 Jun 2024 09:44:18 +0200
Subject: [PATCH 57/61] added filebeat eck
---
templates/filebeat.yaml | 21 ++-------------------
values.yaml | 8 ++++++--
2 files changed, 8 insertions(+), 21 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index e534b5d..65168a2 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -12,23 +12,6 @@ spec:
name: {{ .Release.Name }}-elasticsearch
#clusterName: {{ .Release.Name }}-elasticsearch
config:
- filebeat.yml: |
- filebeat.inputs:
- - type: log
- enabled: true
- paths:
- - /usr/share/filebeat/logs/example2.log
- multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
- multiline.negate: true
- multiline.match: after
- fields:
- logtype: simpl-test-tool
- filebeat.config.modules:
- path: ${path.config}/modules.d/*.yml
- reload.enabled: false
- name: "test"
- output.logstash:
- hosts: ["logstash-beats-ls-beats-0.observability.svc:5044"]
daemonSet:
podTemplate:
metadata:
@@ -77,7 +60,7 @@ data:
example.sh: |
#!/bin/bash
count=0
- > /usr/share/filebeat/logs/example2.log
+ > /usr/share/filebeat/logs/example.log
while [ "$count" -lt {{ .Values.filebeat.totalMessages }} ] || [ {{ .Values.filebeat.totalMessages }} -lt 0 ]
do
{{- if gt (.Values.filebeat.messagesPerMinute | int ) 0 }}
@@ -105,7 +88,7 @@ data:
'Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" The Extremes of Good and Evil by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32'
;;
esac
- echo "2022.04.01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)) | $count | $status | $data" >> /usr/share/filebeat/logs/example2.log
+ echo "2022-04-01 $((RANDOM % 3 + 9)):$((RANDOM % 60)):$((RANDOM % 60)).000 | $count | $status | $data" >> /usr/share/filebeat/logs/example.log
count=$((count + 1))
done
---
\ No newline at end of file
diff --git a/values.yaml b/values.yaml
index 97641d0..5573666 100644
--- a/values.yaml
+++ b/values.yaml
@@ -164,7 +164,11 @@ logstash:
}
}
filter: |-
- filter {}
+ filter {
+ grok {
+ match => { "message" => "%{TIMESTAMP_ISO8601:logdatetime} \| %{DATA:thread} \| %{LOGLEVEL:loglevel} \| %{GREEDYDATA:details}" }
+ }
+ }
output: |-
output {
@@ -198,7 +202,7 @@ filebeat:
- type: log
enabled: true
paths:
- - /usr/share/filebeat/logs/example2.log
+ - /usr/share/filebeat/logs/example.log
multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
multiline.negate: true
multiline.match: after
--
GitLab
From f9bee13325c97f861f6805cd28646139c79556d8 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Fri, 28 Jun 2024 10:02:53 +0200
Subject: [PATCH 58/61] added filebeat eck
---
templates/filebeat.yaml | 1 -
1 file changed, 1 deletion(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index 65168a2..b424530 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -11,7 +11,6 @@ spec:
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
#clusterName: {{ .Release.Name }}-elasticsearch
- config:
daemonSet:
podTemplate:
metadata:
--
GitLab
From 9624288d876bdd50fef85e9d9ffa61227a2cdc35 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Fri, 28 Jun 2024 10:09:55 +0200
Subject: [PATCH 59/61] added filebeat eck
---
templates/filebeat.yaml | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index b424530..a691193 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -11,6 +11,25 @@ spec:
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
#clusterName: {{ .Release.Name }}-elasticsearch
+ config:
+ filebeat.yml: |
+ filebeat.inputs:
+ - type: log
+ enabled: true
+ paths:
+ - /usr/share/filebeat/logs/example.log
+ multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
+ multiline.negate: true
+ multiline.match: after
+ fields:
+ logtype: simpl-test-tool
+ filebeat.config.modules:
+ path: ${path.config}/modules.d/*.yml
+ reload.enabled: false
+ name: "test"
+ output.logstash:
+ hosts: ["logstash-beats-ls-beats-0.observability.svc:5044"]
+
daemonSet:
podTemplate:
metadata:
--
GitLab
From eb405a83816fc49486f4968db4bda1a55cb12de7 Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Fri, 28 Jun 2024 10:25:34 +0200
Subject: [PATCH 60/61] added filebeat eck
---
templates/filebeat.yaml | 19 -------------------
values.yaml | 2 +-
2 files changed, 1 insertion(+), 20 deletions(-)
diff --git a/templates/filebeat.yaml b/templates/filebeat.yaml
index a691193..b424530 100644
--- a/templates/filebeat.yaml
+++ b/templates/filebeat.yaml
@@ -11,25 +11,6 @@ spec:
elasticsearchRef:
name: {{ .Release.Name }}-elasticsearch
#clusterName: {{ .Release.Name }}-elasticsearch
- config:
- filebeat.yml: |
- filebeat.inputs:
- - type: log
- enabled: true
- paths:
- - /usr/share/filebeat/logs/example.log
- multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
- multiline.negate: true
- multiline.match: after
- fields:
- logtype: simpl-test-tool
- filebeat.config.modules:
- path: ${path.config}/modules.d/*.yml
- reload.enabled: false
- name: "test"
- output.logstash:
- hosts: ["logstash-beats-ls-beats-0.observability.svc:5044"]
-
daemonSet:
podTemplate:
metadata:
diff --git a/values.yaml b/values.yaml
index 5573666..4a7e93b 100644
--- a/values.yaml
+++ b/values.yaml
@@ -203,7 +203,7 @@ filebeat:
enabled: true
paths:
- /usr/share/filebeat/logs/example.log
- multiline.pattern: (^([0-9]{4}((\.[0-9]{2}){2})))
+ multiline.pattern: (^([0-9]{4}((\-[0-9]{2}){2})))
multiline.negate: true
multiline.match: after
fields:
--
GitLab
From 003930e47f1c008dc95111018a2a544175d9952a Mon Sep 17 00:00:00 2001
From: "Natalia Szakiel (ext)" <natalia.szakiel.external@eviden.com>
Date: Fri, 28 Jun 2024 10:36:01 +0200
Subject: [PATCH 61/61] added filebeat eck
---
values.yaml | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/values.yaml b/values.yaml
index 4a7e93b..5513fb2 100644
--- a/values.yaml
+++ b/values.yaml
@@ -134,7 +134,11 @@ logstash:
}
}
filter: |-
- filter {}
+ filter {
+ grok {
+ match => { "message" => "%{TIMESTAMP_ISO8601:logdatetime} \| %{DATA:thread} \| %{LOGLEVEL:loglevel} \| %{GREEDYDATA:details}" }
+ }
+ }
output: |-
output {
@@ -165,9 +169,6 @@ logstash:
}
filter: |-
filter {
- grok {
- match => { "message" => "%{TIMESTAMP_ISO8601:logdatetime} \| %{DATA:thread} \| %{LOGLEVEL:loglevel} \| %{GREEDYDATA:details}" }
- }
}
output: |-
--
GitLab