logstash_beats.yaml

{{- if eq .Values.namespaceTag .Values.mainNamespace }}
apiVersion: logstash.k8s.elastic.co/v1alpha1
kind: Logstash
metadata:
  name: logstash-beats
  labels:
    app: logstash
spec:
  image: {{ .Values.logstash.image }}:{{ default .Values.elasticVersion  .Values.logstash.imageTag }}
  version: {{ .Values.elasticVersion }}
  count: {{ .Values.logstash.count_beats }}
  elasticsearchRefs:
  - name: {{ .Release.Name }}-elasticsearch
    clusterName: {{ .Release.Name }}-elasticsearch
  monitoring:
    logs:
      elasticsearchRefs:
        - name: {{ .Release.Name }}-elasticsearch    
  volumeClaimTemplates:
    - metadata:
        name: logstash-data
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: {{ .Values.logstash.diskSpace }}
        storageClassName: {{ .Values.logstash.storageClassName }}
  services: 
{{- range $index :=  until (.Values.logstash.count_beats |int ) -}}
{{- printf "\n"}}  
    - name: beats-{{$index}}
      service:
        spec:
          ports:
          - port: 5044
            name: {{ $.Values.logstash.beats.pipelines_group_name }}
            protocol: TCP
          selector:
            statefulset.kubernetes.io/pod-name: logstash-beats-ls-{{$index}}
{{- end}}
  config:
    xpack.monitoring.enabled: true
    xpack.monitoring.elasticsearch.hosts: ["${ELASTIC_ELASTICSEARCH_ES_HOSTS}"]
    xpack.monitoring.elasticsearch.username: "${MONITORING_USER}"
    xpack.monitoring.elasticsearch.password: "${MONITORING_PASSWORD}"
    xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/ca.crt
  podTemplate:
    metadata:
      labels:
        stack-namespace: {{ .Release.Namespace }} 
    spec:
      securityContext:
        runAsUser: 1000
        fsGroup: 1000    
      initContainers:
      - name: git-clone
        image: alpine/git
        args:
          - clone
          - --single-branch
          - --branch
          - {{ .Values.kibana.dashboardsBranch }}            
          - https://code.europa.eu/simpl/simpl-open/development/monitoring/eck-monitoring.git
          - /mnt/ilm/
        volumeMounts:
          - name: repo
            mountPath: /mnt/ilm/
      - name: load-objects
        command:  ["/bin/sh", "-c", "cd /mnt/ilm/charts/kibana/scripts; chmod +x ./load_objects.sh; ./load_objects.sh 2>&1 "]
        volumeMounts:
          - name: repo
            mountPath: /mnt/ilm/
          - name:  logstash-business-ilm-vol
            mountPath: /usr/share/logstash/ilm/logstash-business-ilm.json
            subPath: logstash-business-ilm.json
          - name:  logstash-technical-ilm-vol
            mountPath: /usr/share/logstash/ilm/logstash-technical-ilm.json
            subPath: logstash-technical-ilm.json
        env:
          - name: RELEASE_NAME
            value: {{ .Release.Name }}
          - name: ELASTIC_PASSWORD
            valueFrom:
              secretKeyRef:
                name: {{ .Release.Name }}-elasticsearch-es-elastic-user 
                key: elastic
      containers:
      - name: logstash
        {{- with .Values.logstash.resources }}
        resources:
          {{- toYaml . | nindent 10 }}
        {{- end }}
        volumeMounts:
        {{- range .Values.logstash.beats.pipelines }}
        - name: pipeline-config-{{- .name }}
          mountPath: /app/elastic/logstash/config/pipelines/{{- .name -}}.config
          subPath: {{ .name -}}.config
          readOnly: false
        {{- end }}
        - name: es-certs
          mountPath: /usr/share/logstash/config/certs
        - name: certs-logstash
          mountPath: /usr/share/logstash/certs-logstash
        - name: repo
          mountPath: /mnt/ilm/
        - name:  logstash-business-ilm-vol
          mountPath: /usr/share/logstash/ilm/logstash-business-ilm.json
          subPath: logstash-business-ilm.json
        - name:  logstash-technical-ilm-vol
          mountPath: /usr/share/logstash/ilm/logstash-technical-ilm.json
          subPath: logstash-technical-ilm.json
        - name:  logstash-test
          mountPath: /usr/share/logstash/test/logstash-replace.sh
          subPath: logstash-replace.sh
        env:
          - name: LS_JAVA_OPTS
            value: {{ .Values.logstash.env.ls_java_opts }}
          - name: LOGSTASH_USER
            valueFrom:
              secretKeyRef:
                name: logstash-writer-secret
                key: username
          - name: LOGSTASH_PASSWORD
            valueFrom:
              secretKeyRef:
                name: logstash-writer-secret
                key: password
          - name: MONITORING_USER
            valueFrom:
              secretKeyRef:
                name: user-monitoring-secret
                key: username
          - name: MONITORING_PASSWORD
            valueFrom:
              secretKeyRef:
                name: user-monitoring-secret
                key: password  
          - name: ELASTIC_ELASTICSEARCH_ES_HOSTS
            value: 'https://{{ .Release.Name }}-elasticsearch-es-http.{{ .Release.Namespace }}.svc:9200'
          - name: ELASTICSEARCH_SSL_CERTIFICATE_VERIFICATION
            value: "true"
          - name: ELASTICSEARCH_SSL_CA_PATH
            value: "/usr/share/logstash/config/certs/ca.crt"
      volumes:
      {{- range .Values.logstash.beats.pipelines }}
      - name: pipeline-config-{{- .name }}
        configMap:
          name: logstash-{{- $.Values.logstash.beats.pipelines_group_name -}}-{{- .name -}}-config
          defaultMode: 511
      {{- end }}
      - name: es-certs
        secret:
          secretName: {{ .Release.Name }}-elasticsearch-http-cert-secret-internal
      - name: certs-logstash
        secret:
          secretName: logstash-secret-{{ .Values.logstash.beats.pipelines_group_name }}
      - name: repo
        emptyDir: {}
      - name: logstash-business-ilm-vol
        configMap:
          name: logstash-business-ilm-configmap
          defaultMode: 511
      - name: logstash-technical-ilm-vol
        configMap:
          name: logstash-technical-ilm-configmap
          defaultMode: 511
      - name: logstash-test
        configMap:
          name: logstash-test
          defaultMode: 511
  pipelinesRef:
    secretName: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
---
apiVersion: v1
kind: Secret
metadata:
  name: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-pipelines-yml
data:
  pipelines.yml: |
   {{ tpl .Values.logstash.pipelines_yml_config $ | nindent 6 | b64enc }}
---
{{- range .Values.logstash.beats.pipelines }}
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-{{ $.Values.logstash.beats.pipelines_group_name }}-{{ .name }}-config
data:
  {{ .name }}.config: |
    {{- tpl .input $ | nindent 4 }}
    {{- tpl .filter $ | nindent 4 }}
    {{- tpl .output $ | nindent 4 }}
---
{{- end }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: logstash-api-{{ .Values.logstash.beats.pipelines_group_name }}
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/proxy-body-size: 50m
    external-dns.alpha.kubernetes.io/hostname: "{{ template "logstash.dns" . }},{{- include "logstash.dns.array" . | trim}}"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
      - {{ template "logstash.dns" . }}
    secretName: logstash-secret-{{ .Values.logstash.beats.pipelines_group_name }}
  rules:
    - host: {{ template "logstash.dns" . }}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api
                port:
                  number: 9600
---
{{ $concatUrl :=  (include "logstash.dns"  .) }}
{{ $prefix := (default "l" .Values.logstash.urlPrefix) }}
{{- range $index_i :=  until (.Values.logstash.count_beats |int ) -}}
{{- printf "\n"}}
apiVersion: v1
kind: ConfigMap
metadata:
  name: tcp-services-{{ $.Values.logstash.beats.pipelines_group_name }}-{{$index_i}}
data:
  5044: "observability/logstash-{{ $.Values.logstash.beats.pipelines_group_name }}-ls-{{$index_i}}:5044"
{{- end }}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: logstash-{{ .Values.logstash.beats.pipelines_group_name }}
spec:
  duration: {{ .Values.logstash.cert.duration }}
  renewBefore: {{ .Values.logstash.cert.renewBefore }}
  commonName: {{ template "logstash.dns" . }}
  secretName: logstash-secret-{{ .Values.logstash.beats.pipelines_group_name }}
  dnsNames:
    - "{{ template "logstash.dns" . }}"
{{- range $index_i :=  until (.Values.logstash.count_beats |int ) }}
    - "{{$prefix}}{{$index_i}}.{{$concatUrl}}"
{{- end }}    
    - "logstash.{{ .Release.Namespace }}" 
    - "logstash.{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Values.namespaceTag }}.{{ .Values.domainSuffix }}"
    - "logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Release.Namespace }}"
    - "logstash-{{ .Values.logstash.beats.pipelines_group_name }}-ls-api.{{ .Release.Namespace }}.svc.cluster.local"
  issuerRef:
    name: elk-clusterissuer
    kind: ClusterIssuer
  privateKey:
    encoding: "PKCS8" 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-business-ilm-configmap
data: 
  logstash-business-ilm.json: |
    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_age": "{{ .Values.logstash.ilm.business.hot.max_age }}",
                "max_primary_shard_size": "{{ .Values.logstash.ilm.business.hot.max_primary_shard_size }}"
              },
              "set_priority": {
                "priority": 100
              }
            },
            "min_age": "0ms"
          },
          "delete": {
            "min_age":  "{{ .Values.logstash.ilm.business.delete.min_age }}",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-technical-ilm-configmap
data: 
  logstash-technical-ilm.json: |
    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_age": "{{ .Values.logstash.ilm.technical.hot.max_age }}",
                "max_primary_shard_size": "{{ .Values.logstash.ilm.technical.hot.max_primary_shard_size }}"
              },
              "set_priority": {
                "priority": 100
              }
            },
            "min_age": "0ms"
          },
          "delete": {
            "min_age":  "{{ .Values.logstash.ilm.technical.delete.min_age }}",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-test
data: 
  logstash-replace.sh: |
    #!/bin/bash
    sed -i 's/@{/${/g' /app/elastic/logstash/config/pipelines/beats-pipeline.config
---
{{- end }}