--- kind: ConfigMap apiVersion: v1 metadata: name: logstash-config namespace: pro data: logstash-config-named-k8s: | input { kafka { bootstrap_servers => ["kafka-0.kafka-svc.pro.svc.cluster.local:9093,kafka-1.kafka-svc.pro.svc.cluster.local:9093,kafka-2.kafka-svc.pro.svc.cluster.local:9093"] group_id => "es-test" topics => ["offline-process-topic"] codec => json } } filter { mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] } grok{ match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"} } date { match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ] } } output { # stdout { codec => rubydebug } if [log_topic] == "offline-process-charge" { elasticsearch { hosts => ["elasticsearch:9200"] index => "logstash-offline-process-charge-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-charge-callback" { elasticsearch { hosts => ["elasticsearch:9200"] index => "logstash-offline-process-charge-callback-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-promotion" { elasticsearch { hosts => ["elasticsearch:9200"] index => "logstash-offline-process-promotion-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-newmedia" { elasticsearch { hosts => ["elasticsearch:9200"] index => "logstash-offline-process-newmedia-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-schedule" { elasticsearch { hosts => ["elasticsearch:9200"] index => "logstash-offline-process-schedule-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-statistics" { elasticsearch { hosts => ["elasticsearch:9200"] index => "logstash-offline-process-statistics-%{+YYYY.MM}" template_overwrite => true } } } logstash_yml: | http.host: "0.0.0.0" --- kind: Deployment apiVersion: apps/v1 metadata: name: logstash-k8s-named namespace: pro labels: app: logstash-k8s-named spec: replicas: 1 selector: matchLabels: app: logstash-k8s-named template: metadata: labels: app: logstash-k8s-named spec: containers: - name: logstash-k8s-named image: logstash:7.6.2 command: ["/bin/sh","-c"] args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"] ports: - containerPort: 5000 name: client1 - containerPort: 9600 name: api volumeMounts: - name: vm-config mountPath: /usr/share/logstash/config volumes: - name: vm-config configMap: name: logstash-config items: - key: logstash-config-named-k8s path: logstash.conf --- kind: Service apiVersion: v1 metadata: labels: app: logstash name: logstash namespace: pro spec: ports: - name: http port: 5000 protocol: TCP targetPort: 5000 selector: app: logstash