--- kind: ConfigMap apiVersion: v1 metadata: name: logstash-configmap namespace: pro data: logstash.yml: | http.host: "0.0.0.0" logstash.conf: | input { kafka { bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"] group_id => "es-test" auto_offset_reset => "latest" consumer_threads => 3 max_partition_fetch_bytes => 4194304 receive_buffer_bytes => 131072 send_buffer_bytes => 524288 topics => ["offline-process-topic"] codec => json } } filter { mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score", "@version", "_id", "[agent][type]", "[agent][id]", "[agent][name]", "[agent][version]", "[agent][ephemeral_id]", "[ecs][version]" ]} grok{ match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"} overwrite => ["message"] } date { match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ] } } output { # stdout { codec => rubydebug } if [log_topic] == "offline-process-charge" { elasticsearch { hosts => ["10.26.27.137:9200"] index => "offline-process-charge-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-charge-callback" { elasticsearch { hosts => ["10.26.27.137:9200"] index => "offline-process-charge-callback-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-promotion" { elasticsearch { hosts => ["10.26.27.137:9200"] index => "offline-process-promotion-%{+YYYY.MM}" template_overwrite => true } } if [log_topic] == "offline-process-schedule" { elasticsearch { hosts => ["10.26.27.137:9200"] index => "offline-process-schedule-%{+YYYY.MM}" template_overwrite => true } } } --- kind: Deployment apiVersion: apps/v1 metadata: name: logstash-k8s-named namespace: pro labels: app: logstash-k8s-named spec: replicas: 3 selector: matchLabels: app: logstash-k8s-named template: metadata: labels: app: logstash-k8s-named spec: containers: - name: logstash-k8s-named imagePullPolicy: IfNotPresent image: hub.evbj.easou.com/pro/logstash:7.13.3 command: ["/bin/sh","-c"] args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config"] ports: - containerPort: 5000 name: client1 - containerPort: 9600 name: api volumeMounts: - name: config-volume mountPath: /usr/share/logstash/config - name: yml-volume mountPath: /usr/share/logstash/config env: - name: LS_HEAP_SIZE value: 4096m - name: LS_HTTP_HOST value: "0.0.0.0" volumes: - name: yml-volume configMap: name: logstash-configmap items: - key: logstash.yml path: logstash.yml - name: config-volume configMap: name: logstash-configmap items: - key: logstash.conf path: logstash.conf --- kind: Service apiVersion: v1 metadata: labels: app: logstash name: logstash namespace: pro spec: ports: - name: http port: 5000 protocol: TCP targetPort: 5000 selector: app: logstash