123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135 |
- ---
- kind: ConfigMap
- apiVersion: v1
- metadata:
- name: logstash-configmap
- namespace: pro
- data:
- logstash.yml: |
- http.host: "0.0.0.0"
- logstash.conf: |
- input {
- kafka {
- bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
- group_id => "es-test"
- auto_offset_reset => "latest"
- consumer_threads => 3
- max_partition_fetch_bytes => 4194304
- receive_buffer_bytes => 131072
- send_buffer_bytes => 524288
- topics => ["offline-process-topic"]
- codec => json
- }
- }
-
- filter {
- mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score", "@version", "_id", "[agent][type]", "[agent][id]", "[agent][name]", "[agent][version]", "[agent][ephemeral_id]", "[ecs][version]" ]}
- grok{
- match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"}
- overwrite => ["message"]
- }
- date {
- match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
- }
- }
- output {
- # stdout { codec => rubydebug }
- if [log_topic] == "offline-process-charge" {
- elasticsearch {
- hosts => ["10.26.27.137:9200"]
- index => "offline-process-charge-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-charge-callback" {
- elasticsearch {
- hosts => ["10.26.27.137:9200"]
- index => "offline-process-charge-callback-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-promotion" {
- elasticsearch {
- hosts => ["10.26.27.137:9200"]
- index => "offline-process-promotion-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-schedule" {
- elasticsearch {
- hosts => ["10.26.27.137:9200"]
- index => "offline-process-schedule-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- }
- ---
- kind: Deployment
- apiVersion: apps/v1
- metadata:
- name: logstash-k8s-named
- namespace: pro
- labels:
- app: logstash-k8s-named
- spec:
- replicas: 3
- selector:
- matchLabels:
- app: logstash-k8s-named
- template:
- metadata:
- labels:
- app: logstash-k8s-named
- spec:
- containers:
- - name: logstash-k8s-named
- imagePullPolicy: IfNotPresent
- image: hub.evbj.easou.com/pro/logstash:7.13.3
- command: ["/bin/sh","-c"]
- args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config"]
- ports:
- - containerPort: 5000
- name: client1
- - containerPort: 9600
- name: api
- volumeMounts:
- - name: config-volume
- mountPath: /usr/share/logstash/config
- - name: yml-volume
- mountPath: /usr/share/logstash/config
- env:
- - name: LS_HEAP_SIZE
- value: 4096m
- - name: LS_HTTP_HOST
- value: "0.0.0.0"
- volumes:
- - name: yml-volume
- configMap:
- name: logstash-configmap
- items:
- - key: logstash.yml
- path: logstash.yml
- - name: config-volume
- configMap:
- name: logstash-configmap
- items:
- - key: logstash.conf
- path: logstash.conf
- ---
- kind: Service
- apiVersion: v1
- metadata:
- labels:
- app: logstash
- name: logstash
- namespace: pro
- spec:
- ports:
- - name: http
- port: 5000
- protocol: TCP
- targetPort: 5000
- selector:
- app: logstash
|