123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156 |
- ---
- kind: ConfigMap
- apiVersion: v1
- metadata:
- name: logstash-config
- namespace: pro
- data:
- logstash-config-named-k8s: |
- input {
- kafka {
- bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
- group_id => "es-test"
- auto_offset_reset => "latest"
- consumer_threads => 5
- max_partition_fetch_bytes => 4194304
- receive_buffer_bytes => 131072
- send_buffer_bytes => 524288
- topics => ["offline-process-topic"]
- codec => json
- }
- }
-
- filter {
- mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
- grok{
- match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"}
- overwrite => ["message"]
- }
- date {
- match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
- }
- }
- output {
- stdout { codec => rubydebug }
- if [log_topic] == "offline-process-charge" {
- elasticsearch {
- hosts => ["elasticsearch:9200"]
- index => "logstash-offline-process-charge-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-charge-callback" {
- elasticsearch {
- hosts => ["elasticsearch:9200"]
- index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-promotion" {
- elasticsearch {
- hosts => ["elasticsearch:9200"]
- index => "logstash-offline-process-promotion-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-newmedia" {
- elasticsearch {
- hosts => ["elasticsearch:9200"]
- index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-schedule" {
- elasticsearch {
- hosts => ["elasticsearch:9200"]
- index => "logstash-offline-process-schedule-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- if [log_topic] == "offline-process-statistics" {
- elasticsearch {
- hosts => ["elasticsearch:9200"]
- index => "logstash-offline-process-statistics-%{+YYYY.MM}"
- template_overwrite => true
- }
- }
- }
- logstash_yml: |
- http.host: "0.0.0.0"
- jvm-options: |
- -Xms4g
- -Xmx4g
- -XX:+UseConcMarkSweepGC
- -XX:CMSInitiatingOccupancyFraction=75
- -XX:+UseCMSInitiatingOccupancyOnly
- -Djava.awt.headless=true
- -Dfile.encoding=UTF-8
- -Djruby.compile.invokedynamic=true
- -Djruby.jit.threshold=0
- -Djruby.regexp.interruptible=true
- -XX:+HeapDumpOnOutOfMemoryError
- -Djava.security.egd=file:/dev/urandom
- -Dlog4j2.isThreadContextMapInheritable=true
- ---
- kind: Deployment
- apiVersion: apps/v1
- metadata:
- name: logstash-k8s-named
- namespace: pro
- labels:
- app: logstash-k8s-named
- spec:
- replicas: 3
- selector:
- matchLabels:
- app: logstash-k8s-named
- template:
- metadata:
- labels:
- app: logstash-k8s-named
- spec:
- containers:
- - name: logstash-k8s-named
- image: logstash:7.13.3
- command: ["/bin/sh","-c"]
- args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
- ports:
- - containerPort: 5000
- name: client1
- - containerPort: 9600
- name: api
- volumeMounts:
- - name: logstash-config
- mountPath: /usr/share/logstash/config
- - name: jvm-options
- mountPath: /usr/share/logstash/jvm.options
- volumes:
- - name: logstash-config
- configMap:
- name: logstash-config
- items:
- - key: logstash-config-named-k8s
- path: logstash.conf
- - name: jvm-options
- configMap:
- name: logstash-config
- items:
- - key: jvm-options
- path: jvm.options
- ---
- kind: Service
- apiVersion: v1
- metadata:
- labels:
- app: logstash
- name: logstash
- namespace: pro
- spec:
- ports:
- - name: http
- port: 5000
- protocol: TCP
- targetPort: 5000
- selector:
- app: logstash
|