|
@@ -48,7 +48,7 @@ spec:
|
|
# topologyKey: kubernetes.io/hostname
|
|
# topologyKey: kubernetes.io/hostname
|
|
containers:
|
|
containers:
|
|
- name: realtime-yuewen
|
|
- name: realtime-yuewen
|
|
- image: hub.evbj.easou.com/pro/realtime-yuewen:r180938
|
|
|
|
|
|
+ image: hub.evbj.easou.com/pro/realtime-yuewen:r181326
|
|
imagePullPolicy: IfNotPresent
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
env:
|
|
- name: MY_NODE_NAME
|
|
- name: MY_NODE_NAME
|
|
@@ -100,22 +100,28 @@ spec:
|
|
port: 8080
|
|
port: 8080
|
|
initialDelaySeconds: 40
|
|
initialDelaySeconds: 40
|
|
periodSeconds: 15
|
|
periodSeconds: 15
|
|
-# - image: hub.evbj.easou.com/pro/filebeat:7.13.3
|
|
|
|
-# imagePullPolicy: IfNotPresent
|
|
|
|
-# name: filebeat
|
|
|
|
-# volumeMounts:
|
|
|
|
-# - name: volume #日志同时挂载在nginx和filebeat中
|
|
|
|
-# mountPath: /data
|
|
|
|
-# - name: filebeat-config
|
|
|
|
-# mountPath: /usr/share/filebeat/filebeat.yml
|
|
|
|
-# subPath: filebeat.yml
|
|
|
|
-# volumes:
|
|
|
|
-# - name: filebeat-config
|
|
|
|
-# configMap:
|
|
|
|
-# name: filebeat-config-realtime-yuewen
|
|
|
|
-# items:
|
|
|
|
-# - key: filebeat.yml
|
|
|
|
-# path: filebeat.yml
|
|
|
|
|
|
+ - image: fluent-bit
|
|
|
|
+ imagePullPolicy: IfNotPresent
|
|
|
|
+ name: fluent-bit
|
|
|
|
+ ports:
|
|
|
|
+ - containerPort: 2020
|
|
|
|
+ name: metrics
|
|
|
|
+ protocol: TCP
|
|
|
|
+ terminationMessagePath: /dev/termination-log
|
|
|
|
+ terminationMessagePolicy: File
|
|
|
|
+ volumeMounts:
|
|
|
|
+ - name: volume #日志同时挂载在nginx和filebeat中
|
|
|
|
+ mountPath: /data
|
|
|
|
+ - name: fluent-bit-config
|
|
|
|
+ mountPath: /fluent-bit/etc/fluent-bit.conf
|
|
|
|
+ subPath: fluent-bit.conf
|
|
|
|
+ volumes:
|
|
|
|
+ - name: fluent-bit-config
|
|
|
|
+ configMap:
|
|
|
|
+ name: fluent-bit-config-realtime-yuewen
|
|
|
|
+ items:
|
|
|
|
+ - key: fluent-bit.conf
|
|
|
|
+ path: fluent-bit.conf
|
|
|
|
|
|
volumeClaimTemplates:
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
- metadata:
|
|
@@ -175,31 +181,28 @@ spec:
|
|
apiVersion: v1
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
kind: ConfigMap
|
|
metadata:
|
|
metadata:
|
|
- name: filebeat-config-realtime-yuewen
|
|
|
|
|
|
+ name: fluent-bit-config-realtime-yuewen
|
|
namespace: pro
|
|
namespace: pro
|
|
data:
|
|
data:
|
|
- filebeat.yml: |
|
|
|
|
- filebeat.inputs:
|
|
|
|
- - type: log
|
|
|
|
- enabled: true
|
|
|
|
- #多行合并
|
|
|
|
- multiline.pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
|
|
|
|
- multiline.negate: true
|
|
|
|
- multiline.match: after
|
|
|
|
- multiline.timeout: 30
|
|
|
|
- fields:
|
|
|
|
- log_topic: realtime-yuewen
|
|
|
|
- fields_under_root: true
|
|
|
|
- paths:
|
|
|
|
- - "/data/logs/app/*.log"
|
|
|
|
- - "/data/logs/*.log"
|
|
|
|
- output.kafka:
|
|
|
|
- version: 2.0.0
|
|
|
|
- enable: true
|
|
|
|
- hosts: ["kafka-0.kafka-headless.pro.svc.cluster.local:9093", "kafka-1.kafka-headless.pro.svc.cluster.local:9093", "kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
|
|
|
|
- topic: 'offline-process-topic'
|
|
|
|
- required_acks: 1
|
|
|
|
- partition.round_robin:
|
|
|
|
- reachable_only: false
|
|
|
|
- compression: gzip
|
|
|
|
- max_message_bytes: 1000000
|
|
|
|
|
|
+ fluent-bit.conf: |
|
|
|
|
+ [INPUT]
|
|
|
|
+ Name tail
|
|
|
|
+ Path /var/log/containers/*.log
|
|
|
|
+ Parser cri
|
|
|
|
+ Tag kube.*
|
|
|
|
+ Mem_Buf_Limit 5MB
|
|
|
|
+ Skip_Long_Lines On
|
|
|
|
+ [OUTPUT]
|
|
|
|
+ Name kafka
|
|
|
|
+ Match *
|
|
|
|
+ Brokers 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
|
|
|
|
+ Topics realtime_yuewen_nginx
|
|
|
|
+ Timestamp_Key @timestamp
|
|
|
|
+ Retry_Limit false
|
|
|
|
+ # hides errors "Receive failed: Disconnected" when kafka kills idle connections
|
|
|
|
+ rdkafka.log.connection.close false
|
|
|
|
+ # producer buffer is not included in http://fluentbit.io/documentation/0.12/configuration/memory_usage.html#estimating
|
|
|
|
+ rdkafka.queue.buffering.max.kbytes 10240
|
|
|
|
+ # for logs you'll probably want this ot be 0 or 1, not more
|
|
|
|
+ rdkafka.request.required.acks 1
|
|
|
|
+
|