|
@@ -3,12 +3,12 @@ apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
kind: StatefulSet
|
|
metadata:
|
|
metadata:
|
|
name: realtime-yuewen
|
|
name: realtime-yuewen
|
|
- namespace: pro
|
|
|
|
|
|
+ namespace: qa
|
|
labels:
|
|
labels:
|
|
app: realtime-yuewen
|
|
app: realtime-yuewen
|
|
spec:
|
|
spec:
|
|
serviceName: realtime-yuewen
|
|
serviceName: realtime-yuewen
|
|
- replicas: 2
|
|
|
|
|
|
+ replicas: 1
|
|
selector:
|
|
selector:
|
|
matchLabels:
|
|
matchLabels:
|
|
app: realtime-yuewen
|
|
app: realtime-yuewen
|
|
@@ -33,21 +33,21 @@ spec:
|
|
mountPath: /data
|
|
mountPath: /data
|
|
#nodeSelector:
|
|
#nodeSelector:
|
|
# app.touchrs: touchrs
|
|
# app.touchrs: touchrs
|
|
-# affinity:
|
|
|
|
-# podAntiAffinity:
|
|
|
|
-# preferredDuringSchedulingIgnoredDuringExecution:
|
|
|
|
-# - weight: 100
|
|
|
|
-# podAffinityTerm:
|
|
|
|
-# labelSelector:
|
|
|
|
-# matchExpressions:
|
|
|
|
-# - key: app
|
|
|
|
-# operator: In
|
|
|
|
-# values:
|
|
|
|
-# - realtime-yuewen
|
|
|
|
-# topologyKey: kubernetes.io/hostname
|
|
|
|
|
|
+ # affinity:
|
|
|
|
+ # podAntiAffinity:
|
|
|
|
+ # preferredDuringSchedulingIgnoredDuringExecution:
|
|
|
|
+ # - weight: 100
|
|
|
|
+ # podAffinityTerm:
|
|
|
|
+ # labelSelector:
|
|
|
|
+ # matchExpressions:
|
|
|
|
+ # - key: app
|
|
|
|
+ # operator: In
|
|
|
|
+ # values:
|
|
|
|
+ # - realtime-yuewen
|
|
|
|
+ # topologyKey: kubernetes.io/hostname
|
|
containers:
|
|
containers:
|
|
- name: realtime-yuewen
|
|
- name: realtime-yuewen
|
|
- image: hub.evbj.easou.com/pro/realtime-yuewen:r181368
|
|
|
|
|
|
+ image: hub.evbj.easou.com/qa/realtime-yuewen:r181373
|
|
imagePullPolicy: IfNotPresent
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
env:
|
|
- name: MY_NODE_NAME
|
|
- name: MY_NODE_NAME
|
|
@@ -67,7 +67,7 @@ spec:
|
|
fieldRef:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
fieldPath: metadata.namespace
|
|
- name: JAVA_OPTS
|
|
- name: JAVA_OPTS
|
|
- value: "-server -Xmx2g -Xms2g -Xss512k -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=product"
|
|
|
|
|
|
+ value: "-server -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8001,suspend=n -Xmx2g -Xms2g -Xss512k -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=test"
|
|
ports:
|
|
ports:
|
|
- containerPort: 8080
|
|
- containerPort: 8080
|
|
name: port
|
|
name: port
|
|
@@ -99,38 +99,22 @@ spec:
|
|
port: 8080
|
|
port: 8080
|
|
initialDelaySeconds: 40
|
|
initialDelaySeconds: 40
|
|
periodSeconds: 15
|
|
periodSeconds: 15
|
|
- - image: hub.evbj.easou.com/pro/fluent-bit
|
|
|
|
|
|
+ - image: hub.evbj.easou.com/dev/flume:1.7.0-20220930-v1
|
|
imagePullPolicy: IfNotPresent
|
|
imagePullPolicy: IfNotPresent
|
|
- name: fluent-bit
|
|
|
|
- ports:
|
|
|
|
- - containerPort: 2020
|
|
|
|
- name: metrics
|
|
|
|
- protocol: TCP
|
|
|
|
- terminationMessagePath: /dev/termination-log
|
|
|
|
- terminationMessagePolicy: File
|
|
|
|
|
|
+ name: flume
|
|
volumeMounts:
|
|
volumeMounts:
|
|
- name: volume #日志同时挂载在nginx和filebeat中
|
|
- name: volume #日志同时挂载在nginx和filebeat中
|
|
mountPath: /data
|
|
mountPath: /data
|
|
- - name: fluent-bit-config
|
|
|
|
- mountPath: /opt/bitnami/fluent-bit/conf/fluent-bit.conf
|
|
|
|
- subPath: fluent-bit.conf
|
|
|
|
- - name: fluent-bit-config-parses
|
|
|
|
- mountPath: /opt/bitnami/fluent-bit/conf/parsers.conf
|
|
|
|
- subPath: parsers.conf
|
|
|
|
|
|
+ - name: flume-config
|
|
|
|
+ mountPath: /app/soft/flume-1.7.0/conf/flume.conf
|
|
|
|
+ subPath: flume.conf
|
|
volumes:
|
|
volumes:
|
|
- - name: fluent-bit-config
|
|
|
|
- configMap:
|
|
|
|
- name: fluent-bit-config-realtime-yuewen
|
|
|
|
- items:
|
|
|
|
- - key: fluent-bit.conf
|
|
|
|
- path: fluent-bit.conf
|
|
|
|
- - name: fluent-bit-config-parses
|
|
|
|
|
|
+ - name: flume-config
|
|
configMap:
|
|
configMap:
|
|
- name: fluent-bit-config-realtime-yuewen
|
|
|
|
|
|
+ name: flume-config-realtime-yuewen
|
|
items:
|
|
items:
|
|
- - key: parsers.conf
|
|
|
|
- path: parsers.conf
|
|
|
|
-
|
|
|
|
|
|
+ - key: flume.conf
|
|
|
|
+ path: flume.conf
|
|
|
|
|
|
volumeClaimTemplates:
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
- metadata:
|
|
@@ -146,7 +130,7 @@ apiVersion: v1
|
|
kind: Service
|
|
kind: Service
|
|
metadata:
|
|
metadata:
|
|
name: realtime-yuewen-svc
|
|
name: realtime-yuewen-svc
|
|
- namespace: pro
|
|
|
|
|
|
+ namespace: qa
|
|
labels:
|
|
labels:
|
|
app: realtime-yuewen-svc
|
|
app: realtime-yuewen-svc
|
|
spec:
|
|
spec:
|
|
@@ -162,7 +146,7 @@ apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
kind: Ingress
|
|
metadata:
|
|
metadata:
|
|
name: realtime-yuewen-ingress
|
|
name: realtime-yuewen-ingress
|
|
- namespace: pro
|
|
|
|
|
|
+ namespace: qa
|
|
spec:
|
|
spec:
|
|
rules:
|
|
rules:
|
|
- host: ywrealtime.appeasou.com
|
|
- host: ywrealtime.appeasou.com
|
|
@@ -175,50 +159,54 @@ spec:
|
|
name: realtime-yuewen-svc
|
|
name: realtime-yuewen-svc
|
|
port:
|
|
port:
|
|
number: 8080
|
|
number: 8080
|
|
- - host: pro-realtime-yuewen.ieasou.cn
|
|
|
|
|
|
+ - host: qa-realtime-yuewen.ieasou.cn
|
|
http:
|
|
http:
|
|
paths:
|
|
paths:
|
|
- - path: /
|
|
|
|
- pathType: Prefix
|
|
|
|
- backend:
|
|
|
|
- service:
|
|
|
|
- name: realtime-yuewen-svc
|
|
|
|
- port:
|
|
|
|
- number: 8080
|
|
|
|
|
|
+ - path: /
|
|
|
|
+ pathType: Prefix
|
|
|
|
+ backend:
|
|
|
|
+ service:
|
|
|
|
+ name: realtime-yuewen-svc
|
|
|
|
+ port:
|
|
|
|
+ number: 8080
|
|
ingressClassName: nginx
|
|
ingressClassName: nginx
|
|
---
|
|
---
|
|
apiVersion: v1
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
kind: ConfigMap
|
|
metadata:
|
|
metadata:
|
|
- name: fluent-bit-config-realtime-yuewen
|
|
|
|
- namespace: pro
|
|
|
|
|
|
+ name: flume-config-realtime-yuewen
|
|
|
|
+ namespace: qa
|
|
data:
|
|
data:
|
|
- fluent-bit.conf: |
|
|
|
|
- [SERVICE]
|
|
|
|
- Flush 5
|
|
|
|
- Daemon off
|
|
|
|
- Log_Level info
|
|
|
|
- [INPUT]
|
|
|
|
- Name tail
|
|
|
|
- Path /data/logs/app/access.log
|
|
|
|
- Mem_Buf_Limit 5MB
|
|
|
|
- Skip_Long_Lines On
|
|
|
|
- [OUTPUT]
|
|
|
|
- Name kafka
|
|
|
|
- Match *
|
|
|
|
- Brokers 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
|
|
|
|
- Topics test_realtime_yuewen_nginx
|
|
|
|
- Retry_Limit false
|
|
|
|
- Message_Key log
|
|
|
|
- # hides errors "Receive failed: Disconnected" when kafka kills idle connections
|
|
|
|
- rdkafka.log.connection.close false
|
|
|
|
- # producer buffer is not included in http://fluentbit.io/documentation/0.12/configuration/memory_usage.html#estimating
|
|
|
|
- rdkafka.queue.buffering.max.kbytes 10240
|
|
|
|
- # for logs you'll probably want this ot be 0 or 1, not more
|
|
|
|
- rdkafka.request.required.acks 1
|
|
|
|
|
|
+ flume.conf: |
|
|
|
|
+ a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
|
|
|
|
+ a1.channels.c1.kafka.bootstrap.servers = 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
|
|
|
|
+ a1.channels.c1.kafka.topic = realtime_yuewen_nginx
|
|
|
|
+ a1.channels.c1.kafka.consumer.group.id = flume2kafka
|
|
|
|
+
|
|
|
|
+ # Define an Avro source called r1 on a1 and tell it
|
|
|
|
+ # to bind to 0.0.0.0:41414. Connect it to channel c1.
|
|
|
|
+ #注意这里分出2个channel出来,一个是到k1,一个到k2
|
|
|
|
+ #如果这里只出一个channel c1的话,下面2个sink channel都为c1,则消息就会随机分到2个sink上,而不是2个sink都有一份
|
|
|
|
+ a1.sources.r1.channels = c1
|
|
|
|
+ a1.sources.r1.type = exec
|
|
|
|
+ a1.sources.r1.command = tail -F /data/logs/app/access.log
|
|
|
|
+ a1.sources.r1.restart = true
|
|
|
|
+ a1.sources.r1.selector.optional = c1
|
|
|
|
+ a1.sources.r1.batchSize = 100000
|
|
|
|
+ a1.sources.r1.interceptors = i1 i2 i3
|
|
|
|
+ a1.sources.r1.interceptors.i1.type = timestamp
|
|
|
|
+ a1.sources.r1.interceptors.i2.type = host
|
|
|
|
+ a1.sources.r1.interceptors.i2.useIP = false
|
|
|
|
+ a1.sources.r1.interceptors.i3.type = regex_filter
|
|
|
|
+ a1.sources.r1.interceptors.i3.regex = ^\\s*$
|
|
|
|
+ a1.sources.r1.interceptors.i3.excludeEvents = true
|
|
|
|
+ #默认是replicationg,还有multiplexer
|
|
|
|
+ #a1.sources.r1.selector.type = replicationg
|
|
|
|
+
|
|
|
|
+ # Finally, now that we've defined all of our components, tell
|
|
|
|
+ # a1 which ones we want to activate.
|
|
|
|
+ a1.sources = r1
|
|
|
|
+ a1.channels = c1
|
|
|
|
+ a1.sinks =
|
|
|
|
+
|
|
|
|
|
|
- parsers.conf: |
|
|
|
|
- [PARSER]
|
|
|
|
- Name multiline_pattern
|
|
|
|
- Format regex
|
|
|
|
- Regex ^\\s*$
|
|
|