123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236 |
- ---
- apiVersion: apps/v1
- kind: StatefulSet
- metadata:
- name: realtime-yuewen-v2
- namespace: pro
- labels:
- app: realtime-yuewen-v2
- spec:
- serviceName: realtime-yuewen-v2
- replicas: 1
- selector:
- matchLabels:
- app: realtime-yuewen-v2
- template:
- metadata:
- labels:
- app: realtime-yuewen-v2
- spec:
- terminationGracePeriodSeconds: 30
- dnsPolicy: ClusterFirstWithHostNet
- dnsConfig:
- nameservers:
- - 10.108.30.101
- - 223.5.5.5
- initContainers:
- - name: init
- image: hub.evbj.easou.com/dev/busybox
- command: [ "chmod","777","-R","/data" ]
- imagePullPolicy: IfNotPresent
- volumeMounts:
- - name: volume
- mountPath: /data
- #nodeSelector:
- # app.touchrs: touchrs
- # affinity:
- # podAntiAffinity:
- # preferredDuringSchedulingIgnoredDuringExecution:
- # - weight: 100
- # podAffinityTerm:
- # labelSelector:
- # matchExpressions:
- # - key: app
- # operator: In
- # values:
- # - realtime-yuewen-v2
- # topologyKey: kubernetes.io/hostname
- containers:
- - name: realtime-yuewen-v2
- image: hub.evbj.easou.com/pro/realtime-yuewen-v2:r188751
- imagePullPolicy: IfNotPresent
- env:
- - name: MY_NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: spec.nodeName
- - name: MY_POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- - name: MY_POD_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: MY_POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: JAVA_OPTS
- value: "-server -Xmx2g -Xms2g -Xss512k -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=product"
- ports:
- - containerPort: 8080
- name: port
- - containerPort: 8001
- name: "jvm-debug"
- volumeMounts:
- - name: volume
- mountPath: /data
- lifecycle:
- preStop:
- exec:
- command: ["/bin/sh","-c","curl -H 'loginName:admin' -H 'password:ea1so2ua3dm4in5' -X POST 127.0.0.1:8080/actuator/shutdown"]
- livenessProbe:
- httpGet:
- path: /actuator/health
- port: 8080
- httpHeaders:
- - name: loginName
- value: admin
- - name: password
- value: ea1so2ua3dm4in5
- initialDelaySeconds: 10 #60s后启动第一次探测
- periodSeconds: 10 # 每隔10s启动一次探测
- timeoutSeconds: 3 # 超时时间3s
- successThreshold: 1 # 成功1次即表示容器健康
- failureThreshold: 2 # 连续5次失败,则判定容器不健康,默认3次
- readinessProbe:
- tcpSocket:
- port: 8080
- initialDelaySeconds: 40
- periodSeconds: 15
- - image: hub.evbj.easou.com/dev/flume:1.7.0-2023081012
- imagePullPolicy: IfNotPresent
- name: flume
- volumeMounts:
- - name: volume #日志同时挂载在nginx和filebeat中
- mountPath: /data
- - name: flume-config
- mountPath: /app/soft/flume-1.7.0/conf/flume.conf
- subPath: flume.conf
- volumes:
- - name: flume-config
- configMap:
- name: flume-config-realtime-yuewen-v2
- items:
- - key: flume.conf
- path: flume.conf
- volumeClaimTemplates:
- - metadata:
- name: volume
- spec:
- accessModes: [ "ReadWriteOnce" ]
- storageClassName: rbd
- resources:
- requests:
- storage: 100Gi
- ---
- apiVersion: v1
- kind: Service
- metadata:
- name: realtime-yuewen-v2-svc
- namespace: pro
- labels:
- app: realtime-yuewen-v2-svc
- spec:
- type: NodePort
- ports:
- - port: 80
- targetPort: 8080
- name: "main"
- selector:
- app: realtime-yuewen-v2
- ---
- apiVersion: networking.k8s.io/v1
- kind: Ingress
- metadata:
- name: realtime-yuewen-v2-ingress
- namespace: pro
- spec:
- rules:
- - host: ywrealtimev2.appeasou.com
- http:
- paths:
- - path: /
- pathType: Prefix
- backend:
- service:
- name: realtime-yuewen-v2-svc
- port:
- number: 8080
- - host: pro-realtime-yuewen-v2.ieasou.cn
- http:
- paths:
- - path: /
- pathType: Prefix
- backend:
- service:
- name: realtime-yuewen-v2-svc
- port:
- number: 8080
- ingressClassName: nginx
- ---
- apiVersion: v1
- kind: ConfigMap
- metadata:
- name: flume-config-realtime-yuewen-v2
- namespace: pro
- data:
- flume.conf: |
- a1.sources = r1
- a1.channels = c1
- a1.sinks =
-
- a1.sources.r1.type = com.github.ningg.flume.source.SpoolDirectoryTailFileSource
- a1.sources.r1.channels = c1
- a1.sources.r1.spoolDir = /data/logs/app/
- a1.sources.r1.fileSuffix = .COMPLETED
- a1.sources.r1.deletePolicy = never
- a1.sources.r1.ignorePattern = ^$
- a1.sources.r1.targetPattern = access.*(\\d){10}.*
- a1.sources.r1.targetFilename = yyyyMMddHH
- a1.sources.r1.trackerDir = .flumespooltail_earning
- a1.sources.r1.consumeOrder = oldest
- a1.sources.r1.bufferMaxLineLength = 500000
- a1.sources.r1.inputCharset = UTF-8
- a1.sources.r1.decodeErrorPolicy = REPLACE
- a1.sources.r1.deserializer = LINE
- a1.sources.r1.interceptors = i1 i2 i3 i4
-
- #配置 interceptor i1
- a1.sources.r1.interceptors.i1.type = regex_filter
- a1.sources.r1.interceptors.i1.regex = ^\\s*$
- a1.sources.r1.interceptors.i1.excludeEvents = true
- #配置 interceptor i2
- a1.sources.r1.fileHeader = true
- a1.sources.r1.basenameHeader = true
- a1.sources.r1.interceptors.i2.type = com.easou.flume.interceptor.RegexExtractorExtInterceptor$Builder
- a1.sources.r1.interceptors.i2.regex = (.*)\\.(.*)\\.(.*)
- a1.sources.r1.interceptors.i2.extractorHeader = true
- a1.sources.r1.interceptors.i2.extractorHeaderKey = basename
- a1.sources.r1.interceptors.i2.serializers = s1 s2 s3
- a1.sources.r1.interceptors.i2.serializers.s1.name = log_type
- a1.sources.r1.interceptors.i2.serializers.s2.name = file_type
- a1.sources.r1.interceptors.i2.serializers.s3.name = log_data
- #配置 interceptor i3
- a1.sources.r1.interceptors.i3.type=com.easou.flume.interceptor.ServerTimeInterceptor$Builder
- a1.sources.r1.interceptors.i3.isServerTime=true
- a1.sources.r1.interceptors.i3.timeType=yyyyMMddHHmmss
- #配置 interceptor i4
- a1.sources.r1.interceptors.i4.type = timestamp
-
-
- a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
- a1.channels.c1.kafka.bootstrap.servers = 10.26.27.71:8092,10.26.22.102:8092,10.26.22.120:8092,10.26.22.121:8092,10.26.22.122:8092
- a1.channels.c1.kafka.topic = yuewen_buydetail
- # 开启kafka幂等性,保证重试不会产生数据重复
- a1.channels.c1.kafka.producer.enable.idempotence = true
- a1.channels.c1.kafka.producer.acks = all
- a1.channels.c1.kafka.producer.max.in.flight.requests.per.connection = 1
- # 重试 20次,每次超时30秒,一次发送数据最长约10分钟。尽量保证kafka不报异常,因为异常后flumesource会重试,导致数据重复
- a1.channels.c1.kafka.producer.retries = 20
- a1.channels.c1.kafka.producer.request.timeout.ms = 30000
- a1.channels.c1.kafka.consumer.group.id = integral2kafka
|