deploy-tmp.yaml 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. ---
  2. apiVersion: apps/v1
  3. kind: StatefulSet
  4. metadata:
  5. name: realtime-yuewen-v2
  6. namespace: pro
  7. labels:
  8. app: realtime-yuewen-v2
  9. spec:
  10. serviceName: realtime-yuewen-v2
  11. replicas: 2
  12. selector:
  13. matchLabels:
  14. app: realtime-yuewen-v2
  15. template:
  16. metadata:
  17. labels:
  18. app: realtime-yuewen-v2
  19. spec:
  20. terminationGracePeriodSeconds: 30
  21. dnsPolicy: ClusterFirstWithHostNet
  22. dnsConfig:
  23. nameservers:
  24. - 10.108.30.101
  25. - 223.5.5.5
  26. initContainers:
  27. - name: init
  28. image: hub.evbj.easou.com/dev/busybox
  29. command: [ "chmod","777","-R","/data" ]
  30. imagePullPolicy: IfNotPresent
  31. volumeMounts:
  32. - name: volume
  33. mountPath: /data
  34. #nodeSelector:
  35. # app.touchrs: touchrs
  36. # affinity:
  37. # podAntiAffinity:
  38. # preferredDuringSchedulingIgnoredDuringExecution:
  39. # - weight: 100
  40. # podAffinityTerm:
  41. # labelSelector:
  42. # matchExpressions:
  43. # - key: app
  44. # operator: In
  45. # values:
  46. # - realtime-yuewen-v2
  47. # topologyKey: kubernetes.io/hostname
  48. containers:
  49. - name: realtime-yuewen-v2
  50. image: hub.evbj.easou.com/pro/realtime-yuewen-v2:r188700
  51. imagePullPolicy: IfNotPresent
  52. env:
  53. - name: MY_NODE_NAME
  54. valueFrom:
  55. fieldRef:
  56. fieldPath: spec.nodeName
  57. - name: MY_POD_IP
  58. valueFrom:
  59. fieldRef:
  60. fieldPath: status.podIP
  61. - name: MY_POD_NAME
  62. valueFrom:
  63. fieldRef:
  64. fieldPath: metadata.name
  65. - name: MY_POD_NAMESPACE
  66. valueFrom:
  67. fieldRef:
  68. fieldPath: metadata.namespace
  69. - name: JAVA_OPTS
  70. value: "-server -Xmx2g -Xms2g -Xss512k -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=product"
  71. ports:
  72. - containerPort: 8080
  73. name: port
  74. - containerPort: 8001
  75. name: "jvm-debug"
  76. volumeMounts:
  77. - name: volume
  78. mountPath: /data
  79. lifecycle:
  80. preStop:
  81. exec:
  82. command: ["/bin/sh","-c","curl -H 'loginName:admin' -H 'password:ea1so2ua3dm4in5' -X POST 127.0.0.1:8080/actuator/shutdown"]
  83. livenessProbe:
  84. httpGet:
  85. path: /actuator/health
  86. port: 8080
  87. httpHeaders:
  88. - name: loginName
  89. value: admin
  90. - name: password
  91. value: ea1so2ua3dm4in5
  92. initialDelaySeconds: 10 #60s后启动第一次探测
  93. periodSeconds: 10 # 每隔10s启动一次探测
  94. timeoutSeconds: 3 # 超时时间3s
  95. successThreshold: 1 # 成功1次即表示容器健康
  96. failureThreshold: 2 # 连续5次失败,则判定容器不健康,默认3次
  97. readinessProbe:
  98. tcpSocket:
  99. port: 8080
  100. initialDelaySeconds: 40
  101. periodSeconds: 15
  102. - image: hub.evbj.easou.com/dev/flume:1.7.0-2023081012
  103. imagePullPolicy: IfNotPresent
  104. name: flume
  105. volumeMounts:
  106. - name: volume #日志同时挂载在nginx和filebeat中
  107. mountPath: /data
  108. volumes:
  109. - name: flume-config
  110. configMap:
  111. name: flume-config-realtime-yuewen-v2
  112. items:
  113. - key: flume.conf
  114. path: flume.conf
  115. volumeClaimTemplates:
  116. - metadata:
  117. name: volume
  118. spec:
  119. accessModes: [ "ReadWriteOnce" ]
  120. storageClassName: rbd
  121. resources:
  122. requests:
  123. storage: 100Gi
  124. ---
  125. apiVersion: v1
  126. kind: Service
  127. metadata:
  128. name: realtime-yuewen-v2-svc
  129. namespace: pro
  130. labels:
  131. app: realtime-yuewen-v2-svc
  132. spec:
  133. type: NodePort
  134. ports:
  135. - port: 80
  136. targetPort: 8080
  137. name: "main"
  138. selector:
  139. app: realtime-yuewen-v2
  140. ---
  141. apiVersion: networking.k8s.io/v1
  142. kind: Ingress
  143. metadata:
  144. name: realtime-yuewen-v2-ingress
  145. namespace: pro
  146. spec:
  147. rules:
  148. - host: ywrealtimev2.appeasou.com
  149. http:
  150. paths:
  151. - path: /
  152. pathType: Prefix
  153. backend:
  154. service:
  155. name: realtime-yuewen-v2-svc
  156. port:
  157. number: 8080
  158. - host: pro-realtime-yuewen-v2.ieasou.cn
  159. http:
  160. paths:
  161. - path: /
  162. pathType: Prefix
  163. backend:
  164. service:
  165. name: realtime-yuewen-v2-svc
  166. port:
  167. number: 8080
  168. ingressClassName: nginx
  169. ---
  170. apiVersion: v1
  171. kind: ConfigMap
  172. metadata:
  173. name: flume-config-realtime-yuewen-v2
  174. namespace: pro
  175. data:
  176. flume.conf: |
  177. a1.sources = r1 r2
  178. a1.channels = c1 c2
  179. a1.sinks =
  180. a1.sources.r1.type = com.github.ningg.flume.source.SpoolDirectoryTailFileSource
  181. a1.sources.r1.channels = c1
  182. a1.sources.r1.spoolDir = /data/logs/app/
  183. a1.sources.r1.fileSuffix = .COMPLETED
  184. a1.sources.r1.deletePolicy = never
  185. a1.sources.r1.ignorePattern = ^$
  186. a1.sources.r1.targetPattern = ywbuy_earning.*(\\d){10}.*
  187. a1.sources.r1.targetFilename = yyyyMMddHH
  188. a1.sources.r1.trackerDir = .flumespooltail_earning
  189. a1.sources.r1.consumeOrder = oldest
  190. a1.sources.r1.bufferMaxLineLength = 500000
  191. a1.sources.r1.inputCharset = UTF-8
  192. a1.sources.r1.decodeErrorPolicy = REPLACE
  193. a1.sources.r1.deserializer = LINE
  194. a1.sources.r1.interceptors = i1 i2 i3 i4
  195. #配置 interceptor i1
  196. a1.sources.r1.interceptors.i1.type = regex_filter
  197. a1.sources.r1.interceptors.i1.regex = ^\\s*$
  198. a1.sources.r1.interceptors.i1.excludeEvents = true
  199. #配置 interceptor i2
  200. a1.sources.r1.fileHeader = true
  201. a1.sources.r1.basenameHeader = true
  202. a1.sources.r1.interceptors.i2.type = com.easou.flume.interceptor.RegexExtractorExtInterceptor$Builder
  203. a1.sources.r1.interceptors.i2.regex = (.*)\\.(.*)\\.(.*)
  204. a1.sources.r1.interceptors.i2.extractorHeader = true
  205. a1.sources.r1.interceptors.i2.extractorHeaderKey = basename
  206. a1.sources.r1.interceptors.i2.serializers = s1 s2 s3
  207. a1.sources.r1.interceptors.i2.serializers.s1.name = log_type
  208. a1.sources.r1.interceptors.i2.serializers.s2.name = file_type
  209. a1.sources.r1.interceptors.i2.serializers.s3.name = log_data
  210. #配置 interceptor i3
  211. a1.sources.r1.interceptors.i3.type=com.easou.flume.interceptor.ServerTimeInterceptor$Builder
  212. a1.sources.r1.interceptors.i3.isServerTime=true
  213. a1.sources.r1.interceptors.i3.timeType=yyyyMMddHHmmss
  214. #配置 interceptor i4
  215. a1.sources.r1.interceptors.i4.type = timestamp
  216. a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
  217. a1.channels.c1.kafka.bootstrap.servers = 10.26.27.71:8092,10.26.22.102:8092,10.26.22.120:8092,10.26.22.121:8092,10.26.22.122:8092
  218. a1.channels.c1.kafka.topic = yuewen_buydetail2
  219. # 开启kafka幂等性,保证重试不会产生数据重复
  220. a1.channels.c1.kafka.producer.enable.idempotence = true
  221. a1.channels.c1.kafka.producer.acks = all
  222. a1.channels.c1.kafka.producer.max.in.flight.requests.per.connection = 1
  223. # 重试 20次,每次超时30秒,一次发送数据最长约10分钟。尽量保证kafka不报异常,因为异常后flumesource会重试,导致数据重复
  224. a1.channels.c1.kafka.producer.retries = 20
  225. a1.channels.c1.kafka.producer.request.timeout.ms = 30000
  226. a1.channels.c1.kafka.consumer.group.id = integral2kafka
  227. #####################
  228. a1.sources.r2.type = com.github.ningg.flume.source.SpoolDirectoryTailFileSource
  229. a1.sources.r2.channels = c2
  230. a1.sources.r2.spoolDir = /data/logs/app/
  231. a1.sources.r2.fileSuffix = .COMPLETED
  232. a1.sources.r2.deletePolicy = never
  233. a1.sources.r2.ignorePattern = ^$
  234. a1.sources.r2.targetPattern = ywbuy_expending.*(\\d){10}.*
  235. a1.sources.r2.targetFilename = yyyyMMddHH
  236. a1.sources.r2.trackerDir = .flumespooltail_expending
  237. a1.sources.r2.consumeOrder = oldest
  238. a1.sources.r2.bufferMaxLineLength = 500000
  239. a1.sources.r2.inputCharset = UTF-8
  240. a1.sources.r2.decodeErrorPolicy = REPLACE
  241. a1.sources.r2.deserializer = LINE
  242. a1.sources.r2.interceptors = i1 i2 i3 i4
  243. #配置 interceptor i1
  244. a1.sources.r2.interceptors.i1.type = regex_filter
  245. a1.sources.r2.interceptors.i1.regex = ^\\s*$
  246. a1.sources.r2.interceptors.i1.excludeEvents = true
  247. #配置 interceptor i2
  248. a1.sources.r2.fileHeader = true
  249. a1.sources.r2.basenameHeader = true
  250. a1.sources.r2.interceptors.i2.type = com.easou.flume.interceptor.RegexExtractorExtInterceptor$Builder
  251. a1.sources.r2.interceptors.i2.regex = (.*)\\.(.*)\\.(.*)
  252. a1.sources.r2.interceptors.i2.extractorHeader = true
  253. a1.sources.r2.interceptors.i2.extractorHeaderKey = basename
  254. a1.sources.r2.interceptors.i2.serializers = s1 s2 s3
  255. a1.sources.r2.interceptors.i2.serializers.s1.name = log_type
  256. a1.sources.r2.interceptors.i2.serializers.s2.name = file_type
  257. a1.sources.r2.interceptors.i2.serializers.s3.name = log_data
  258. #配置 interceptor i3
  259. a1.sources.r2.interceptors.i3.type=com.easou.flume.interceptor.ServerTimeInterceptor$Builder
  260. a1.sources.r2.interceptors.i3.isServerTime=true
  261. a1.sources.r2.interceptors.i3.timeType=yyyyMMddHHmmss
  262. #配置 interceptor i4
  263. a1.sources.r2.interceptors.i4.type = timestamp
  264. a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
  265. a1.channels.c2.kafka.bootstrap.servers = 10.26.27.71:8092,10.26.22.102:8092,10.26.22.120:8092,10.26.22.121:8092,10.26.22.122:8092
  266. a1.channels.c2.kafka.topic = yuewen_buydetail2
  267. # 开启kafka幂等性,保证重试不会产生数据重复
  268. a1.channels.c2.kafka.producer.enable.idempotence = true
  269. a1.channels.c2.kafka.producer.acks = all
  270. a1.channels.c2.kafka.producer.max.in.flight.requests.per.connection = 1
  271. # 重试 20次,每次超时30秒,一次发送数据最长约10分钟。尽量保证kafka不报异常,因为异常后flumesource会重试,导致数据重复
  272. a1.channels.c2.kafka.producer.retries = 20
  273. a1.channels.c2.kafka.producer.request.timeout.ms = 30000
  274. a1.channels.c2.kafka.consumer.group.id = integral2kafka
  275. # a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
  276. # a1.channels.c1.kafka.bootstrap.servers = 10.26.27.71:8092,10.26.22.102:8092,10.26.22.120:8092,10.26.22.121:8092,10.26.22.122:8092
  277. # a1.channels.c1.kafka.topic = yuewen_new_buydetail
  278. # a1.channels.c1.kafka.consumer.group.id = flume2kafka
  279. #
  280. # # Define an Avro source called r1 on a1 and tell it
  281. # # to bind to 0.0.0.0:41414. Connect it to channel c1.
  282. # #注意这里分出2个channel出来,一个是到k1,一个到k2
  283. # #如果这里只出一个channel c1的话,下面2个sink channel都为c1,则消息就会随机分到2个sink上,而不是2个sink都有一份
  284. # a1.sources.r1.channels = c1
  285. # a1.sources.r1.type = exec
  286. # a1.sources.r1.command = tail -F /data/logs/app/access.log
  287. # a1.sources.r1.restart = true
  288. # a1.sources.r1.selector.optional = c1
  289. # a1.sources.r1.batchSize = 100000
  290. # a1.sources.r1.interceptors = i1 i2 i3
  291. # a1.sources.r1.interceptors.i1.type = timestamp
  292. # a1.sources.r1.interceptors.i2.type = host
  293. # a1.sources.r1.interceptors.i2.useIP = false
  294. # a1.sources.r1.interceptors.i3.type = regex_filter
  295. # a1.sources.r1.interceptors.i3.regex = ^\\s*$
  296. # a1.sources.r1.interceptors.i3.excludeEvents = true
  297. # #默认是replicationg,还有multiplexer
  298. # #a1.sources.r1.selector.type = replicationg
  299. #
  300. # # Finally, now that we've defined all of our components, tell
  301. # # a1 which ones we want to activate.
  302. # a1.sources = r1
  303. # a1.channels = c1
  304. # a1.sinks =