deploy-tmp-logstash.yaml 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-configmap
  6. namespace: pro
  7. data:
  8. logstash.yml: |
  9. http.host: "0.0.0.0"
  10. logstash.conf: |
  11. input {
  12. kafka {
  13. bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
  14. group_id => "es-test"
  15. auto_offset_reset => "latest"
  16. consumer_threads => 3
  17. max_partition_fetch_bytes => 4194304
  18. receive_buffer_bytes => 131072
  19. send_buffer_bytes => 524288
  20. topics => ["offline-process-topic"]
  21. codec => json
  22. }
  23. }
  24. filter {
  25. mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score", "@version", "_id", "[agent][type]", "[agent][id]", "[agent][name]", "[agent][version]", "[agent][ephemeral_id]", "[ecs][version]" ]}
  26. grok{
  27. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"}
  28. overwrite => ["message"]
  29. }
  30. date {
  31. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  32. }
  33. }
  34. output {
  35. # stdout { codec => rubydebug }
  36. if [log_topic] == "offline-process-charge" {
  37. elasticsearch {
  38. hosts => ["10.26.27.137:9200"]
  39. index => "offline-process-charge-%{+YYYY.MM}"
  40. template_overwrite => true
  41. }
  42. }
  43. if [log_topic] == "offline-process-charge-callback" {
  44. elasticsearch {
  45. hosts => ["10.26.27.137:9200"]
  46. index => "offline-process-charge-callback-%{+YYYY.MM}"
  47. template_overwrite => true
  48. }
  49. }
  50. if [log_topic] == "offline-process-promotion" {
  51. elasticsearch {
  52. hosts => ["10.26.27.137:9200"]
  53. index => "offline-process-promotion-%{+YYYY.MM}"
  54. template_overwrite => true
  55. }
  56. }
  57. if [log_topic] == "offline-process-schedule" {
  58. elasticsearch {
  59. hosts => ["10.26.27.137:9200"]
  60. index => "offline-process-schedule-%{+YYYY.MM}"
  61. template_overwrite => true
  62. }
  63. }
  64. }
  65. ---
  66. kind: Deployment
  67. apiVersion: apps/v1
  68. metadata:
  69. name: logstash-k8s-named
  70. namespace: pro
  71. labels:
  72. app: logstash-k8s-named
  73. spec:
  74. replicas: 3
  75. selector:
  76. matchLabels:
  77. app: logstash-k8s-named
  78. template:
  79. metadata:
  80. labels:
  81. app: logstash-k8s-named
  82. spec:
  83. containers:
  84. - name: logstash-k8s-named
  85. imagePullPolicy: IfNotPresent
  86. image: hub.evbj.easou.com/pro/logstash:7.13.3
  87. command: ["/bin/sh","-c"]
  88. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config"]
  89. ports:
  90. - containerPort: 5000
  91. name: client1
  92. - containerPort: 9600
  93. name: api
  94. volumeMounts:
  95. - name: config-volume
  96. mountPath: /usr/share/logstash/config
  97. - name: yml-volume
  98. mountPath: /usr/share/logstash/config
  99. env:
  100. - name: LS_HEAP_SIZE
  101. value: 4096m
  102. - name: LS_HTTP_HOST
  103. value: "0.0.0.0"
  104. volumes:
  105. - name: yml-volume
  106. configMap:
  107. name: logstash-configmap
  108. items:
  109. - key: logstash.yml
  110. path: logstash.yml
  111. - name: config-volume
  112. configMap:
  113. name: logstash-configmap
  114. items:
  115. - key: logstash.conf
  116. path: logstash.conf
  117. ---
  118. kind: Service
  119. apiVersion: v1
  120. metadata:
  121. labels:
  122. app: logstash
  123. name: logstash
  124. namespace: pro
  125. spec:
  126. ports:
  127. - name: http
  128. port: 5000
  129. protocol: TCP
  130. targetPort: 5000
  131. selector:
  132. app: logstash