deploy-tmp-logstash.yaml 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: pro
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. auto_offset_reset => "latest"
  14. consumer_threads => 5
  15. max_partition_fetch_bytes => 4194304
  16. receive_buffer_bytes => 131072
  17. send_buffer_bytes => 524288
  18. topics => ["offline-process-topic"]
  19. codec => json
  20. }
  21. }
  22. filter {
  23. mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
  24. grok{
  25. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"}
  26. overwrite => ["message"]
  27. }
  28. date {
  29. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  30. }
  31. }
  32. output {
  33. stdout { codec => rubydebug }
  34. if [log_topic] == "offline-process-charge" {
  35. elasticsearch {
  36. hosts => ["elasticsearch:9200"]
  37. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  38. template_overwrite => true
  39. }
  40. }
  41. if [log_topic] == "offline-process-charge-callback" {
  42. elasticsearch {
  43. hosts => ["elasticsearch:9200"]
  44. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  45. template_overwrite => true
  46. }
  47. }
  48. if [log_topic] == "offline-process-promotion" {
  49. elasticsearch {
  50. hosts => ["elasticsearch:9200"]
  51. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  52. template_overwrite => true
  53. }
  54. }
  55. if [log_topic] == "offline-process-newmedia" {
  56. elasticsearch {
  57. hosts => ["elasticsearch:9200"]
  58. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  59. template_overwrite => true
  60. }
  61. }
  62. if [log_topic] == "offline-process-schedule" {
  63. elasticsearch {
  64. hosts => ["elasticsearch:9200"]
  65. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  66. template_overwrite => true
  67. }
  68. }
  69. if [log_topic] == "offline-process-statistics" {
  70. elasticsearch {
  71. hosts => ["elasticsearch:9200"]
  72. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  73. template_overwrite => true
  74. }
  75. }
  76. }
  77. logstash_yml: |
  78. http.host: "0.0.0.0"
  79. jvm-options: |
  80. -Xms4g
  81. -Xmx4g
  82. -XX:+UseConcMarkSweepGC
  83. -XX:CMSInitiatingOccupancyFraction=75
  84. -XX:+UseCMSInitiatingOccupancyOnly
  85. -Djava.awt.headless=true
  86. -Dfile.encoding=UTF-8
  87. -Djruby.compile.invokedynamic=true
  88. -Djruby.jit.threshold=0
  89. -Djruby.regexp.interruptible=true
  90. -XX:+HeapDumpOnOutOfMemoryError
  91. -Djava.security.egd=file:/dev/urandom
  92. -Dlog4j2.isThreadContextMapInheritable=true
  93. ---
  94. kind: Deployment
  95. apiVersion: apps/v1
  96. metadata:
  97. name: logstash-k8s-named
  98. namespace: pro
  99. labels:
  100. app: logstash-k8s-named
  101. spec:
  102. replicas: 1
  103. selector:
  104. matchLabels:
  105. app: logstash-k8s-named
  106. template:
  107. metadata:
  108. labels:
  109. app: logstash-k8s-named
  110. spec:
  111. containers:
  112. - name: logstash-k8s-named
  113. image: logstash:7.13.3
  114. command: ["/bin/sh","-c"]
  115. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config"]
  116. ports:
  117. - containerPort: 5000
  118. name: client1
  119. - containerPort: 9600
  120. name: api
  121. volumeMounts:
  122. - name: core-config
  123. mountPath: /usr/share/logstash/config
  124. - name: jvm-config
  125. mountPath: /usr/share/logstash/config
  126. volumes:
  127. - name: jvm-config
  128. configMap:
  129. name: logstash-config
  130. items:
  131. - key: jvm-options
  132. path: jvm.options
  133. - name: core-config
  134. configMap:
  135. name: logstash-config
  136. items:
  137. - key: logstash-config-named-k8s
  138. path: logstash.conf
  139. ---
  140. kind: Service
  141. apiVersion: v1
  142. metadata:
  143. labels:
  144. app: logstash
  145. name: logstash
  146. namespace: pro
  147. spec:
  148. ports:
  149. - name: http
  150. port: 5000
  151. protocol: TCP
  152. targetPort: 5000
  153. selector:
  154. app: logstash