deploy-tmp-logstash.yaml 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: pro
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. auto_offset_reset => "latest"
  14. consumer_threads => 5
  15. max_partition_fetch_bytes => 4194304
  16. receive_buffer_bytes => 131072
  17. send_buffer_bytes => 524288
  18. topics => ["offline-process-topic"]
  19. codec => json
  20. }
  21. }
  22. filter {
  23. mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
  24. grok{
  25. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"}
  26. overwrite => ["message"]
  27. }
  28. date {
  29. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  30. }
  31. }
  32. output {
  33. stdout { codec => rubydebug }
  34. if [log_topic] == "offline-process-charge" {
  35. elasticsearch {
  36. hosts => ["elasticsearch:9200"]
  37. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  38. template_overwrite => true
  39. }
  40. }
  41. if [log_topic] == "offline-process-charge-callback" {
  42. elasticsearch {
  43. hosts => ["elasticsearch:9200"]
  44. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  45. template_overwrite => true
  46. }
  47. }
  48. if [log_topic] == "offline-process-promotion" {
  49. elasticsearch {
  50. hosts => ["elasticsearch:9200"]
  51. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  52. template_overwrite => true
  53. }
  54. }
  55. if [log_topic] == "offline-process-newmedia" {
  56. elasticsearch {
  57. hosts => ["elasticsearch:9200"]
  58. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  59. template_overwrite => true
  60. }
  61. }
  62. if [log_topic] == "offline-process-schedule" {
  63. elasticsearch {
  64. hosts => ["elasticsearch:9200"]
  65. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  66. template_overwrite => true
  67. }
  68. }
  69. if [log_topic] == "offline-process-statistics" {
  70. elasticsearch {
  71. hosts => ["elasticsearch:9200"]
  72. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  73. template_overwrite => true
  74. }
  75. }
  76. }
  77. logstash_yml: |
  78. http.host: "0.0.0.0"
  79. ---
  80. kind: Deployment
  81. apiVersion: apps/v1
  82. metadata:
  83. name: logstash-k8s-named
  84. namespace: pro
  85. labels:
  86. app: logstash-k8s-named
  87. spec:
  88. replicas: 3
  89. selector:
  90. matchLabels:
  91. app: logstash-k8s-named
  92. template:
  93. metadata:
  94. labels:
  95. app: logstash-k8s-named
  96. spec:
  97. containers:
  98. - name: logstash-k8s-named
  99. image: logstash:7.13.3
  100. command: ["/bin/sh","-c"]
  101. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
  102. ports:
  103. - containerPort: 5000
  104. name: client1
  105. - containerPort: 9600
  106. name: api
  107. volumeMounts:
  108. - name: vm-config
  109. mountPath: /usr/share/logstash/config
  110. volumes:
  111. - name: vm-config
  112. configMap:
  113. name: logstash-config
  114. items:
  115. - key: logstash-config-named-k8s
  116. path: logstash.conf
  117. ---
  118. kind: Service
  119. apiVersion: v1
  120. metadata:
  121. labels:
  122. app: logstash
  123. name: logstash
  124. namespace: pro
  125. spec:
  126. ports:
  127. - name: http
  128. port: 5000
  129. protocol: TCP
  130. targetPort: 5000
  131. selector:
  132. app: logstash