deploy-tmp-logstash.yaml 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: pro
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. auto_offset_reset => "latest"
  14. consumer_threads => 10
  15. max_partition_fetch_bytes => 4194304
  16. receive_buffer_bytes => 131072
  17. send_buffer_bytes => 524288
  18. topics => ["offline-process-topic"]
  19. codec => json
  20. }
  21. }
  22. filter {
  23. mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
  24. grok{
  25. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
  26. }
  27. date {
  28. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  29. }
  30. }
  31. output {
  32. # stdout { codec => rubydebug }
  33. if [log_topic] == "offline-process-charge" {
  34. elasticsearch {
  35. hosts => ["elasticsearch:9200"]
  36. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  37. template_overwrite => true
  38. }
  39. }
  40. if [log_topic] == "offline-process-charge-callback" {
  41. elasticsearch {
  42. hosts => ["elasticsearch:9200"]
  43. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  44. template_overwrite => true
  45. }
  46. }
  47. if [log_topic] == "offline-process-promotion" {
  48. elasticsearch {
  49. hosts => ["elasticsearch:9200"]
  50. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  51. template_overwrite => true
  52. }
  53. }
  54. if [log_topic] == "offline-process-newmedia" {
  55. elasticsearch {
  56. hosts => ["elasticsearch:9200"]
  57. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  58. template_overwrite => true
  59. }
  60. }
  61. if [log_topic] == "offline-process-schedule" {
  62. elasticsearch {
  63. hosts => ["elasticsearch:9200"]
  64. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  65. template_overwrite => true
  66. }
  67. }
  68. if [log_topic] == "offline-process-statistics" {
  69. elasticsearch {
  70. hosts => ["elasticsearch:9200"]
  71. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  72. template_overwrite => true
  73. }
  74. }
  75. }
  76. logstash_yml: |
  77. http.host: "0.0.0.0"
  78. ---
  79. kind: Deployment
  80. apiVersion: apps/v1
  81. metadata:
  82. name: logstash-k8s-named
  83. namespace: pro
  84. labels:
  85. app: logstash-k8s-named
  86. spec:
  87. replicas: 1
  88. selector:
  89. matchLabels:
  90. app: logstash-k8s-named
  91. template:
  92. metadata:
  93. labels:
  94. app: logstash-k8s-named
  95. spec:
  96. containers:
  97. - name: logstash-k8s-named
  98. image: logstash:7.13.3
  99. command: ["/bin/sh","-c"]
  100. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
  101. ports:
  102. - containerPort: 5000
  103. name: client1
  104. - containerPort: 9600
  105. name: api
  106. volumeMounts:
  107. - name: vm-config
  108. mountPath: /usr/share/logstash/config
  109. volumes:
  110. - name: vm-config
  111. configMap:
  112. name: logstash-config
  113. items:
  114. - key: logstash-config-named-k8s
  115. path: logstash.conf
  116. ---
  117. kind: Service
  118. apiVersion: v1
  119. metadata:
  120. labels:
  121. app: logstash
  122. name: logstash
  123. namespace: pro
  124. spec:
  125. ports:
  126. - name: http
  127. port: 5000
  128. protocol: TCP
  129. targetPort: 5000
  130. selector:
  131. app: logstash