deploy-tmp-logstash.yaml 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: pro
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. auto_offset_reset => "latest"
  14. topics => ["offline-process-topic"]
  15. codec => json
  16. }
  17. }
  18. filter {
  19. mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
  20. grok{
  21. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
  22. }
  23. date {
  24. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  25. }
  26. }
  27. output {
  28. # stdout { codec => rubydebug }
  29. if [log_topic] == "offline-process-charge" {
  30. elasticsearch {
  31. hosts => ["elasticsearch:9200"]
  32. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  33. template_overwrite => true
  34. }
  35. }
  36. if [log_topic] == "offline-process-charge-callback" {
  37. elasticsearch {
  38. hosts => ["elasticsearch:9200"]
  39. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  40. template_overwrite => true
  41. }
  42. }
  43. if [log_topic] == "offline-process-promotion" {
  44. elasticsearch {
  45. hosts => ["elasticsearch:9200"]
  46. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  47. template_overwrite => true
  48. }
  49. }
  50. if [log_topic] == "offline-process-newmedia" {
  51. elasticsearch {
  52. hosts => ["elasticsearch:9200"]
  53. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  54. template_overwrite => true
  55. }
  56. }
  57. if [log_topic] == "offline-process-schedule" {
  58. elasticsearch {
  59. hosts => ["elasticsearch:9200"]
  60. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  61. template_overwrite => true
  62. }
  63. }
  64. if [log_topic] == "offline-process-statistics" {
  65. elasticsearch {
  66. hosts => ["elasticsearch:9200"]
  67. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  68. template_overwrite => true
  69. }
  70. }
  71. }
  72. logstash_yml: |
  73. http.host: "0.0.0.0"
  74. ---
  75. kind: Deployment
  76. apiVersion: apps/v1
  77. metadata:
  78. name: logstash-k8s-named
  79. namespace: pro
  80. labels:
  81. app: logstash-k8s-named
  82. spec:
  83. replicas: 1
  84. selector:
  85. matchLabels:
  86. app: logstash-k8s-named
  87. template:
  88. metadata:
  89. labels:
  90. app: logstash-k8s-named
  91. spec:
  92. containers:
  93. - name: logstash-k8s-named
  94. image: logstash:7.13.3
  95. command: ["/bin/sh","-c"]
  96. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
  97. ports:
  98. - containerPort: 5000
  99. name: client1
  100. - containerPort: 9600
  101. name: api
  102. volumeMounts:
  103. - name: vm-config
  104. mountPath: /usr/share/logstash/config
  105. volumes:
  106. - name: vm-config
  107. configMap:
  108. name: logstash-config
  109. items:
  110. - key: logstash-config-named-k8s
  111. path: logstash.conf
  112. ---
  113. kind: Service
  114. apiVersion: v1
  115. metadata:
  116. labels:
  117. app: logstash
  118. name: logstash
  119. namespace: pro
  120. spec:
  121. ports:
  122. - name: http
  123. port: 5000
  124. protocol: TCP
  125. targetPort: 5000
  126. selector:
  127. app: logstash