deploy-tmp-logstash.yaml 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: qa
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-svc.qa.svc.cluster.local:9093,kafka-1.kafka-svc.qa.svc.cluster.local:9093,kafka-2.kafka-svc.qa.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. topics => ["offline-process-topic"]
  14. codec => json
  15. }
  16. }
  17. filter {
  18. grok{
  19. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:thread_trace_id}*\] \[%{WORD:biz_id}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
  20. }
  21. mutate {
  22. remove_field => ["@version","beat.name","input.type"]
  23. }
  24. date {
  25. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  26. }
  27. }
  28. output {
  29. stdout { codec => rubydebug }
  30. if [fields][log_topic] == "offline-process-charge" {
  31. elasticsearch {
  32. hosts => ["elasticsearch:9200"]
  33. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  34. template_overwrite => true
  35. }
  36. }
  37. if [fields][log_topic] == "offline-process-charge-callback" {
  38. elasticsearch {
  39. hosts => ["elasticsearch:9200"]
  40. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  41. template_overwrite => true
  42. }
  43. }
  44. if [fields][log_topic] == "offline-process-promotion" {
  45. elasticsearch {
  46. hosts => ["elasticsearch:9200"]
  47. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  48. template_overwrite => true
  49. }
  50. }
  51. if [fields][log_topic] == "offline-process-newmedia" {
  52. elasticsearch {
  53. hosts => ["elasticsearch:9200"]
  54. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  55. template_overwrite => true
  56. }
  57. }
  58. if [fields][log_topic] == "offline-process-schedule" {
  59. elasticsearch {
  60. hosts => ["elasticsearch:9200"]
  61. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  62. template_overwrite => true
  63. }
  64. }
  65. if [fields][log_topic] == "offline-process-statistics" {
  66. elasticsearch {
  67. hosts => ["elasticsearch:9200"]
  68. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  69. template_overwrite => true
  70. }
  71. }
  72. }
  73. logstash_yml: |
  74. http.host: "0.0.0.0"
  75. ---
  76. kind: Deployment
  77. apiVersion: apps/v1
  78. metadata:
  79. name: logstash-k8s-named
  80. namespace: qa
  81. labels:
  82. app: logstash-k8s-named
  83. spec:
  84. replicas: 1
  85. selector:
  86. matchLabels:
  87. app: logstash-k8s-named
  88. template:
  89. metadata:
  90. labels:
  91. app: logstash-k8s-named
  92. spec:
  93. containers:
  94. - name: logstash-k8s-named
  95. image: logstash:7.6.2
  96. command: ["/bin/sh","-c"]
  97. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
  98. ports:
  99. - containerPort: 5000
  100. name: client1
  101. - containerPort: 9600
  102. name: api
  103. volumeMounts:
  104. - name: vm-config
  105. mountPath: /usr/share/logstash/config
  106. volumes:
  107. - name: vm-config
  108. configMap:
  109. name: logstash-config
  110. items:
  111. - key: logstash-config-named-k8s
  112. path: logstash.conf
  113. ---
  114. kind: Service
  115. apiVersion: v1
  116. metadata:
  117. labels:
  118. app: logstash
  119. name: logstash
  120. namespace: qa
  121. spec:
  122. ports:
  123. - name: http
  124. port: 5000
  125. protocol: TCP
  126. targetPort: 5000
  127. selector:
  128. app: logstash