deploy-tmp-logstash.yaml 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: qa
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-svc.qa.svc.cluster.local:9093,kafka-1.kafka-svc.qa.svc.cluster.local:9093,kafka-2.kafka-svc.qa.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. topics => ["offline-process-topic"]
  14. codec => json
  15. }
  16. }
  17. filter {
  18. grok{
  19. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:thread_trace_id}*\] \[%{WORD:biz_id}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
  20. }
  21. date {
  22. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  23. }
  24. }
  25. output {
  26. stdout { codec => rubydebug }
  27. if [fields][log_topic] == "offline-process-charge" {
  28. elasticsearch {
  29. hosts => ["elasticsearch:9200"]
  30. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  31. template_overwrite => true
  32. }
  33. }
  34. if [fields][log_topic] == "offline-process-charge-callback" {
  35. elasticsearch {
  36. hosts => ["elasticsearch:9200"]
  37. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  38. template_overwrite => true
  39. }
  40. }
  41. if [fields][log_topic] == "offline-process-promotion" {
  42. elasticsearch {
  43. hosts => ["elasticsearch:9200"]
  44. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  45. template_overwrite => true
  46. }
  47. }
  48. if [fields][log_topic] == "offline-process-newmedia" {
  49. elasticsearch {
  50. hosts => ["elasticsearch:9200"]
  51. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  52. template_overwrite => true
  53. }
  54. }
  55. if [fields][log_topic] == "offline-process-schedule" {
  56. elasticsearch {
  57. hosts => ["elasticsearch:9200"]
  58. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  59. template_overwrite => true
  60. }
  61. }
  62. if [fields][log_topic] == "offline-process-statistics" {
  63. elasticsearch {
  64. hosts => ["elasticsearch:9200"]
  65. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  66. template_overwrite => true
  67. }
  68. }
  69. }
  70. logstash_yml: |
  71. http.host: "0.0.0.0"
  72. ---
  73. kind: Deployment
  74. apiVersion: apps/v1
  75. metadata:
  76. name: logstash-k8s-named
  77. namespace: qa
  78. labels:
  79. app: logstash-k8s-named
  80. spec:
  81. replicas: 1
  82. selector:
  83. matchLabels:
  84. app: logstash-k8s-named
  85. template:
  86. metadata:
  87. labels:
  88. app: logstash-k8s-named
  89. spec:
  90. containers:
  91. - name: logstash-k8s-named
  92. image: logstash:7.6.2
  93. command: ["/bin/sh","-c"]
  94. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
  95. ports:
  96. - containerPort: 5000
  97. name: client1
  98. - containerPort: 9600
  99. name: api
  100. volumeMounts:
  101. - name: vm-config
  102. mountPath: /usr/share/logstash/config
  103. volumes:
  104. - name: vm-config
  105. configMap:
  106. name: logstash-config
  107. items:
  108. - key: logstash-config-named-k8s
  109. path: logstash.conf
  110. ---
  111. kind: Service
  112. apiVersion: v1
  113. metadata:
  114. labels:
  115. app: logstash
  116. name: logstash
  117. namespace: qa
  118. spec:
  119. ports:
  120. - name: http
  121. port: 5000
  122. protocol: TCP
  123. targetPort: 5000
  124. selector:
  125. app: logstash