deploy-tmp-logstash.yaml 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128
  1. ---
  2. kind: ConfigMap
  3. apiVersion: v1
  4. metadata:
  5. name: logstash-config
  6. namespace: pro
  7. data:
  8. logstash-config-named-k8s: |
  9. input {
  10. kafka {
  11. bootstrap_servers => ["kafka-0.kafka-svc.pro.svc.cluster.local:9093,kafka-1.kafka-svc.pro.svc.cluster.local:9093,kafka-2.kafka-svc.pro.svc.cluster.local:9093"]
  12. group_id => "es-test"
  13. topics => ["offline-process-topic"]
  14. codec => json
  15. }
  16. }
  17. filter {
  18. mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
  19. grok{
  20. match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
  21. }
  22. date {
  23. match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
  24. }
  25. }
  26. output {
  27. # stdout { codec => rubydebug }
  28. if [log_topic] == "offline-process-charge" {
  29. elasticsearch {
  30. hosts => ["elasticsearch:9200"]
  31. index => "logstash-offline-process-charge-%{+YYYY.MM}"
  32. template_overwrite => true
  33. }
  34. }
  35. if [log_topic] == "offline-process-charge-callback" {
  36. elasticsearch {
  37. hosts => ["elasticsearch:9200"]
  38. index => "logstash-offline-process-charge-callback-%{+YYYY.MM}"
  39. template_overwrite => true
  40. }
  41. }
  42. if [log_topic] == "offline-process-promotion" {
  43. elasticsearch {
  44. hosts => ["elasticsearch:9200"]
  45. index => "logstash-offline-process-promotion-%{+YYYY.MM}"
  46. template_overwrite => true
  47. }
  48. }
  49. if [log_topic] == "offline-process-newmedia" {
  50. elasticsearch {
  51. hosts => ["elasticsearch:9200"]
  52. index => "logstash-offline-process-newmedia-%{+YYYY.MM}"
  53. template_overwrite => true
  54. }
  55. }
  56. if [log_topic] == "offline-process-schedule" {
  57. elasticsearch {
  58. hosts => ["elasticsearch:9200"]
  59. index => "logstash-offline-process-schedule-%{+YYYY.MM}"
  60. template_overwrite => true
  61. }
  62. }
  63. if [log_topic] == "offline-process-statistics" {
  64. elasticsearch {
  65. hosts => ["elasticsearch:9200"]
  66. index => "logstash-offline-process-statistics-%{+YYYY.MM}"
  67. template_overwrite => true
  68. }
  69. }
  70. }
  71. logstash_yml: |
  72. http.host: "0.0.0.0"
  73. ---
  74. kind: Deployment
  75. apiVersion: apps/v1
  76. metadata:
  77. name: logstash-k8s-named
  78. namespace: pro
  79. labels:
  80. app: logstash-k8s-named
  81. spec:
  82. replicas: 1
  83. selector:
  84. matchLabels:
  85. app: logstash-k8s-named
  86. template:
  87. metadata:
  88. labels:
  89. app: logstash-k8s-named
  90. spec:
  91. containers:
  92. - name: logstash-k8s-named
  93. image: logstash:7.6.2
  94. command: ["/bin/sh","-c"]
  95. args: ["/usr/share/logstash/bin/logstash -f /usr/share/logstash/config/logstash.conf"]
  96. ports:
  97. - containerPort: 5000
  98. name: client1
  99. - containerPort: 9600
  100. name: api
  101. volumeMounts:
  102. - name: vm-config
  103. mountPath: /usr/share/logstash/config
  104. volumes:
  105. - name: vm-config
  106. configMap:
  107. name: logstash-config
  108. items:
  109. - key: logstash-config-named-k8s
  110. path: logstash.conf
  111. ---
  112. kind: Service
  113. apiVersion: v1
  114. metadata:
  115. labels:
  116. app: logstash
  117. name: logstash
  118. namespace: pro
  119. spec:
  120. ports:
  121. - name: http
  122. port: 5000
  123. protocol: TCP
  124. targetPort: 5000
  125. selector:
  126. app: logstash