|
@@ -11,7 +11,7 @@ data:
|
|
|
bootstrap_servers => ["kafka-0.kafka-headless.pro.svc.cluster.local:9093,kafka-1.kafka-headless.pro.svc.cluster.local:9093,kafka-2.kafka-headless.pro.svc.cluster.local:9093"]
|
|
|
group_id => "es-test"
|
|
|
auto_offset_reset => "latest"
|
|
|
- consumer_threads => 10
|
|
|
+ consumer_threads => 5
|
|
|
max_partition_fetch_bytes => 4194304
|
|
|
receive_buffer_bytes => 131072
|
|
|
send_buffer_bytes => 524288
|
|
@@ -21,9 +21,10 @@ data:
|
|
|
}
|
|
|
|
|
|
filter {
|
|
|
- mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
|
|
|
+ mutate { remove_field => [ "[input][type]", "[prospector][type]", "[beat][name]", "[beat][hostname]", "source", "_score" ] }
|
|
|
grok{
|
|
|
- match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
|
|
|
+ match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:biz_id}*\] \[%{NUMBER:uid}*\] \[%{WORD:udid}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:message}"}
|
|
|
+ overwrite => ["message"]
|
|
|
}
|
|
|
date {
|
|
|
match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
|
|
@@ -31,7 +32,7 @@ data:
|
|
|
}
|
|
|
|
|
|
output {
|
|
|
- # stdout { codec => rubydebug }
|
|
|
+ stdout { codec => rubydebug }
|
|
|
if [log_topic] == "offline-process-charge" {
|
|
|
elasticsearch {
|
|
|
hosts => ["elasticsearch:9200"]
|