浏览代码

dsfsadfasdfsdfasdf

yangxg 4 年之前
父节点
当前提交
642ab09e1b
共有 4 个文件被更改,包括 53 次插入5 次删除
  1. 7 4
      deploy-tmp-logstash.yaml
  2. 22 0
      logstash.conf
  3. 21 0
      logtest.txt
  4. 3 1
      offline-process-charge.yml

+ 7 - 4
deploy-tmp-logstash.yaml

@@ -16,16 +16,19 @@ data:
     }
     
     filter {
-      grok {
-        match => ["message", "(?<timestamp>%{MONTHDAY}-%{MONTH}-%{YEAR} %{TIME}) queries: client %{IPV4:c_ip}#%{NUMBER:c_port}: query: %{NOTSPACE:queryrec} %{NOTSPACE:dnsclass} %{NOTSPACE:dnstype} \+ \(%{IPV4:dnsbind}\)"]
+      grok{
+          match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:thread_trace_id}*\] \[%{WORD:biz_id}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
+      }
+      date {
+          match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss.SSS" ]
       }
     }
 
     output {
-      #stdout { codec => rubydebug }
+      stdout { codec => rubydebug }
       elasticsearch {
         hosts => ["elasticsearch:9200"]
-        index => "logstash-filebeat-%{+YYYY.MM.dd}"
+        index => "logstash-filebeat-%{+YYYY.MM}"
       }
     }
   logstash_yml: |

+ 22 - 0
logstash.conf

@@ -0,0 +1,22 @@
+input {
+    stdin {
+        codec => multiline {
+            pattern => "^\["
+            negate => true
+            what => "previous"
+        }
+    }
+}
+filter{
+    grok{
+        match => {"message" => "(?m)^\[%{DATA:logdate}\] \[%{WORD:thread_trace_id}*\] \[%{WORD:biz_id}*\] \[%{LOGLEVEL:level}%{SPACE}*\] \[%{DATA:threadName}\] \[%{DATA:loggerName}\] \[%{DATA:method_line}\] - %{GREEDYDATA:msg}"}
+    }
+    date{
+        match => ["logdate","yyyy-MM-dd HH:mm:ss.SSS"]
+    }
+}
+output {
+    stdout {
+    }
+}
+

+ 21 - 0
logtest.txt

@@ -0,0 +1,21 @@
+[2021-06-25 10:20:37.761] [d5f4eb4ddbef415aad5ee2700460a164] [56e677441880481f990616c064dd7ed1] [INFO ] [POOL-T-1 :1] [c.e.i.v.o.service.impl.charge.AndroidChargeService] [updateToDB:573] - update ChargeStatus end the invoice:d6638a2ed73a407b9ce1ce3ede46cf26
+[2021-06-26 18:03:39.759] [] [] [WARN ] [ActiveMQ Transport: tcp:///10.26.29.47:61618@57180] [o.a.activemq.transport.failover.FailoverTransport] [handleTransportFailure:280] - Transport (tcp://10.26.29.47:61618) failed , attempting to automatically reconnect: {}
+java.net.SocketException: Connection reset
+        at java.net.SocketInputStream.read(SocketInputStream.java:209)
+        at java.net.SocketInputStream.read(SocketInputStream.java:141)
+        at org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50)
+        at org.apache.activemq.transport.tcp.TcpTransport$2.fill(TcpTransport.java:634)
+        at org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:59)
+        at org.apache.activemq.transport.tcp.TcpTransport$2.read(TcpTransport.java:619)
+        at java.io.DataInputStream.readInt(DataInputStream.java:387)
+        at org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:268)
+        at org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:240)
+        at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:232)
+        at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:215)
+        at java.lang.Thread.run(Thread.java:745)
+[2021-06-26 18:36:57.130] [95b7984b0fae4ecbbb58482a0c8606f1] [] [INFO ] [POOL-T-0 :1] [c.e.n.v.o.framework.core.monitor.MonitorService] [sendMsg:69] - result null
+[2021-06-26 22:55:39.194] [] [] [DEBUG] [scheduling-1] [druid.sql.Statement] [statementLog:134] - {conn-10002, pstmt-20002} executed. 1.14746 millis. select
+
+        id, app_id, cp_id,create_time
+
+        from account_free_cpid

+ 3 - 1
offline-process-charge.yml

@@ -162,13 +162,15 @@ data:
     setup.template.pattern: offline-process-charge*
     setup.dashboards.enabled: true
     output.kafka:
+      version: 0.10.2.1
+      enable: true
       # initial brokers for reading cluster metadata
       hosts: ["kafka-0.kafka-svc.qa.svc.cluster.local:9093", "kafka-1.kafka-svc.qa.svc.cluster.local:9093", "kafka-2.kafka-svc.qa.svc.cluster.local:9093"]
       # message topic selection + partitioning
       topic: 'offline-process-charge-topic'
       partition.round_robin:
         reachable_only: false
-      required_acks: 1
+      required_acks: 0
       compression: gzip
       max_message_bytes: 1000000
     setup.kibana: