Kaynağa Gözat

deploy qa/realtime-yuewen:r181373 阅文实时接口k8s日志收集

datard 2 yıl önce
ebeveyn
işleme
eaf7666190
3 değiştirilmiş dosya ile 76 ekleme ve 88 silme
  1. 6 6
      .drone.yml
  2. 1 1
      Dockerfile
  3. 69 81
      deploy-tmp.yaml

+ 6 - 6
.drone.yml

@@ -19,9 +19,9 @@ steps:
   pull: always
   # privileged: true
   environment:
-    JNLP_ENV: pro
-    JNLP_REPLICAS: 2
-    JNLP_TAG: r181368
+    JNLP_ENV: qa
+    JNLP_REPLICAS: 1
+    JNLP_TAG: r181373
     JNLP_VERSION: v1  # default v1, v2 for canary
     DEPLOY_ENV: k8s-1 # 可以选择把应用部署到集群:k8s-1 or k8s-2
     JNLP_REPO: hub.evbj.easou.com
@@ -54,8 +54,8 @@ steps:
     corpid: ww419ee4063735e1c0
     corp_secret: zpiRBLETH9eLwIMQ4eJ_r_dcm3BPSGeLHvTcft8Ot-M
     agent_id: 1000004
-    title: "Pipeline pro/realtime-yuewen:r181368 Success"
-    description: "${DRONE_BUILD_LINK} pro/realtime-yuewen:r181368 部署完成 阅文实时接口k8s日志收集"
+    title: "Pipeline qa/realtime-yuewen:r181373 Success"
+    description: "${DRONE_BUILD_LINK} qa/realtime-yuewen:r181373 部署完成 阅文实时接口k8s日志收集"
     msg_url: ${DRONE_BUILD_LINK}
     btn_txt: "否"
   when:
@@ -68,7 +68,7 @@ steps:
     corpid: ww419ee4063735e1c0
     corp_secret: zpiRBLETH9eLwIMQ4eJ_r_dcm3BPSGeLHvTcft8Ot-M
     agent_id: 1000004
-    title: "Pipeline pro/realtime-yuewen:r181368 Failure"
+    title: "Pipeline qa/realtime-yuewen:r181373 Failure"
     description: "${DRONE_BUILD_LINK} 部署失败,请检查配置!阅文实时接口k8s日志收集"
     msg_url: ${DRONE_BUILD_LINK}
     btn_txt: "否"

+ 1 - 1
Dockerfile

@@ -1 +1 @@
-FROM hub.evbj.easou.com/pro/realtime-yuewen:r181368
+FROM hub.evbj.easou.com/qa/realtime-yuewen:r181373

+ 69 - 81
deploy-tmp.yaml

@@ -3,12 +3,12 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: realtime-yuewen
-  namespace: pro
+  namespace: qa
   labels:
     app: realtime-yuewen
 spec:
   serviceName: realtime-yuewen
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: realtime-yuewen
@@ -33,21 +33,21 @@ spec:
               mountPath: /data
       #nodeSelector:
       #  app.touchrs: touchrs
-#      affinity:
-#        podAntiAffinity:
-#          preferredDuringSchedulingIgnoredDuringExecution:
-#          - weight: 100
-#            podAffinityTerm:
-#              labelSelector:
-#                matchExpressions:
-#                - key: app
-#                  operator: In
-#                  values:
-#                  - realtime-yuewen
-#              topologyKey: kubernetes.io/hostname
+      #      affinity:
+      #        podAntiAffinity:
+      #          preferredDuringSchedulingIgnoredDuringExecution:
+      #          - weight: 100
+      #            podAffinityTerm:
+      #              labelSelector:
+      #                matchExpressions:
+      #                - key: app
+      #                  operator: In
+      #                  values:
+      #                  - realtime-yuewen
+      #              topologyKey: kubernetes.io/hostname
       containers:
         - name: realtime-yuewen
-          image: hub.evbj.easou.com/pro/realtime-yuewen:r181368
+          image: hub.evbj.easou.com/qa/realtime-yuewen:r181373
           imagePullPolicy: IfNotPresent
           env:
             - name: MY_NODE_NAME
@@ -67,7 +67,7 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
             - name: JAVA_OPTS
-              value: "-server  -Xmx2g -Xms2g -Xss512k -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=product"
+              value: "-server  -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8001,suspend=n  -Xmx2g -Xms2g -Xss512k -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=test"
           ports:
             - containerPort: 8080
               name: port
@@ -99,38 +99,22 @@ spec:
               port: 8080
             initialDelaySeconds: 40
             periodSeconds: 15
-        - image: hub.evbj.easou.com/pro/fluent-bit
+        - image: hub.evbj.easou.com/dev/flume:1.7.0-20220930-v1
           imagePullPolicy: IfNotPresent
-          name: fluent-bit
-          ports:
-            - containerPort: 2020
-              name: metrics
-              protocol: TCP
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
+          name: flume
           volumeMounts:
             - name: volume #日志同时挂载在nginx和filebeat中
               mountPath: /data
-            - name: fluent-bit-config
-              mountPath: /opt/bitnami/fluent-bit/conf/fluent-bit.conf
-              subPath: fluent-bit.conf
-            - name: fluent-bit-config-parses
-              mountPath: /opt/bitnami/fluent-bit/conf/parsers.conf
-              subPath: parsers.conf
+            - name: flume-config
+              mountPath: /app/soft/flume-1.7.0/conf/flume.conf
+              subPath: flume.conf
       volumes:
-        - name: fluent-bit-config
-          configMap:
-            name: fluent-bit-config-realtime-yuewen
-            items:
-            - key: fluent-bit.conf
-              path: fluent-bit.conf
-        - name: fluent-bit-config-parses
+        - name: flume-config
           configMap:
-            name: fluent-bit-config-realtime-yuewen
+            name: flume-config-realtime-yuewen
             items:
-            - key: parsers.conf
-              path: parsers.conf
-
+              - key: flume.conf
+                path: flume.conf
 
   volumeClaimTemplates:
     - metadata:
@@ -146,7 +130,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: realtime-yuewen-svc
-  namespace: pro
+  namespace: qa
   labels:
     app: realtime-yuewen-svc
 spec:
@@ -162,7 +146,7 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: realtime-yuewen-ingress
-  namespace: pro
+  namespace: qa
 spec:
   rules:
     - host: ywrealtime.appeasou.com
@@ -175,50 +159,54 @@ spec:
                 name: realtime-yuewen-svc
                 port:
                   number: 8080
-    - host: pro-realtime-yuewen.ieasou.cn
+    - host: qa-realtime-yuewen.ieasou.cn
       http:
         paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: realtime-yuewen-svc
-                port:
-                  number: 8080
+            - path: /
+              pathType: Prefix
+              backend:
+                service:
+                  name: realtime-yuewen-svc
+                  port:
+                    number: 8080
   ingressClassName: nginx
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: fluent-bit-config-realtime-yuewen
-  namespace: pro
+  name: flume-config-realtime-yuewen
+  namespace: qa
 data:
-  fluent-bit.conf: |
-    [SERVICE]
-        Flush     5
-        Daemon    off
-        Log_Level info
-    [INPUT]
-        Name tail
-        Path /data/logs/app/access.log
-        Mem_Buf_Limit 5MB
-        Skip_Long_Lines On
-    [OUTPUT]
-        Name           kafka
-        Match          *
-        Brokers        10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
-        Topics         test_realtime_yuewen_nginx
-        Retry_Limit    false
-        Message_Key log
-        # hides errors "Receive failed: Disconnected" when kafka kills idle connections
-        rdkafka.log.connection.close false
-        # producer buffer is not included in http://fluentbit.io/documentation/0.12/configuration/memory_usage.html#estimating
-        rdkafka.queue.buffering.max.kbytes 10240
-        # for logs you'll probably want this ot be 0 or 1, not more
-        rdkafka.request.required.acks 1
+  flume.conf: |
+    a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
+    a1.channels.c1.kafka.bootstrap.servers = 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
+    a1.channels.c1.kafka.topic = realtime_yuewen_nginx
+    a1.channels.c1.kafka.consumer.group.id = flume2kafka
+    
+    # Define an Avro source called r1 on a1 and tell it
+    # to bind to 0.0.0.0:41414. Connect it to channel c1.
+    #注意这里分出2个channel出来,一个是到k1,一个到k2
+    #如果这里只出一个channel c1的话,下面2个sink channel都为c1,则消息就会随机分到2个sink上,而不是2个sink都有一份
+    a1.sources.r1.channels = c1
+    a1.sources.r1.type = exec
+    a1.sources.r1.command = tail -F /data/logs/app/access.log
+    a1.sources.r1.restart = true
+    a1.sources.r1.selector.optional = c1
+    a1.sources.r1.batchSize = 100000
+    a1.sources.r1.interceptors = i1 i2 i3
+    a1.sources.r1.interceptors.i1.type = timestamp
+    a1.sources.r1.interceptors.i2.type = host
+    a1.sources.r1.interceptors.i2.useIP = false
+    a1.sources.r1.interceptors.i3.type = regex_filter
+    a1.sources.r1.interceptors.i3.regex = ^\\s*$
+    a1.sources.r1.interceptors.i3.excludeEvents = true
+    #默认是replicationg,还有multiplexer
+    #a1.sources.r1.selector.type = replicationg
+    
+    # Finally, now that we've defined all of our components, tell
+    # a1 which ones we want to activate.
+    a1.sources = r1
+    a1.channels = c1
+    a1.sinks =
+
 
-  parsers.conf: |
-    [PARSER]
-        Name   multiline_pattern
-        Format regex
-        Regex  ^\\s*$