flume.conf 2.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152
  1. a1.channels.c1.type = file
  2. a1.channels.c1.checkpointDir = /data/flume_data/checkpoint
  3. a1.channels.c1.dataDirs = /data/flume_data/data
  4. a1.channels.c2.type = file
  5. a1.channels.c2.checkpointDir = /data/flume_data1/checkpoint
  6. a1.channels.c2.dataDirs = /data/flume_data1/data
  7. a1.channels.c2.transactionCapacity = 200000
  8. a1.channels.c2.capacity = 1000000
  9. a1.channels.c2.write-timeout = 250
  10. a1.channels.c2.keep-alive = 250
  11. a1.channels.c3.type = org.apache.flume.channel.kafka.KafkaChannel
  12. #a1.channels.c3.transactionCapacity = 100000
  13. #a1.channels.c3.capacity = 10000000
  14. a1.channels.c3.brokerList = 10.26.27.212:8092,10.26.22.76:8092,10.26.22.72:8092,10.26.22.73:8092,10.26.22.74:8092
  15. a1.channels.c3.topic = home_easou_com_nginx
  16. a1.channels.c3.zookeeperConnect = 10.26.27.212:2181,10.26.22.76:2181,10.26.22.72:2181,10.26.22.73:2181,10.26.22.74:2181
  17. #a1.channels.c3.write-timeout = 150
  18. #a1.channels.c3.keep-alive = 260
  19. # Define an Avro source called r1 on a1 and tell it
  20. # to bind to 0.0.0.0:41414. Connect it to channel c1.
  21. #注意这里分出2个channel出来,一个是到k1,一个到k2
  22. #如果这里只出一个channel c1的话,下面2个sink channel都为c1,则消息就会随机分到2个sink上,而不是2个sink都有一份
  23. a1.sources.r1.channels = c1 c2 c3
  24. a1.sources.r1.type = exec
  25. a1.sources.r1.command = tail -F /log/nginx/visit/access.log
  26. a1.sources.r1.restart = true
  27. a1.sources.r1.interceptors = i1 i2
  28. a1.sources.r1.interceptors.i1.type = timestamp
  29. a1.sources.r1.interceptors.i2.type = host
  30. a1.sources.r1.interceptors.i2.useIP = false
  31. #默认是replicationg,还有multiplexer
  32. #a1.sources.r1.selector.type = replicationg
  33. a1.sinks.k1.type = avro
  34. a1.sinks.k1.channel = c1
  35. a1.sinks.k1.hostname = 10.26.26.67
  36. a1.sinks.k1.port = 41414
  37. a1.sinks.k2.type = avro
  38. a1.sinks.k2.channel = c2
  39. a1.sinks.k2.hostname = 10.26.33.113
  40. a1.sinks.k2.port = 41414
  41. a1.sinks.k2.request-timeout = 100000
  42. a1.sinks.k2.connect-timeout = 100000
  43. # Finally, now that we've defined all of our components, tell
  44. # a1 which ones we want to activate.
  45. a1.sources = r1
  46. a1.channels = c1 c2 c3
  47. a1.sinks = k1 k2