flume.conf 3.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273
  1. a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
  2. a1.channels.c1.kafka.bootstrap.servers = 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
  3. a1.channels.c1.kafka.topic = api_searchlog_novel
  4. a1.channels.c1.kafka.consumer.group.id = flume2kafka
  5. # Define an Avro source called r1 on a1 and tell it
  6. # to bind to 0.0.0.0:41414. Connect it to channel c1.
  7. #注意这里分出2个channel出来,一个是到k1,一个到k2
  8. #如果这里只出一个channel c1的话,下面2个sink channel都为c1,则消息就会随机分到2个sink上,而不是2个sink都有一份
  9. a1.sources.r1.channels = c1
  10. a1.sources.r1.type = exec
  11. a1.sources.r1.command = tail -F /log/resin-books/words/access.log
  12. a1.sources.r1.restart = true
  13. a1.sources.r1.batchSize = 10000
  14. a1.sources.r1.interceptors = i1 i2 i3
  15. a1.sources.r1.interceptors.i1.type = timestamp
  16. a1.sources.r1.interceptors.i2.type = host
  17. a1.sources.r1.interceptors.i2.useIP = false
  18. a1.sources.r1.interceptors.i3.type = regex_filter
  19. a1.sources.r1.interceptors.i3.regex = ^\\s*$
  20. a1.sources.r1.interceptors.i3.excludeEvents = true
  21. #默认是replicationg,还有multiplexer
  22. #a1.sources.r1.selector.type = replicationg
  23. # Finally, now that we've defined all of our components, tell
  24. # a1 which ones we want to activate.
  25. a1.sources = r1 r2 r3
  26. a1.channels = c1 c2 c3
  27. a1.sinks =
  28. #########searchrecord apirs的BI统计搜索日志
  29. a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
  30. a1.channels.c2.kafka.bootstrap.servers = 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
  31. a1.channels.c2.kafka.topic = api_searchrecord_bi
  32. a1.channels.c2.kafka.consumer.group.id = flume2kafka
  33. # Define an Avro source called r1 on a1 and tell it
  34. #注意这里分出2个channel出来,一个是到k1,一个到k2
  35. #如果这里只出一个channel c1的话,下面2个sink channel都为c1,则消息就会随机分到2个sink上,而不是2个sink都有一份
  36. a1.sources.r2.channels = c2
  37. a1.sources.r2.type = exec
  38. a1.sources.r2.command = tail -F /log/resin-books/search/searchrecord.log
  39. a1.sources.r2.restart = true
  40. a1.sources.r2.batchSize = 10000
  41. a1.sources.r2.interceptors = i1 i2 i3
  42. a1.sources.r2.interceptors.i1.type = timestamp
  43. a1.sources.r2.interceptors.i2.type = host
  44. a1.sources.r2.interceptors.i2.useIP = false
  45. a1.sources.r2.interceptors.i3.type = regex_filter
  46. a1.sources.r2.interceptors.i3.regex = ^\\s*$
  47. a1.sources.r2.interceptors.i3.excludeEvents = true
  48. #默认是replicationg,还有multiplexer
  49. #a1.sources.r1.selector.type = replicationg
  50. #####bookincharge
  51. a1.channels.c3.type = org.apache.flume.channel.kafka.KafkaChannel
  52. a1.channels.c3.kafka.bootstrap.servers = 10.26.22.124:8092,10.26.22.121:8092,10.26.22.120:8092,10.26.22.102:8092,10.26.22.122:8092
  53. a1.channels.c3.kafka.topic = api_bookincharge_novel
  54. a1.channels.c3.kafka.consumer.group.id = flume2kafka
  55. # Define an Avro source called r1 on a1 and tell it
  56. # to bind to 0.0.0.0:41414. Connect it to channel c1.
  57. a1.sources.r3.channels = c3
  58. a1.sources.r3.type = exec
  59. a1.sources.r3.command = tail -F /data/log/bookincharge/bookincharge.log
  60. a1.sources.r3.restart = true
  61. a1.sources.r3.batchSize = 10000
  62. a1.sources.r3.interceptors = i1 i2 i3
  63. a1.sources.r3.interceptors.i1.type = timestamp
  64. a1.sources.r3.interceptors.i2.type = host
  65. a1.sources.r3.interceptors.i2.useIP = false
  66. a1.sources.r3.interceptors.i3.type = regex_filter
  67. a1.sources.r3.interceptors.i3.regex = ^\\s*$
  68. a1.sources.r3.interceptors.i3.excludeEvents = true
  69. #默认是replicationg,还有multiplexer
  70. #a1.sources.r2.selector.type = replicationg