[TOC] # 簡介 **flume** cloudera公司研發 適合下游數據消費者不多的情況 適合數據安全性要求不多的情況 適合與hadoop生態圈對接的操作 **kafka** linkedin公司研發 適合數據下游消費眾多的情況 適合數據安全性要求比較高的操作,支持replication # 代碼 配置flume(flume-kafka.conf) ~~~ # define a1.sources = r1 a1.sinks = k1 a1.channels = c1 # source a1.sources.r1.type = exec # 加-c +0表示從頭開始讀并開始監控,這個文件有可能被很多人監控過 a1.sources.r1.command = tail -F -c +0 /opt/module/datas/flume.log a1.sources.r1.shell = /bin/bash -c # sink a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.bootstrap.servers = master:9092,slave1:9092,slave2:9092 a1.sinks.k1.kafka.topic = first a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 1 a1.sinks.k1.kafka.producer.linger.ms = 1 # channel a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100 # bind a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1 ~~~