论坛 › Flume › flume的几个经典配置案例

liuxiaoyun111 发表于 2015-11-27 19:59:59

flume的几个经典配置案例

　　   一、source是spooldir、channel是memory，sink是hbase
　　         a1.sources = r1
         a1.sinks = hbaseSink
　　         a1.channels = memChannel
　　

　　         ##########the type of channel ismemory#########

　　         a1.channels.memChannel.type = memory
         a1.channels.memChannel.capacity = 10000
         a1.channels.memChannel.transactionCapacity = 1000
         a1.channels.memChannel.byteCapacityBufferPercentage = 20
         a1.channles.memChannel.byteCapacity = 80000

　　         ##########the type ofchannel is memory#########

　　

　　         ##########the type of sink is hbase#########
         a1.sinks.hbaseSink.type = hbase
         a1.sinks.hbaseSink.table=mytable
   a1.sinks.hbaseSink.columnFamily=cf
a1.sinks.hbaseSink.serializer=org.apache.flume.sink.hbase.RegexHbaseEventSerializer
         a1.sinks.hbaseSink.zookeeperQuorum=flumekafka1，flumekafka2，flumekafka3
         ##########the type of sink is hbase#########

　　         
　　         a1.sources.r1.channels = memChannel
         a1.sinks.k1.channel = memChannel

　　

　　       二、source是spooldir、channel是memory，sink是hdfs
　　
　　            a1.sources = r1
            a1.sinks = hdfsSink
　　            a1.channels = memChannel
　　

　　             ##########the type of channel is memory#########

　　             a1.channels.memChannel.type = memory
             a1.channels.memChannel.capacity = 10000
             a1.channels.memChannel.transactionCapacity = 1000
             a1.channels.memChannel.byteCapacityBufferPercentage = 20
             a1.channles.memChannel.byteCapacity = 80000

　　             ##########the type of channel is memory#########

　　

　　             ##########the type of sink is hdfs#########
         a1.sinks.hdfsSink.type=hdfs
   a1.sinks.hdfsSink.hdfs.path=hdfs://master:9000/flume1
   a1.sinks.hdfsSink.hdfs.filePrefix=lwq
   a1.sinks.sink1.hdfs.round = true
   a1.sinks.hdfsSink.hdfs.rollInterval = 36
   a1.sinks.hdfsSink.hdfs.rollSize = 128000000
   a1.sinks.hdfsSink.hdfs.rollCount = 0
   a1.sinks.hdfsSink.hdfs.batchSize = 1000
   a1.sinks.hdfsSink.hdfs.roundValue = 1
   a1.sinks.hdfsSink.hdfs.roundUnit = minute
   a1.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
   a1.sinks.hdfsSink.hdfs.fileType = DataStream
   a1.sinks.hdfsSink.hdfs.writeFormat = Text
             ##########the type of sink is hdfs#########

　　         
　　             a1.sources.r1.channels = memChannel
             a1.sinks.k1.channel = memChannel
　　

       三、source是spooldir、channel是memory，sink是kafka
　　         
            a1.sources = r1
            a1.sinks = kafkaSink
            a1.channels =memChannel


             ##########the type of channel is memory#########

             a1.channels.memChannel.type = memory
             a1.channels.memChannel.capacity = 10000
             a1.channels.memChannel.transactionCapacity = 1000
             a1.channels.memChannel.byteCapacityBufferPercentage = 20
             a1.channles.memChannel.byteCapacity = 80000

             ##########the type of channel is memory#########



             ##########the type of sink is kafka#########
            a1.sinks.kafkaSink.type = org.apache.flume.sink.kafka.KafkaSink
             a1.sinks.kafkaSink.topic=flume1
                     a1.sinks.kafkaSink.brokerList=flumekafka1:9092,flumekafka2:9092,flumekafka3:9092
             a1.sinks.kafkaSink.requiredAcks=1
             a1.sinks.kafkaSink.batchSize = 20
             ##########the type of sink is kafka#########

         
             a1.sources.r1.channels =memChannel
             a1.sinks.kafkaSink.channel = memChannel
　　
　　      四、source是spooldir、channel是memory，sink是kafka和hdfs

　　         
a1.sources = r1
a1.sinks = kafkaSink hdfsSink
a1.channels = kafkaChannel hdfsChannel

a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /home/bms/flumeSpool
a1.sources.r1.fileHeader = true
a1.sources.r1.deletePolicy = immediate

##########the type of channel is kafka#########
a1.channels.kafkaChannel .type = memory
a1.channels.kafkaChannel .capacity = 10000
a1.channels.kafkaChannel .transactionCapacity = 1000
a1.channels.kafkaChannel .byteCapacityBufferPercentage = 20
a1.channles.kafkaChannel .byteCapacity = 80000
##########the type of channel is kafka#########

##########the type of channel is hdfs#########
a1.channels.hdfsChannel.type = memory
a1.channels.hdfsChannel.capacity = 10000
a1.channels.hdfsChannel.transactionCapacity = 1000
a1.channels.hdfsChannel.byteCapacityBufferPercentage = 20
a1.channles.hdfsChannel.byteCapacity = 80000
##########the type of channel ishdfs#########　　
##########the type of sink is kafka#########
a1.sinks.kafkaSink .type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.kafkaSink .topic=flume1
a1.sinks.kafkaSink .brokerList=flumekafka1:9092,flumekafka2:9092,flumekafka3:9092
a1.sinks.kafkaSink .requiredAcks=1
a1.sinks.kafkaSink .batchSize = 20
##########the type of sink is kafka#########


##########the type of sink is kafka#########
a1.sinks.hdfsSink.type=hdfs
a1.sinks.hdfsSink.hdfs.path=hdfs://master:9000/flume1
a1.sinks.hdfsSink.hdfs.filePrefix=lwq
a1.sinks.sink1.hdfs.round = true
a1.sinks.hdfsSink.hdfs.rollInterval = 36
a1.sinks.hdfsSink.hdfs.rollSize = 128000000
a1.sinks.hdfsSink.hdfs.rollCount = 0
a1.sinks.hdfsSink.hdfs.batchSize = 1000
a1.sinks.hdfsSink.hdfs.roundValue = 1
a1.sinks.hdfsSink.hdfs.roundUnit = minute
a1.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
a1.sinks.hdfsSink.hdfs.fileType = DataStream
a1.sinks.hdfsSink.hdfs.writeFormat = Text
##########the type of sink is hdfs#########

a1.sources.r1.channels = kafkaChannel hdfsChannel
a1.sinks.kafkaSink .channel =kafkaChannel
a1.sinks.hdfsSink.channel = hdfsChannel

a1.sources.r1.selector.type = replicating
　　

查看完整版本: flume的几个经典配置案例