flume的几个经典配置案例

liuxiaoyun111

　　   一、source是spooldir、channel是memory，sink是hbase
　　           a1.sources = r1
           a1.sinks = hbaseSink
　　           a1.channels = memChannel
　　

　　           ##########the type of channel ismemory#########

　　           a1.channels.memChannel.type = memory
           a1.channels.memChannel.capacity = 10000
           a1.channels.memChannel.transactionCapacity = 1000
           a1.channels.memChannel.byteCapacityBufferPercentage = 20
           a1.channles.memChannel.byteCapacity = 80000

　　           ##########the type ofchannel is memory#########

　　

　　           ##########the type of sink is hbase#########
           a1.sinks.hbaseSink.type = hbase
           a1.sinks.hbaseSink.table=mytable
   a1.sinks.hbaseSink.columnFamily=cf
  a1.sinks.hbaseSink.serializer=org.apache.flume.sink.hbase.RegexHbaseEventSerializer
           a1.sinks.hbaseSink.zookeeperQuorum=flumekafka1，flumekafka2，flumekafka3
           ##########the type of sink is hbase#########

　　           
　　           a1.sources.r1.channels = memChannel
           a1.sinks.k1.channel = memChannel

　　

　　       二、source是spooldir、channel是memory，sink是hdfs
　　
　　              a1.sources = r1
              a1.sinks = hdfsSink
　　              a1.channels = memChannel
　　

　　             ##########the type of channel is memory#########

　　             a1.channels.memChannel.type = memory
             a1.channels.memChannel.capacity = 10000
             a1.channels.memChannel.transactionCapacity = 1000
             a1.channels.memChannel.byteCapacityBufferPercentage = 20
             a1.channles.memChannel.byteCapacity = 80000

　　             ##########the type of channel is memory#########

　　

　　             ##########the type of sink is hdfs#########
           a1.sinks.hdfsSink.type=hdfs
     a1.sinks.hdfsSink.hdfs.path=hdfs://master:9000/flume1
     a1.sinks.hdfsSink.hdfs.filePrefix=lwq
     a1.sinks.sink1.hdfs.round = true
     a1.sinks.hdfsSink.hdfs.rollInterval = 36
     a1.sinks.hdfsSink.hdfs.rollSize = 128000000
     a1.sinks.hdfsSink.hdfs.rollCount = 0
     a1.sinks.hdfsSink.hdfs.batchSize = 1000
     a1.sinks.hdfsSink.hdfs.roundValue = 1
     a1.sinks.hdfsSink.hdfs.roundUnit = minute
     a1.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
     a1.sinks.hdfsSink.hdfs.fileType = DataStream
     a1.sinks.hdfsSink.hdfs.writeFormat = Text
             ##########the type of sink is hdfs#########

　　           
　　             a1.sources.r1.channels = memChannel
             a1.sinks.k1.channel = memChannel
　　

       三、source是spooldir、channel是memory，sink是kafka
　　         
              a1.sources = r1
              a1.sinks = kafkaSink
              a1.channels =memChannel


             ##########the type of channel is memory#########

             a1.channels.memChannel.type = memory
             a1.channels.memChannel.capacity = 10000
             a1.channels.memChannel.transactionCapacity = 1000
             a1.channels.memChannel.byteCapacityBufferPercentage = 20
             a1.channles.memChannel.byteCapacity = 80000

             ##########the type of channel is memory#########



             ##########the type of sink is kafka#########
            a1.sinks.kafkaSink.type = org.apache.flume.sink.kafka.KafkaSink
             a1.sinks.kafkaSink.topic=flume1
                       a1.sinks.kafkaSink.brokerList=flumekafka1:9092,flumekafka2:9092,flumekafka3:9092
             a1.sinks.kafkaSink.requiredAcks=1
             a1.sinks.kafkaSink.batchSize = 20
             ##########the type of sink is kafka#########

           
             a1.sources.r1.channels =memChannel
             a1.sinks.kafkaSink.channel = memChannel
　　  
　　        四、source是spooldir、channel是memory，sink是kafka和hdfs

　　         
a1.sources = r1
a1.sinks = kafkaSink hdfsSink
a1.channels = kafkaChannel hdfsChannel

a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /home/bms/flumeSpool
a1.sources.r1.fileHeader = true
a1.sources.r1.deletePolicy = immediate

##########the type of channel is kafka#########
a1.channels.kafkaChannel .type = memory
a1.channels.kafkaChannel .capacity = 10000
a1.channels.kafkaChannel .transactionCapacity = 1000
a1.channels.kafkaChannel .byteCapacityBufferPercentage = 20
a1.channles.kafkaChannel .byteCapacity = 80000
##########the type of channel is kafka#########

##########the type of channel is hdfs#########
a1.channels.hdfsChannel.type = memory
a1.channels.hdfsChannel.capacity = 10000
a1.channels.hdfsChannel.transactionCapacity = 1000
a1.channels.hdfsChannel.byteCapacityBufferPercentage = 20
a1.channles.hdfsChannel.byteCapacity = 80000
##########the type of channel ishdfs#########　　
##########the type of sink is kafka#########
a1.sinks.kafkaSink .type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.kafkaSink .topic=flume1
a1.sinks.kafkaSink .brokerList=flumekafka1:9092,flumekafka2:9092,flumekafka3:9092
a1.sinks.kafkaSink .requiredAcks=1
a1.sinks.kafkaSink .batchSize = 20
##########the type of sink is kafka#########


##########the type of sink is kafka#########
a1.sinks.hdfsSink.type=hdfs
a1.sinks.hdfsSink.hdfs.path=hdfs://master:9000/flume1
a1.sinks.hdfsSink.hdfs.filePrefix=lwq
a1.sinks.sink1.hdfs.round = true
a1.sinks.hdfsSink.hdfs.rollInterval = 36
a1.sinks.hdfsSink.hdfs.rollSize = 128000000
a1.sinks.hdfsSink.hdfs.rollCount = 0
a1.sinks.hdfsSink.hdfs.batchSize = 1000
a1.sinks.hdfsSink.hdfs.roundValue = 1
a1.sinks.hdfsSink.hdfs.roundUnit = minute
a1.sinks.hdfsSink.hdfs.useLocalTimeStamp = true
a1.sinks.hdfsSink.hdfs.fileType = DataStream
a1.sinks.hdfsSink.hdfs.writeFormat = Text
##########the type of sink is hdfs#########

a1.sources.r1.channels = kafkaChannel hdfsChannel
a1.sinks.kafkaSink .channel =kafkaChannel
a1.sinks.hdfsSink.channel = hdfsChannel

a1.sources.r1.selector.type = replicating