1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60 |
[iyunv@bdroute-s1 ansible]# ansible-playbook flume.yml -e 'host="test" action="started" app="syh_php_api" logfile="/data/nginx/test.log"'
PLAY [test] *******************************************************************
TASK: [flume | install flume agent] *******************************************
ok: [10.0.0.174]
TASK: [flume | sync config] ***************************************************
changed: [10.0.0.174]
TASK: [flume | sync script] ***************************************************
ok: [10.0.0.174]
TASK: [flume | manager flume] *************************************************
changed: [10.0.0.174]
PLAY RECAP ********************************************************************
10.0.0.174 : ok=4 changed=2 unreachable=0 failed=0
[iyunv@h000174 ~]# ps aux|grep flume
root 8049 1.7 0.1 8540576 97628 ? Sl 19:39 0:09 /usr/bin/java -Xmx20m -cp /usr/local/common/flume/conf:/usr/local/flume/lib/* -Djava.library.path= org.apache.flume.node.Application --conf-file /usr/local/common/flume/conf/flume.conf --name a1
root 10337 0.0 0.0 103244 880 pts/0 S+ 19:47 0:00 grep flume
[iyunv@h000174 ~]# cat /usr/local/common/flume/conf/flume.conf
#Define source, channel, sink
a1.sources = r1
a1.channels = c1
a1.sinks = k1 k2
#Configure source 1
a1.sources.r1.type = exec
a1.sources.r1.channels = c1
a1.sources.r1.command = tail -F /data/nginx/test.log
a1.sources.r1.shell = /bin/bash -c
a1.sources.r1.restartThrottle = 10000
a1.sources.r1.restart = true
a1.sources.r1.batchSize = 20
a1.sources.r1.batchTimeout = 3000
#Configure channel 1
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /data/flume/checkpoint/syh_php_api
a1.channels.c1.dataDirs = /data/flume/data/syh_php_api
a1.channels.c1.capacity = 200000000
a1.channels.c1.transactionCapacity = 1000
a1.channels.c1.keep-alive = 30
a1.channels.c1.write-timeout = 30
a1.channels.c1.checkpoint-timeout=600
#Configure sinkgroups
a1.sinkgroups = g1
a1.sinkgroups.g1.sinks = k1 k2
a1.sinkgroups.g1.processor.type = load_balance
a1.sinkgroups.g1.processor.backoff = true
a1.sinkgroups.g1.processor.selector = round_robin
a1.sinkgroups.g1.processor.selector.maxTimeOut = 600000
#Configure sink 1
a1.sinks.k1.type = avro
a1.sinks.k1.channel = c1
a1.sinks.k1.hostname = hdcollect-s1.wepiao.com
a1.sinks.k1.port = 40003
#Configure k1
a1.sinks.k2.type = avro
a1.sinks.k2.channel = c1
a1.sinks.k2.hostname = hdcollect-s2.wepiao.com
a1.sinks.k2.port = 40003 |