flume-ng avro形式传输数据配置 flume-ng多节点实例
flume-ng avro方式传输数据配置 flume-ng多节点实例
tail-to-avro
agent1.sources = source1 agent1.sinks = sink1 agent1.channels = channel1 # Describe/configure spooldir source1 #agent1.sources.source1.type = spooldir #agent1.sources.source1.spoolDir = /var/log/apache/flumeSpool1 #agent1.sources.source1.fileHeader = true # Describe/configure tail -F source1 agent1.sources.source1.type = exec agent1.sources.source1.command = tail -F /tmp/log.log agent1.sources.source1.channels = channel1 # Describe/configure nc source1 #agent1.sources.source1.type = netcat #agent1.sources.source1.bind = localhost #agent1.sources.source1.port = 44444 #configure host for source agent1.sources.source1.interceptors = i1 agent1.sources.source1.interceptors.i1.type = host agent1.sources.source1.interceptors.i1.hostHeader = hostname # Describe logger sink1 #agent1.sinks.sink1.type = logger # Describe avro sink1 agent1.sinks.sink1.type = avro agent1.sinks.sink1.hostname = 172.16.10.175 agent1.sinks.sink1.port = 4545 # Use a channel which buffers events in memory agent1.channels.channel1.type = memory agent1.channels.channel1.keep-alive = 120 agent1.channels.channel1.capacity = 500000 agent1.channels.channel1.transactionCapacity = 600 # Bind the source and sink to the channel agent1.sources.source1.channels = channel1 agent1.sinks.sink1.channel = channel1
avro-to-rollfile
agent1.sources = source1 agent1.sinks = sink1 agent1.channels = channel1 # Describe/configure avro source agent1.sources.source1.type = avro agent1.sources.source1.bind =172.16.10.175 agent1.sources.source1.port = 4545 # Describe logger sink1 #agent1.sinks.sink1.type = logger # Describe file sink1 agent1.sinks.sink1.type = file_roll agent1.sinks.sink1.sink.directory = /var/log/flume # Use a channel which buffers events in memory agent1.channels.channel1.type = memory agent1.channels.channel1.keep-alive = 120 agent1.channels.channel1.capacity = 500000 agent1.channels.channel1.transactionCapacity = 600 # Bind the source and sink to the channel agent1.sources.source1.channels = channel1 agent1.sinks.sink1.channel = channel1
启动:
./flume-ng agent -c /usr/local/flumeng/apache-flume-1.5.0-bin/conf/ -f /usr/local/flumeng/apache-flume-1.5.0-bin/conf/flume-single.properties -n agent1 -Dflume.root.logger=INFO,console