Kafka
Flume:
sourceAgent.channels = logCh kafkaCh
sourceAgent.sources = sourceDataSource
sourceAgent.sinks = logSink kafkaSink
sourceAgent.channels.kafkaCh.type = memory
sourceAgent.channels.kafkaCh.capacity = 1000000
sourceAgent.channels.kafkaCh.transactionCapacity = 1000000
sourceAgent.channels.kafkaCh.keep-alive = 10
sourceAgent.channels.logCh.type = memory
sourceAgent.channels.logCh.capacity = 1000000
sourceAgent.channels.logCh.transactionCapacity = 1000000
sourceAgent.channels.logCh.keep-alive = 10
sourceAgent.sources.sourceDataSource.type = org.apache.flume.source.http.HTTPSource
sourceAgent.sources.sourceDataSource.bind= 172.21.9.74
sourceAgent.sources.sourceDataSource.port = 30001
sourceAgent.sources.sourceDataSource.channels = logCh kafkaCh
sourceAgent.sinks.logSink.channel = logCh
sourceAgent.sinks.logSink.type = logger
sourceAgent.sinks.kafkaSink.channel = kafkaCh
sourceAgent.sinks.kafkaSink.type = org.apache.flume.sink.kafka.KafkaSink
sourceAgent.sinks.kafkaSink.topic = icestone-gateway-event-topic
sourceAgent.sinks.kafkaSink.brokerList =172.21.9.70:6667,172.21.9.72:6667,172.21.9.73:6667,172.21.9.74:6667
sourceAgent.sinks.kafkaSink.flumeBatchSize = 20
sourceAgent.sinks.kafkaSink.producer.acks = 1
sourceAgent.sinks.kafkaSink.producer.linger.ms = 1
sourceAgent.sinks.kafkaSink.producer.compression.type = none
HDFS
YARN
Hbase
Oozie
Spark
MR2