Scala操作Kafka

SparkはKafkaをサポート

         ，    
1.spark-streaming-kafka-0-8_2.11-2.1.0.jar 
2.kafka  jar  
3.jar     spark/jars/kafka

生産者

import org.apache.spark.streaming.kafka._
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
import org.json4s.jackson.Serialization.write

val props = new HashMap[String, Object]()
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092")
// value           （Deserializer）
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer")
// key           （Deserializer）
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer")
//      Kafka   
val producer = new KafkaProducer[String, String](props)

// rdd.colect  rdd        ，  write   rdd     json  
val str = write(rdd.collect)
//    Kafka  ，topic "result"
val message = new ProducerRecord[String, String]("result", null, str)
//  Kafka    
producer.send(message)

消費者

val zkQuorum = "localhost:2181" //Zookeeper     
val group = "1" //topic   group，            ，    1，  val group = "test-consumer-group"
val topics = "wordsender" //topics   
val numThreads = 1 //  topic    
val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap
val lineMap = KafkaUtils.createStream(ssc, zkQuorum, group, topicMap)//