SparkStreamingの単語カウントケース


import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
  *     nc -lk 9999
  * */
/**
  * Created by Administrator on 2018/5/11.
  */
object SparkStreaming1 {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("workcount").setMaster("local[2]")
    //        
    //             transiformation  
    val sc = new SparkContext(conf)
    val scc = new StreamingContext(sc,Seconds(2))
    //  StreamingContext    
    val dstream: ReceiverInputDStream[String] = scc.socketTextStream("hadoop02",9999)
    //            
    /**
      *       
      * */
    dstream.
      flatMap(line=>line.split(" ")).
      map(x=>(x,1)).
      reduceByKey(_+_).print()
    /**
      * print    collect
      * */
    scc.start()
    scc.awaitTermination()
    scc.stop()
    /**
      *         
      * */
  }
}