SparkStreamingの単語カウントケース
1027 ワード
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* nc -lk 9999
* */
/**
* Created by Administrator on 2018/5/11.
*/
object SparkStreaming1 {
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setAppName("workcount").setMaster("local[2]")
//
// transiformation
val sc = new SparkContext(conf)
val scc = new StreamingContext(sc,Seconds(2))
// StreamingContext
val dstream: ReceiverInputDStream[String] = scc.socketTextStream("hadoop02",9999)
//
/**
*
* */
dstream.
flatMap(line=>line.split(" ")).
map(x=>(x,1)).
reduceByKey(_+_).print()
/**
* print collect
* */
scc.start()
scc.awaitTermination()
scc.stop()
/**
*
* */
}
}