Spark変換演算子交差を求めるSCALA実装(insertsection)

2310 ワード

package day06
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object MyTransformation_scala_3 {
  def myInsertsection(sc: SparkContext): String = {
    val RDD1 = sc.parallelize(List("aa", "bb", "cc"))
    val RDD2 = sc.parallelize(List("aa", "cc"))
    val insertsectionRDD: RDD[(String)] = RDD1.intersection(RDD2)
    insertsectionRDD.collect().toList.toString()
  }
  def main(args: Array[String]) {
    val conf: SparkConf = new SparkConf().setMaster("local").setAppName("MyTransformation_scala_3")
    var sc: SparkContext = new SparkContext(conf)
    var result: String = ""
    result = myInsertsection(sc)
    println(result)
    sc.stop()
  }
}