intellijデバッグspark scalaプログラムエラー

2107 ワード

spark   cdh spark-2.0.1

package main.scala
 import org.apache.spark.rdd.RDD
 import org.apache.spark.{SparkConf, SparkContext}
 /* Created by tomtang on 2016/12/16.
  */
object tomhelloworld {

  def FILE_NAME:String = "word_count_results_"
  def main(args: Array[String]) {

//    if (args.length < 1) {
//      println("Usage:SparkWordCount FileName");
//      System.exit(1);
//    }
    val conf = {
      new SparkConf().setMaster("local[*]").setAppName("spark_test")
    }
    val sc = new SparkContext(conf)
    val textFile = sc.textFile("D:\\IdeaProjects\\first_scala\\data\\testdata.txt")
    val wordCounts: RDD[(String, Int)] = textFile.flatMap(line => line split " ").map(
      word => (word, 1)).reduceByKey((a, b) => a + b)
    //print the results,for debug use.
    //println("Word Count program running results:")
    //wordCounts.collect().foreach(e => {
    //val (k,v) = e
    //println(k+"="+v)
    //});
    wordCounts.saveAsTextFile(FILE_NAME+System.currentTimeMillis())
    println("Word Count program running results are successfully saved.")

    println("Hello World")
  }
}

  :
Exception in thread "main" java.lang.NoSuchMethodError: scala.Predef$.refArrayOps([Ljava/lang/Object;)Lscala/collection/mutable/ArrayOps;
at org.apache.spark.util.Utils$.getCallSite(Utils.scala:1410)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:77)
at sparkTraining.SimpleApp$.main(testScala.scala:9)
at sparkTraining.SimpleApp.main(testScala.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)

  http://blog.csdn.net/u013887486/article/details/53201900
           Scala        Scala,       2.11.7

   scala     。


ivy 2を選択します.10.4