SparkSQL構成(データソースとしてHIVE)

HIVEの構成(mysqlをメタデータとして格納、hdfsをデータとして格納):
1.hive-envを修正する.sh(hive-default.xml.templateコピーから変更可能)

#hadoop    

export HADOOP_HOME=/usr/local/hadoop
# Hive Configuration Directory can be controlled by:
export HIVE_CONF_DIR=/usr/local/hive/conf
# Folder containing extra ibraries required for hive compilation/execution can be controlled by:
export HIVE_AUX_JARS_PATH=/usr/local/hive/lib

2.   hive-site.xml(    hive-default.xml.template  )

#       mysql    

  
    javax.jdo.option.ConnectionURL
    jdbc:mysql://master:3306/hive?createDatabaseIfNotExist=true
    JDBC connect string for a JDBC metastore
  

  
    javax.jdo.option.ConnectionPassword
    youpassword
    password to use against metastore database
  

 
    javax.jdo.option.ConnectionUserName
    root
    Username to use against metastore database
  

  hive      

    ./HIVE_HOME/bin/hive         ！

-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

  spark

1.  spark-env.sh

#           ，  ：     ，     no resource。。。。。。，

export SCALA_HOME=/usr/local/spark
export JAVA_HOME=/usr/local/jdk1.8.0
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
export SPARK_MASTER_IP=master
export SPARK_WORKER_MEMORY=800m
export SPARK_EXECUTOR_MEMORY=800m
export SPARK_DRIVER_MEMORY=800m
export SPARK_WORKER_CORES=4

export MASTER=spark://master:7077

2.  spark-defaults.conf

spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two thr"
spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://master:9000/historyserverforSpark

#      spark        web UI
spark.yarn.historyServer.address        master:18080

spark.history.fs.logDirectory   hdfs://master:9000/historyserverforSpark 

3.  slaves（     work  ）

slave1
slave2

-------------------------------------------------------

 spark/conf     hive-site.xml,    

hive.metastore.uris  
    thrift://master:9083  
    Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore. 

4.   hive    

 hive --servie meatastore

5.   sparkSQL

./bin/spark-bin