ssh XXX@XXX

XXXX.bj

scp 开发机用户名@开发机ip地址:/home/XXXXX/文件 .    #最后一个.表示cd的根目录下

object WordCount {
  def main(args: Array[String]) {
//    val inputFile =  "file:///home/mi/coding/coding/Scala/word-count/input/README.txt"
//    val inputFile =  "file://README.txt"
    val inputFile =  "/user/XXXX/lintong/README.txt"
    val conf = new SparkConf().setAppName("WordCount").setMaster("yarn-client")
    val sc = new SparkContext(conf)
    val textFile = sc.textFile(inputFile)
    val wordCount = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
//    wordCount.foreach(println)
//    wordCount.saveAsTextFile("file:///home/mi/coding/coding/Scala/word-count/output/READMEOUT.txt")
    wordCount.saveAsTextFile("/user/XXXX/lintong/READMEOUT.txt")
  }
}

hadoop fs -put /homeXXX/文件名 ./lintong    #.注意.的目录地址是用户的根目录

hadoop fs -ls ./lintong

spark-submit --cluster XXXXX \
    --master yarn-client \
    --num-executors 3 \
    --class "包名.类名" \
    --queue XXXXX \
    word-count_2.11-1.0.jar

wordCount.saveAsTextFile("/user/XXXX/lintong/READMEOUT.txt")

hadoop fs -ls ./lintong/READMEOUT.txt

lintong/READMEOUT.txt/_SUCCESS
lintong/READMEOUT.txt/part-00000

import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.conf.Configuration
System.setProperty("java.security.krb5.conf", "/etc/krb5.conf")
val configuration = new Configuration()
configuration.set("hadoop.security.authentication", "Kerberos")
UserGroupInformation.setConfiguration(configuration)
UserGroupInformation.loginUserFromKeytab("xxx@XXXX", "xxx.keytab")