package org.jeecg.modules.duan.scala

import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}

object App extends {

  def main(args: Array[String]): Unit = {
    Logger.getLogger("org").setLevel(Level.ERROR)
    val conf = new SparkConf().setAppName("wordcount")
    val sc = new SparkContext(conf)

    //数据文件从HDFS上读取
    //val rdd = sc.textFile("hdfs://vm1:8020/wordcount")

    //模拟数据
    val data: List[String] = List("abc abc bcd bcd", "111 111 111 111")
    sc.parallelize(data)
      .flatMap(_.split(" "))
      .map((_, 1))
      .reduceByKey(_ + _)
      .sortBy(x => x._2, false)
      .map(x => x._1 + "\t" + x._2)
      .saveAsTextFile("hdfs://vm1:8020/wordcount")
  }
}
