package org.jeecg.cyl

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._

object handle {
  def main(args: Array[String]): Unit = {

    // 创建Spark Session
    val spark = SparkSession.builder()
      .appName("Weather Temperature Prediction")
      .master("local[*]")
      .getOrCreate()

    // 加载数据
    val mumBabyDF = spark.read
      .option("header", "true") // 使用首行作为列名
      .option("inferSchema", "true") // 推断数据类型
      .csv("C:\\Users\\23972\\Desktop\\Completion-template-cmd\\system\\start\\src\\main\\java\\org\\jeecg\\cyl\\tianchi_mum_baby.csv")

    val tradeHistoryDF = spark.read
      .option("header", "true")
      .option("inferSchema", "true")
      .csv("C:\\Users\\23972\\Desktop\\Completion-template-cmd\\system\\start\\src\\main\\java\\org\\jeecg\\cyl\\tianchi_mum_baby_trade_history.csv")
    // 显示数据的缺失值情况
    mumBabyDF.describe().show()
    tradeHistoryDF.describe().show()

    // 去除含有任何缺失值的行
    val mumBabyNoMissingDF = mumBabyDF.na.drop()
    val tradeHistoryNoMissingDF = tradeHistoryDF.na.drop()

    // 或者使用某些默认值填充缺失值
    //    val mumBabyFilledDF = mumBabyDF.na.fill("Unknown")
    val tradeHistoryFilledDF = tradeHistoryDF.na.fill(Map("buy_mount" -> 0.0))

    // 使用IQR识别并过滤异常值
    def removeOutliers(df: DataFrame, colName: String): DataFrame = {
      val quantiles = df.stat.approxQuantile(colName, Array(0.25, 0.75), 0.05)
      val Q1 = quantiles(0)
      val Q3 = quantiles(1)
      val IQR = Q3 - Q1
      val lowerRange = Q1 - 1.5 * IQR
      val upperRange = Q3 + 1.5 * IQR

      df.filter(col(colName).between(lowerRange, upperRange))
    }

    val tradeHistoryNoOutliersDF = removeOutliers(tradeHistoryFilledDF, "buy_mount")
    // 将清洗后的数据保存为新的CSV文件
    mumBabyNoMissingDF.write.option("header", "true").csv("new_mum_baby.csv")
    tradeHistoryNoOutliersDF.write.option("header", "true").csv("new_trade_history.csv")

  }

}