package com.roy.spark.ml;

import org.apache.spark.SparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;

/**
 * @author ：楼兰
 * @date ：Created in 2021/5/25
 * @description:
 **/

public class LoadDataDemo {

    public static void main(String[] args) {
        final SparkSession spark = SparkSession.builder().appName("LoadDataDemo")
                .master("local[*]")
//                .config(...)
                .getOrCreate();
//        final SparkContext sc = spark.sparkContext();
        final Dataset<Row> csv = spark.read().format("csv")
                .option("inferSchema","true")
                .option("header","true")
                .load("SparkDemo/src/main/resources/iris.csv");
        System.out.println("读取到的数据集");
        csv.show();
        final Dataset<Row>[] csvs = csv.randomSplit(new double[]{0.7, 0.3});

        final Dataset<Row> csv_train = csvs[0];
        final Dataset<Row> csv_test = csvs[1];

        System.out.println("拆分训练集");
        csv_train.show();
        System.out.println("拆分测试集");
        csv_test.show();


        spark.close();

    }
}
