package com.roy.spark;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;
import scala.Tuple2;

import java.util.Arrays;
import java.util.List;

/**
 * @author ：楼兰
 * @date ：Created in 2021/5/22
 * @description: 只需要引入Spark的核心库，就可以直接使用。
 *  spark读取本地文件，计算wordcount
 **/
public class SparkWordCount {
    public static void main(String[] args) {
        final SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("wordcount");
        //spark应用启动时就会默认创建sc
        final SparkContext sc = new SparkContext(sparkConf);

        final JavaRDD<String> lines = sc.textFile("E:\\tmp\\test1.txt",2).toJavaRDD();

        final JavaRDD<String> words = lines.flatMap(s -> Arrays.asList(s.split(" ")).iterator());
        final JavaPairRDD<String, Integer> ones = words.mapToPair(s -> new Tuple2<>(s, 1));
        final JavaPairRDD<String, Integer> counts = ones.reduceByKey((i1, i2) -> i1 + i2);
        final List<Tuple2<String, Integer>> output = counts.collect();
        for(Tuple2 tuple2:output){
            System.out.println(tuple2._1+" : "+tuple2._2);
        }
        sc.stop();

    }
}
