Spark中MapValues算子（可以将value的值加起来，相当于reducebykey；也可以将value的个数加起来，相当于countbykey）

65 阅读 0 评论 43 点赞

我是靠谱客的博主标致小土豆，最近开发中收集的这篇文章主要介绍Spark中MapValues算子（可以将value的值加起来，相当于reducebykey；也可以将value的个数加起来，相当于countbykey），觉得挺不错的，现在分享给大家，希望可以做个参考。

概述

package com.bjsxt;

import java.util.Arrays;
import java.util.List;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;
/**
* MapValues算子
* 传进来的值，传出去的格式可以自己设置，亦即可以定义
* 可以将tuple2中value的值加起来,相当于reduceByKey
* 也可以将key的个数加起来，相当于countByKey
*
* @author Administrator
*
*/
public class MapValues {
   public static void main(String[] args) {
       SparkConf conf=new SparkConf().setAppName("test").setMaster("local");
       JavaSparkContext sc=new JavaSparkContext(conf);
       List<Tuple2<String, Integer>> list = Arrays.asList(
               new Tuple2<>("m1", 1),
               new Tuple2<>("m2", 2),
               new Tuple2<>("m1", 3),
               new Tuple2<>("m2", 4),
               new Tuple2<>("m1", 5),
               new Tuple2<>("m2", 6),
               new Tuple2<>("m1", 7)
               );
       JavaPairRDD<String,Integer> rdd1 = sc.parallelizePairs(list);
       JavaPairRDD<String, Tuple2<Integer, Integer>> rdd2 = rdd1.mapValues(new Function<Integer, Tuple2<Integer,Integer>>() {

           @Override
           public Tuple2<Integer, Integer> call(Integer speed) throws Exception {

               return new Tuple2<Integer, Integer>(speed, 1);
           }
       });
       JavaPairRDD<String, Tuple2<Integer, Integer>> result = rdd2.reduceByKey(new Function2<Tuple2<Integer,Integer>, Tuple2<Integer,Integer>, Tuple2<Integer,Integer>>() {

           @Override
           public Tuple2<Integer, Integer> call(Tuple2<Integer, Integer> t1, Tuple2<Integer, Integer> t2)
                   throws Exception {
               // TODO Auto-generated method stub
               return new Tuple2<Integer, Integer>(t1._1+t2._1, t1._2+t2._2);
           }
       });
       result.foreach(new VoidFunction<Tuple2<String,Tuple2<Integer,Integer>>>() {

           @Override
           public void call(Tuple2<String, Tuple2<Integer, Integer>> arg0) throws Exception {
               System.out.println(arg0);
           }
       });
   }
}