hadoop - 我正在尝试将文件中的所有数字相加，该文件包含以空格分隔的数字，并且使用 MapReduce 包含在多行中

Question

我的输出出错了。输入文件是:

1 2 3 4
5 4 3 2

输出应该是key: sum value: 24

MapReduce 产生的输出:key: sum value: 34

我在 Ubuntu 14.04 中使用 OpenJDK 7 来运行 jar 文件，而 jar 文件是在 Eclipse Juna 中创建的，使用的 java 版本是 Oracle JDK 7 来编译它。NumberDriver.java

包裹数量和；

import java.io.*;
//import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
//import org.apache.hadoop.mapreduce.Mapper;
//import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class NumberDriver {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // TODO Auto-generated method stub
            Configuration conf=new Configuration();
            String[] otherArgs=new GenericOptionsParser(conf,args).getRemainingArgs();
            if(otherArgs.length!=2)
            {
                System.err.println("Error");
                System.exit(2);
            }
            Job job=new Job(conf, "number sum");
            job.setJarByClass(NumberDriver.class);
            job.setMapperClass(NumberMapper.class);
            job.setReducerClass(NumberReducer.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(IntWritable.class);
            FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
            FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
            System.exit(job.waitForCompletion(true)?0:1);
    }

}

NumberMapper.java

package numbersum;
import java.io.*;
import java.util.StringTokenizer;

//import org.apache.hadoop.conf.Configuration;
//import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
//import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
//import org.apache.hadoop.mapreduce.Reducer;
//import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
//import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
//import org.apache.hadoop.util.GenericOptionsParser;
//import org.hsqldb.Tokenizer;

public class NumberMapper extends Mapper <LongWritable, Text, Text, IntWritable> 
    {
        int sum;
        public void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException
        {
            StringTokenizer itr=new StringTokenizer(value.toString());
            while(itr.hasMoreTokens())
            {
                sum+=Integer.parseInt(itr.nextToken());
            }
            context.write(new Text("sum"),new IntWritable(sum));
        }
    }

NumberReducer.java

package numbersum;
import java.io.*;
//import java.util.StringTokenizer;

//import org.apache.hadoop.conf.Configuration;
//import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
//import org.apache.hadoop.mapreduce.Job;
//import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
//import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
//import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
//import org.apache.hadoop.util.GenericOptionsParser;

public class NumberReducer extends Reducer <Text, IntWritable, Text, IntWritable>
    {
        public void reduce(Text key,Iterable<IntWritable> values, Context context)throws IOException, InterruptedException
        {
            int sum=0;
            for(IntWritable value:values)
                {
                    sum+=value.get();
                }
            context.write(key,new IntWritable(sum));
        }
    }

Answer 1

我的最佳猜测:

    int sum; // <-- Why a class member?
    public void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException
    {
        int sum = 0; //Why not here?
        StringTokenizer itr=new StringTokenizer(value.toString());

---

猜测的原因:第一张 map :1 + 2 + 3 + 4 = 10第二张 map :(10 +) 2 + 3 + 4 + 5 = 34

..意思是，之前的值被保留。