hadoop - mapreduce 的 Reducer 函数中的 ArrayIndexOutOfBoundException

Question

我不明白错误是什么，当我删除 job.setSortComparatorClass(LongWritable.DecreasingComparator.class);

我得到了输出，但是当我尝试使用它时出现了这个异常。

我试图根据值从 reducer 中以降序获取输出，因此我使用了 setsortcomparator 类，所以请帮助我

package topten.mostviewed.movies;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MostViewdReducer extends Reducer<Text,IntWritable,Text,LongWritable>
{
    public void reduce(Text key,Iterable<IntWritable> values,Context context) throws IOException,InterruptedException
    {
        int sum = 0;
        for(IntWritable value:values)
        {
            sum = sum+1;
        }
        context.write(key, new LongWritable(sum));
    }
}
package topten.mostviewed.movies;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class MostViewdDriver 
{

   // @SuppressWarnings("unchecked")
    public static void main(String[] args) throws Exception
    {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        if (otherArgs.length != 2)
        {
            System.err.println("Usage: movie <input> <out>");
            System.exit(2);
        }
    Job job = new Job(conf, "Movie ");
    job.setJarByClass(MostViewdDriver.class);
    job.setMapperClass(MostviewdMapper.class);
    job.setReducerClass(MostViewdReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);
    job.setSortComparatorClass(LongWritable.DecreasingComparator.class);
//  job.setSortComparatorClass((Class<? extends RawComparator>) LongWritable.class);

    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

我得到的异常如下:

18/10/11 11:35:05 INFO mapreduce.Job: Task Id : attempt_1539236679371_0004_r_000000_2, Status : FAILED Error: java.lang.ArrayIndexOutOfBoundsException: 7 at org.apache.hadoop.io.WritableComparator.readInt(WritableComparator.java:212) at org.apache.hadoop.io.WritableComparator.readLong(WritableComparator.java:226) at org.apache.hadoop.io.LongWritable$Comparator.compare(LongWritable.java:91) at org.apache.hadoop.io.LongWritable$DecreasingComparator.compare(LongWritable.java:106) at org.apache.hadoop.mapreduce.task.ReduceContextImpl.nextKeyValue(ReduceContextImpl.java:158) at org.apache.hadoop.mapreduce.task.ReduceContextImpl.nextKey(ReduceContextImpl.java:121) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.nextKey(WrappedReducer.java:307) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:170) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:168) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1642) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:163)

Answer 1

您的映射输出键是整数，但您尝试使用用于长整数的比较器。将 LongWritable.DecreasingComparator.class 替换为 IntWritable.DecreasingComparator.class。