hadoop - 代码不会在 wordcount 程序中跳过两个单词

Question

此代码计算单词数并跳过文件中的两个给定单词(in & of):-请帮助它为什么不跳过这些词。

import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

class skipwc_mapper extends
            Mapper<LongWritable, Text, Text, IntWritable> {

        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {

            String line = value.toString();
            StringTokenizer t = new StringTokenizer(line);
            Text word = null;
            while (t.hasMoreTokens()) {
                word = new Text(t.nextToken());
                context.write(word, new IntWritable(1));
            }
        }
    }

    class skipwc_reducer extends
            Reducer<Text, IntWritable, Text, IntWritable> {
        protected void reduce(Text key, Iterable<IntWritable> values,
                Context context) throws IOException, InterruptedException {
            int tot = 0;
            if (key.toString() != "in" && key.toString() != "of") {
                while (values.iterator().hasNext()) {
                    tot += values.iterator().next().get();
                }
                context.write(key, new IntWritable(tot));
            }
        }
    }

    public static class skipwc_runner {
        public static void main(String[] args) throws IOException,
                InterruptedException, ClassNotFoundException {
            Configuration conf = new Configuration();
            Job job = new Job(conf);
            job.setJarByClass(skipwc_runner.class);

            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(IntWritable.class);

            job.setMapperClass(skipwc_mapper.class);
            job.setReducerClass(skipwc_reducer.class);

            job.setInputFormatClass(TextInputFormat.class);
            job.setOutputFormatClass(TextOutputFormat.class);

            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileOutputFormat.setOutputPath(job, new Path(args[1]));

            System.exit(job.waitForCompletion(true) ? 0 : 1);
        }
    }
}

Answer 1

使用equals 方法来比较字符串，例如:

if (!"in".equals(key.toString()) && !"of".equals(key.toString())) 

此外，如果您在 mapper 中跳过/in 而不是 reducer，这将是有益的，因为它会在排序和混洗阶段之前有效地删除数据，从而避免额外的 IO。