maven - 在CDH4上运行简单的MR作业

Question

我正在尝试使用CDH4运行一个简单的MR作业。我收到最奇怪的错误，我也不知道为什么。基本上，我的程序读取一个文件，使用一个标识映射器，然后reducer仅仅为该值发出一个键和一个字符串。我不明白为什么我的脚本无法正常工作。我在CDH3中从未遇到过这样的问题。任何建议都很好

错误:

14/03/26 20:35:45 INFO mapred.JobClient: Task Id : attempt_201403171159_0109_m_000002_2, Status : FAILED
java.lang.NumberFormatException: For input string: "256MB"
        at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
        at java.lang.Integer.parseInt(Integer.java:492)
        at java.lang.Integer.parseInt(Integer.java:527)
        at org.apache.hadoop.conf.Configuration.getInt(Configuration.java:1060)
        at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.init(MapTask.java:809)
        at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:376)
        at org.apache.hadoop.mapred.MapTask.access$100(MapTask.java:85)
        at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:584)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:656)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:330)
        at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
        at org.apache.hadoop

Maven依赖项:

<dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-core</artifactId>
            <version>2.0.0-mr1-cdh4.4.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.0.0-cdh4.4.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-tools</artifactId>
            <version>2.0.0-mr1-cdh4.4.0</version>
        </dependency>

Maven仓库:

<repositories>
        <repository>
            <id>cloudera</id>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>
        <repository>
            <id>maven-hadoop</id>
            <name>Hadoop Releases</name>
            <url>https://repository.cloudera.com/content/repositories/releases/</url>
        </repository>
    </repositories>

MR代码:

package com.some.packagename;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


public class MyMRJob extends Configured implements Tool {

    private static String inputPath = "someHDFSInputPath";
    private static String outputPath = "someHDFSOutputPath";

    public static void main(String[] args) throws Exception {


        Configuration conf = new Configuration();
        conf.set("mapred.job.tracker", "jtserver:8021");
        conf.set("fs.defaultFS", "hdfs://nnserver:8020");   
        ToolRunner.run(conf, new MyMRJob(), args);

    }

    public final int run(final String[] args) throws Exception {

        // Initialize
        Job job = new Job(super.getConf(),MyMRJob.class.getSimpleName());

        // General Configs
        job.setJarByClass(MyMRJob.class);       

        // Inputs       
        TextInputFormat.setInputPaths(job, inputPath);
        job.setInputFormatClass(TextInputFormat.class);

        // Mapper
        job.setMapperClass(TheMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        // Reducer
        job.setReducerClass(TheReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        // Output
        TextOutputFormat.setOutputPath(job, new Path(outputPath));
        job.setOutputFormatClass(TextOutputFormat .class);

        // Run the job
        boolean b = job.waitForCompletion(true);
        if (!b)
            throw new IOException("Error with the job - it has failed!");

        return 1;
    }

    private static class TheMapper extends Mapper<Text, Text, Text, Text> {
        protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
            context.write(key, value);
        }
    }

    public static class TheReducer extends Reducer<Text, Text, Text, Text> {

        public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            context.write(key, new Text("some value"));
        }
    }


}

Answer 1

看看你的

mapred-site.xml

它可能具有“256MB”之类的配置，尤其是以下属性。

mapred.child.java.opts and io.sort.mb