hadoop - 如何使用Spring Batch Yarn Application重新启动Map Reduce Hadoop作业？

Question

我有Map Reduce应用程序，我想使用Spring Batch Yarn逻辑。
就像下面链接中提到的那样。

https://spring.io/guides/gs/yarn-batch-restart/

但我希望它真的很具体，因为通常我们在hadoop map减少工作中拥有。
仅查找Spring Batch Yarn的类和配置。考虑到我的hadoop map减少逻辑已经就位并且可以正常工作。

提前致谢 !!

Answer 1

这是您正在寻找的ApplicationContext配置

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util"
    xmlns:context="http://www.springframework.org/schema/context"
    xmlns:hdp="http://www.springframework.org/schema/hadoop" xmlns:batch="http://www.springframework.org/schema/batch"
    xsi:schemaLocation="
    http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
    http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd
     http://www.springframework.org/schema/context  http://www.springframework.org/schema/context/spring-context.xsd
     http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch.xsd
    http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-4.2.xsd">

    <context:property-placeholder location="classpath:application.properties" />
    <hdp:configuration namenode-principal="hdfs://xx.yy.com" rm-manager-uri="xx.yy.com"
        security-method="kerb" user-keytab="location" rm-manager-principal="username"
        user-principal="username">
        fs.default.name=${fs.default.name}
        mapred.job.tracker=${mapred.job.tracker}
    </hdp:configuration>

    <hdp:job id="wordCountJobId" input-path="${input.path}"
        output-path="${output.path}" jar-by-class="com.xx.poc.Application"
        mapper="com.xx.poc.Map" reducer="com.xx.poc.Reduce" />

    <hdp:job-runner id="wordCountJobRunner" job-ref="wordCountJobId"
        run-at-startup="true" />
</beans>