gpt4 book ai didi

org.apache.hadoop.mapred.YARNRunner.createApplicationSubmissionContext()方法的使用及代码示例

转载 作者:知者 更新时间:2024-03-15 01:11:31 28 4
gpt4 key购买 nike

本文整理了Java中org.apache.hadoop.mapred.YARNRunner.createApplicationSubmissionContext()方法的一些代码示例,展示了YARNRunner.createApplicationSubmissionContext()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。YARNRunner.createApplicationSubmissionContext()方法的具体详情如下:
包路径:org.apache.hadoop.mapred.YARNRunner
类名称:YARNRunner
方法名:createApplicationSubmissionContext

YARNRunner.createApplicationSubmissionContext介绍

[英]Constructs all the necessary information to start the MR AM.
[中]构建启动MR AM所需的所有信息。

代码示例

代码示例来源:origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

private ApplicationSubmissionContext buildSubmitContext(
   YARNRunner yarnRunner, JobConf jobConf) throws IOException {
  File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
  OutputStream out = new FileOutputStream(jobxml);
  conf.writeXml(out);
  out.close();

  File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
  out = new FileOutputStream(jobsplit);
  out.close();

  File jobsplitmetainfo = new File(testWorkDir,
    MRJobConfig.JOB_SPLIT_METAINFO);
  out = new FileOutputStream(jobsplitmetainfo);
  out.close();

  return yarnRunner.createApplicationSubmissionContext(jobConf,
    testWorkDir.toString(), new Credentials());
 }
}

代码示例来源:origin: org.apache.hadoop/hadoop-mapreduce-client-jobclient

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
 
 addHistoryToken(ts);
 ApplicationSubmissionContext appContext =
  createApplicationSubmissionContext(conf, jobSubmitDir, ts);
 // Submit to ResourceManager
 try {
  ApplicationId applicationId =
    resMgrDelegate.submitApplication(appContext);
  ApplicationReport appMaster = resMgrDelegate
    .getApplicationReport(applicationId);
  String diagnostics =
    (appMaster == null ?
      "application report is null" : appMaster.getDiagnostics());
  if (appMaster == null
    || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
    || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
   throw new IOException("Failed to run job : " +
     diagnostics);
  }
  return clientCache.getClient(jobId).getJobStatus(jobId);
 } catch (YarnException e) {
  throw new IOException(e);
 }
}

代码示例来源:origin: com.github.jiayuhan-it/hadoop-mapreduce-client-jobclient

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
 
 addHistoryToken(ts);
 
 // Construct necessary information to start the MR AM
 ApplicationSubmissionContext appContext =
  createApplicationSubmissionContext(conf, jobSubmitDir, ts);
 // Submit to ResourceManager
 try {
  ApplicationId applicationId =
    resMgrDelegate.submitApplication(appContext);
  ApplicationReport appMaster = resMgrDelegate
    .getApplicationReport(applicationId);
  String diagnostics =
    (appMaster == null ?
      "application report is null" : appMaster.getDiagnostics());
  if (appMaster == null
    || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
    || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
   throw new IOException("Failed to run job : " +
     diagnostics);
  }
  return clientCache.getClient(jobId).getJobStatus(jobId);
 } catch (YarnException e) {
  throw new IOException(e);
 }
}

代码示例来源:origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

@Before
public void setUp() throws Exception {
 resourceMgrDelegate = mock(ResourceMgrDelegate.class);
 conf = new YarnConfiguration();
 conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
 clientCache = new ClientCache(conf, resourceMgrDelegate);
 clientCache = spy(clientCache);
 yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
 yarnRunner = spy(yarnRunner);
 submissionContext = mock(ApplicationSubmissionContext.class);
 doAnswer(
   new Answer<ApplicationSubmissionContext>() {
    @Override
    public ApplicationSubmissionContext answer(InvocationOnMock invocation)
      throws Throwable {
     return submissionContext;
    }
   }
   ).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class),
     any(String.class), any(Credentials.class));
 appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
 jobId = TypeConverter.fromYarn(appId);
 if (testWorkDir.exists()) {
  FileContext.getLocalFSFileContext().delete(new Path(testWorkDir.toString()), true);
 }
 testWorkDir.mkdirs();
}

28 4 0
Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
广告合作:1813099741@qq.com 6ren.com