本文整理了Java中org.apache.gobblin.yarn.YarnHelixUtils类的一些代码示例，展示了YarnHelixUtils类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。YarnHelixUtils类的具体详情如下：
包路径：org.apache.gobblin.yarn.YarnHelixUtils
类名称：YarnHelixUtils

YarnHelixUtils介绍

[英]A utility class for Gobblin on Yarn.
[中]一个实用类，用于纱线上的Gobblin。

代码示例

代码示例来源：origin: apache/incubator-gobblin

/**
 * Read the {@link Token}s stored in the token file.
 */
@VisibleForTesting
Collection<Token<? extends TokenIdentifier>> readDelegationTokens(Path tokenFilePath) throws IOException {
 LOGGER.info("Reading updated token from token file: " + tokenFilePath);
 return YarnHelixUtils.readTokensFromFile(tokenFilePath, this.fs.getConf());
}

代码示例来源：origin: apache/incubator-gobblin

private void addAppRemoteFiles(String hdfsFileList, Map<String, LocalResource> resourceMap)
  throws IOException {
 for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
  YarnHelixUtils.addFileAsLocalResource(this.fs, new Path(hdfsFilePath), LocalResourceType.FILE, resourceMap);
 }
}

代码示例来源：origin: apache/incubator-gobblin

/**
 * Write the current delegation token to the token file.
 */
@VisibleForTesting
synchronized void writeDelegationTokenToFile() throws IOException {
 if (this.fs.exists(this.tokenFilePath)) {
  LOGGER.info("Deleting existing token file " + this.tokenFilePath);
  this.fs.delete(this.tokenFilePath, false);
 }
 LOGGER.info("Writing new or renewed token to token file " + this.tokenFilePath);
 YarnHelixUtils.writeTokenToFile(this.token, this.tokenFilePath, this.fs.getConf());
 // Only grand access to the token file to the login user
 this.fs.setPermission(this.tokenFilePath, new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
}

代码示例来源：origin: apache/incubator-gobblin

private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
  throws IOException {
 Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
 Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
 Map<String, LocalResource> resourceMap = Maps.newHashMap();
 addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(
   new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
 if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
  addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap);
 }
 ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
 containerLaunchContext.setLocalResources(resourceMap);
 containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
 containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
 if (UserGroupInformation.isSecurityEnabled()) {
  containerLaunchContext.setTokens(this.tokens.duplicate());
 }
 return containerLaunchContext;
}

代码示例来源：origin: apache/incubator-gobblin

amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory())));
if (UserGroupInformation.isSecurityEnabled()) {

代码示例来源：origin: apache/incubator-gobblin

private void addRemoteAppFiles(String hdfsFileList, Map<String, LocalResource> resourceMap) throws IOException {
 for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
  Path srcFilePath = new Path(hdfsFilePath);
  YarnHelixUtils.addFileAsLocalResource(
    srcFilePath.getFileSystem(this.yarnConfiguration), srcFilePath, LocalResourceType.FILE, resourceMap);
 }
}

代码示例来源：origin: apache/incubator-gobblin

@Test(dependsOnMethods = "testGetNewDelegationTokenForLoginUser")
public void testWriteDelegationTokenToFile() throws IOException {
 this.yarnAppSecurityManager.writeDelegationTokenToFile();
 Assert.assertTrue(this.localFs.exists(this.tokenFilePath));
 assertToken(YarnHelixUtils.readTokensFromFile(this.tokenFilePath, this.configuration));
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
  throws IOException {
 Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
 Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
 Map<String, LocalResource> resourceMap = Maps.newHashMap();
 addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(
   new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
 if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
  addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap);
 }
 ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
 containerLaunchContext.setLocalResources(resourceMap);
 containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
 containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
 if (UserGroupInformation.isSecurityEnabled()) {
  containerLaunchContext.setTokens(this.tokens.duplicate());
 }
 return containerLaunchContext;
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

/**
 * Write the current delegation token to the token file.
 */
@VisibleForTesting
synchronized void writeDelegationTokenToFile() throws IOException {
 if (this.fs.exists(this.tokenFilePath)) {
  LOGGER.info("Deleting existing token file " + this.tokenFilePath);
  this.fs.delete(this.tokenFilePath, false);
 }
 LOGGER.info("Writing new or renewed token to token file " + this.tokenFilePath);
 YarnHelixUtils.writeTokenToFile(this.token, this.tokenFilePath, this.fs.getConf());
 // Only grand access to the token file to the login user
 this.fs.setPermission(this.tokenFilePath, new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
}

代码示例来源：origin: apache/incubator-gobblin

private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException {
 if (!this.fs.exists(destDir)) {
  LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
  return;
 }
 FileStatus[] statuses = this.fs.listStatus(destDir);
 if (statuses != null) {
  for (FileStatus status : statuses) {
   YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
  }
 }
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

/**
 * Read the {@link Token}s stored in the token file.
 */
@VisibleForTesting
Collection<Token<? extends TokenIdentifier>> readDelegationTokens(Path tokenFilePath) throws IOException {
 LOGGER.info("Reading updated token from token file: " + tokenFilePath);
 return YarnHelixUtils.readTokensFromFile(tokenFilePath, this.fs.getConf());
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory())));
if (UserGroupInformation.isSecurityEnabled()) {

代码示例来源：origin: apache/incubator-gobblin

private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
  Path destDir) throws IOException {
 for (String localFilePath : SPLITTER.split(localFilePathList)) {
  Path srcFilePath = new Path(localFilePath);
  Path destFilePath = new Path(destDir, srcFilePath.getName());
  this.fs.copyFromLocalFile(srcFilePath, destFilePath);
  if (resourceMap.isPresent()) {
   YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

private void addAppJars(String jarFilePathList, Optional<Map<String, LocalResource>> resourceMap,
  Path destDir) throws IOException {
 for (String jarFilePath : SPLITTER.split(jarFilePathList)) {
  Path srcFilePath = new Path(jarFilePath);
  Path destFilePath = new Path(destDir, srcFilePath.getName());
  this.fs.copyFromLocalFile(srcFilePath, destFilePath);
  if (resourceMap.isPresent()) {
   YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

private void addLibJars(Path srcLibJarDir, Optional<Map<String, LocalResource>> resourceMap, Path destDir)
  throws IOException {
 FileSystem localFs = FileSystem.getLocal(this.yarnConfiguration);
 FileStatus[] libJarFiles = localFs.listStatus(srcLibJarDir);
 if (libJarFiles == null || libJarFiles.length == 0) {
  return;
 }
 for (FileStatus libJarFile : libJarFiles) {
  Path destFilePath = new Path(destDir, libJarFile.getPath().getName());
  this.fs.copyFromLocalFile(libJarFile.getPath(), destFilePath);
  if (resourceMap.isPresent()) {
   YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

private void addJobConfPackage(String jobConfPackagePath, Path destDir, Map<String, LocalResource> resourceMap)
  throws IOException {
 Path srcFilePath = new Path(jobConfPackagePath);
 Path destFilePath = new Path(destDir, srcFilePath.getName() + GobblinClusterConfigurationKeys.TAR_GZ_FILE_SUFFIX);
 StreamUtils.tar(FileSystem.getLocal(this.yarnConfiguration), this.fs, srcFilePath, destFilePath);
 YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.ARCHIVE, resourceMap);
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private void addAppRemoteFiles(String hdfsFileList, Map<String, LocalResource> resourceMap)
  throws IOException {
 for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
  YarnHelixUtils.addFileAsLocalResource(this.fs, new Path(hdfsFilePath), LocalResourceType.FILE, resourceMap);
 }
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private void addRemoteAppFiles(String hdfsFileList, Map<String, LocalResource> resourceMap) throws IOException {
 for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
  Path srcFilePath = new Path(hdfsFilePath);
  YarnHelixUtils.addFileAsLocalResource(
    srcFilePath.getFileSystem(this.yarnConfiguration), srcFilePath, LocalResourceType.FILE, resourceMap);
 }
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException {
 if (!this.fs.exists(destDir)) {
  LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
  return;
 }
 FileStatus[] statuses = this.fs.listStatus(destDir);
 if (statuses != null) {
  for (FileStatus status : statuses) {
   YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
  }
 }
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private void addAppJars(String jarFilePathList, Optional<Map<String, LocalResource>> resourceMap,
  Path destDir) throws IOException {
 for (String jarFilePath : SPLITTER.split(jarFilePathList)) {
  Path srcFilePath = new Path(jarFilePath);
  Path destFilePath = new Path(destDir, srcFilePath.getName());
  this.fs.copyFromLocalFile(srcFilePath, destFilePath);
  if (resourceMap.isPresent()) {
   YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
  }
 }
}