本文整理了Java中org.apache.twill.internal.yarn.YarnUtils类的一些代码示例，展示了YarnUtils类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。YarnUtils类的具体详情如下：
包路径：org.apache.twill.internal.yarn.YarnUtils
类名称：YarnUtils

YarnUtils介绍

[英]Collection of helper methods to simplify YARN calls.
[中]简化纱线调用的助手方法集合。

代码示例

代码示例来源：origin: org.apache.twill/twill-yarn

/**
 * Creates a {@link Credentials} for the application submission.
 */
private Credentials createSubmissionCredentials() {
 Credentials credentials = new Credentials();
 try {
  // Acquires delegation token for the location
  List<Token<?>> tokens = YarnUtils.addDelegationTokens(config, appLocation.getLocationFactory(), credentials);
  if (LOG.isDebugEnabled()) {
   for (Token<?> token : tokens) {
    LOG.debug("Delegation token acquired for {}, {}", appLocation, token);
   }
  }
 } catch (IOException e) {
  LOG.warn("Failed to acquire delegation token for location {}", appLocation);
 }
 // Copy the user provided credentials.
 // It will override the location delegation tokens acquired above if user supplies it.
 credentials.addAll(this.credentials);
 return credentials;
}

代码示例来源：origin: apache/twill

/**
 * Helper method to create adapter class for bridging between Hadoop 2.0 and 2.1.
 */
private static <T> T createAdapter(Class<T> clz) {
 String className = clz.getPackage().getName();
 if (getHadoopVersion().equals(HadoopVersions.HADOOP_20)) {
  className += ".Hadoop20" + clz.getSimpleName();
 } else {
  className += ".Hadoop21" + clz.getSimpleName();
 }
 try {
  //noinspection unchecked
  return (T) Class.forName(className).newInstance();
 } catch (Exception e) {
  throw Throwables.propagate(e);
 }
}

代码示例来源：origin: org.apache.twill/twill-yarn

private void addRMToken(ContainerLaunchContext context, YarnClient yarnClient) {
 if (!UserGroupInformation.isSecurityEnabled()) {
  return;
 }
 try {
  Credentials credentials = YarnUtils.decodeCredentials(context.getContainerTokens());
  Configuration config = yarnClient.getConfig();
  Token<TokenIdentifier> token = convertToken(
   yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))),
   YarnUtils.getRMAddress(config));
  LOG.debug("Added RM delegation token {}", token);
  credentials.addToken(token.getService(), token);
  context.setContainerTokens(YarnUtils.encodeCredentials(credentials));
 } catch (IOException e) {
  throw new RuntimeException("Failed to acquire RM delegation token", e);
 }
}

代码示例来源：origin: apache/twill

/**
 * Clones the delegation token to individual host behind the same logical address.
 *
 * @param config the hadoop configuration
 * @throws IOException if failed to get information for the current user.
 */
public static void cloneHaNnCredentials(Configuration config) throws IOException {
 String scheme = URI.create(config.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
                    CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT)).getScheme();
 // Loop through all name services. Each name service could have multiple name node associated with it.
 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : getHaNnRpcAddresses(config).entrySet()) {
  String nsId = entry.getKey();
  Map<String, InetSocketAddress> addressesInNN = entry.getValue();
  if (!HAUtil.isHAEnabled(config, nsId) || addressesInNN == null || addressesInNN.isEmpty()) {
   continue;
  }
  // The client may have a delegation token set for the logical
  // URI of the cluster. Clone this token to apply to each of the
  // underlying IPC addresses so that the IPC code can find it.
  URI uri = URI.create(scheme + "://" + nsId);
  LOG.info("Cloning delegation token for uri {}", uri);
  cloneDelegationTokenForLogicalUri(UserGroupInformation.getCurrentUser(), uri, addressesInNN.values());
 }
}

代码示例来源：origin: org.apache.twill/twill-yarn

/**
 * Helper method to get delegation tokens for the given LocationFactory.
 * @param config The hadoop configuration.
 * @param locationFactory The LocationFactory for generating tokens.
 * @param credentials Credentials for storing tokens acquired.
 * @return List of delegation Tokens acquired.
 */
public static List<Token<?>> addDelegationTokens(Configuration config,
                         LocationFactory locationFactory,
                         Credentials credentials) throws IOException {
 if (!UserGroupInformation.isSecurityEnabled()) {
  LOG.debug("Security is not enabled");
  return ImmutableList.of();
 }
 try (FileSystem fileSystem = getFileSystem(locationFactory)) {
  if (fileSystem == null) {
   return ImmutableList.of();
  }
  String renewer = YarnUtils.getYarnTokenRenewer(config);
  Token<?>[] tokens = fileSystem.addDelegationTokens(renewer, credentials);
  LOG.debug("Added HDFS DelegationTokens: {}", Arrays.toString(tokens));
  return tokens == null ? ImmutableList.<Token<?>>of() : ImmutableList.copyOf(tokens);
 }
}

代码示例来源：origin: org.apache.twill/twill-yarn

@Override
protected Resource adjustCapability(Resource resource) {
 int cores = YarnUtils.getVirtualCores(resource);
 int updatedCores = Math.max(Math.min(cores, YarnUtils.getVirtualCores(maxCapability)),
               YarnUtils.getVirtualCores(minCapability));
 // Try and set the virtual cores, which older versions of YARN don't support this.
 if (cores != updatedCores && YarnUtils.setVirtualCores(resource, updatedCores)) {
  LOG.info("Adjust virtual cores requirement from {} to {}.", cores, updatedCores);
 }
 int updatedMemory = Math.min(resource.getMemory(), maxCapability.getMemory());
 int minMemory = minCapability.getMemory();
 updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory;
 if (resource.getMemory() != updatedMemory) {
  LOG.info("Adjust memory requirement from {} to {} MB.", resource.getMemory(), updatedMemory);
  resource.setMemory(updatedMemory);
 }
 return resource;
}

代码示例来源：origin: org.apache.twill/twill-yarn

services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString());
Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());
credentials.addToken(token.getService(), token);
context.setTokens(YarnUtils.encodeCredentials(credentials));

代码示例来源：origin: org.apache.twill/twill-yarn

public static String getYarnTokenRenewer(Configuration config) throws IOException {
 String rmHost = getRMAddress(config).getHostName();
 String renewer = SecurityUtil.getServerPrincipal(config.get(YarnConfiguration.RM_PRINCIPAL), rmHost);
 if (renewer == null || renewer.length() == 0) {
  throw new IOException("No Kerberos principal for Yarn RM to use as renewer");
 }
 return renewer;
}

代码示例来源：origin: org.apache.twill/twill-yarn

@Override
 public ProcessController<YarnApplicationReport> call() throws Exception {
  return yarnAppClient.createProcessController(
   YarnUtils.createApplicationId(amLiveNodeData.getAppIdClusterTime(), amLiveNodeData.getAppId()));
 }
};

代码示例来源：origin: org.apache.twill/twill-yarn

@Override
 public int getVirtualCores() {
  return YarnUtils.getVirtualCores(container.getResource());
 }
}

代码示例来源：origin: org.apache.twill/twill-yarn

public static YarnLocalResource createLocalResource(LocalFile localFile) {
 Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0.");
 Preconditions.checkArgument(localFile.getSize() >= 0, "File size should be >= 0.");
 YarnLocalResource resource = createAdapter(YarnLocalResource.class);
 resource.setVisibility(LocalResourceVisibility.APPLICATION);
 resource.setResource(ConverterUtils.getYarnUrlFromURI(localFile.getURI()));
 resource.setTimestamp(localFile.getLastModified());
 resource.setSize(localFile.getSize());
 return setLocalResourceType(resource, localFile);
}

代码示例来源：origin: co.cask.cdap/cdap-app-fabric

/**
 * Gets a JHS delegation token and stores it in the given Credentials.
 *
 * @return the same Credentials instance as the one given in parameter.
 */
public static Credentials obtainToken(Configuration configuration, Credentials credentials) {
 if (!UserGroupInformation.isSecurityEnabled()) {
  return credentials;
 }
 String historyServerAddress = configuration.get("mapreduce.jobhistory.address");
 HostAndPort hostAndPort = HostAndPort.fromString(historyServerAddress);
 try {
  ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(new YarnConfiguration(configuration));
  MRClientCache clientCache = new MRClientCache(configuration, resourceMgrDelegate);
  MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
  GetDelegationTokenRequest request = new GetDelegationTokenRequestPBImpl();
  request.setRenewer(YarnUtils.getYarnTokenRenewer(configuration));
  InetSocketAddress address = new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPort());
  Token<TokenIdentifier> token =
   ConverterUtils.convertFromYarn(hsProxy.getDelegationToken(request).getDelegationToken(), address);
  credentials.addToken(new Text(token.getService()), token);
  LOG.debug("Adding JobHistoryServer delegation token {}.", token);
  return credentials;
 } catch (Exception e) {
  throw Throwables.propagate(e);
 }
}

代码示例来源：origin: org.apache.twill/twill-yarn

public static YarnLaunchContext createLaunchContext() {
 return createAdapter(YarnLaunchContext.class);
}

代码示例来源：origin: org.apache.twill/twill-yarn

YarnUtils.cloneHaNnCredentials(config);
this.credentials = credentials;

代码示例来源：origin: apache/twill

services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString());
Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());
credentials.addToken(token.getService(), token);
context.setTokens(YarnUtils.encodeCredentials(credentials));

代码示例来源：origin: apache/twill

/**
 * Helper method to get delegation tokens for the given LocationFactory.
 * @param config The hadoop configuration.
 * @param locationFactory The LocationFactory for generating tokens.
 * @param credentials Credentials for storing tokens acquired.
 * @return List of delegation Tokens acquired.
 */
public static List<Token<?>> addDelegationTokens(Configuration config,
                         LocationFactory locationFactory,
                         Credentials credentials) throws IOException {
 if (!UserGroupInformation.isSecurityEnabled()) {
  LOG.debug("Security is not enabled");
  return ImmutableList.of();
 }
 try (FileSystem fileSystem = getFileSystem(locationFactory)) {
  if (fileSystem == null) {
   return ImmutableList.of();
  }
  String renewer = YarnUtils.getYarnTokenRenewer(config);
  Token<?>[] tokens = fileSystem.addDelegationTokens(renewer, credentials);
  LOG.debug("Added HDFS DelegationTokens: {}", Arrays.toString(tokens));
  return tokens == null ? ImmutableList.<Token<?>>of() : ImmutableList.copyOf(tokens);
 }
}

代码示例来源：origin: apache/twill

public static String getYarnTokenRenewer(Configuration config) throws IOException {
 String rmHost = getRMAddress(config).getHostName();
 String renewer = SecurityUtil.getServerPrincipal(config.get(YarnConfiguration.RM_PRINCIPAL), rmHost);
 if (renewer == null || renewer.length() == 0) {
  throw new IOException("No Kerberos principal for Yarn RM to use as renewer");
 }
 return renewer;
}

代码示例来源：origin: apache/twill

@Override
 public ProcessController<YarnApplicationReport> call() throws Exception {
  return yarnAppClient.createProcessController(
   YarnUtils.createApplicationId(amLiveNodeData.getAppIdClusterTime(), amLiveNodeData.getAppId()));
 }
};

代码示例来源：origin: cdapio/cdap

private void reportYarnResources() {
 try {
  long totalMemory = 0L;
  long totalVCores = 0L;
  long usedMemory = 0L;
  long usedVCores = 0L;
  for (NodeReport nodeReport : yarnClient.getNodeReports(NodeState.RUNNING)) {
   Resource capability = nodeReport.getCapability();
   Resource used = nodeReport.getUsed();
   totalMemory += capability.getMemory();
   totalVCores += YarnUtils.getVirtualCores(capability);
   usedMemory += used.getMemory();
   usedVCores += YarnUtils.getVirtualCores(used);
  }
  MetricsContext collector = getCollector();
  LOG.trace("YARN Cluster memory total={}MB, used={}MB", totalMemory, usedMemory);
  collector.gauge("resources.total.memory", totalMemory);
  collector.gauge("resources.used.memory", usedMemory);
  collector.gauge("resources.available.memory", totalMemory - usedMemory);
  LOG.trace("YARN Cluster vcores total={}, used={}", totalVCores, usedVCores);
  collector.gauge("resources.total.vcores", totalVCores);
  collector.gauge("resources.used.vcores", usedVCores);
  collector.gauge("resources.available.vcores", totalVCores - usedVCores);
 } catch (Exception e) {
  LOG.warn("Failed to gather YARN NodeReports", e);
 }
}

代码示例来源：origin: apache/twill

public static YarnLocalResource createLocalResource(LocalFile localFile) {
 Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0.");
 Preconditions.checkArgument(localFile.getSize() >= 0, "File size should be >= 0.");
 YarnLocalResource resource = createAdapter(YarnLocalResource.class);
 resource.setVisibility(LocalResourceVisibility.APPLICATION);
 resource.setResource(ConverterUtils.getYarnUrlFromURI(localFile.getURI()));
 resource.setTimestamp(localFile.getLastModified());
 resource.setSize(localFile.getSize());
 return setLocalResourceType(resource, localFile);
}