本文整理了Java中org.apache.gobblin.yarn.YarnService类的一些代码示例，展示了YarnService类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。YarnService类的具体详情如下：
包路径：org.apache.gobblin.yarn.YarnService
类名称：YarnService

YarnService介绍

[英]This class is responsible for all Yarn-related stuffs including ApplicationMaster registration, ApplicationMaster un-registration, Yarn container management, etc.
[中]该课程负责所有与纱线相关的工作，包括ApplicationMaster注册、ApplicationMaster取消注册、纱线容器管理等。

代码示例

代码示例来源：origin: apache/incubator-gobblin

/**
 * Build the {@link YarnService} for the Application Master.
 */
private YarnService buildYarnService(Config config, String applicationName, String applicationId,
  YarnConfiguration yarnConfiguration, FileSystem fs)
  throws Exception {
 return new YarnService(config, applicationName, applicationId, yarnConfiguration, fs, this.eventBus);
}

代码示例来源：origin: apache/incubator-gobblin

private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
  throws IOException {
 Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
 Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
 Map<String, LocalResource> resourceMap = Maps.newHashMap();
 addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(
   new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
 if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
  addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap);
 }
 ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
 containerLaunchContext.setLocalResources(resourceMap);
 containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
 containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
 if (UserGroupInformation.isSecurityEnabled()) {
  containerLaunchContext.setTokens(this.tokens.duplicate());
 }
 return containerLaunchContext;
}

代码示例来源：origin: apache/incubator-gobblin

eventMetadataBuilder = Optional.of(buildContainerStatusEventMetadata(containerStatus));
 eventMetadataBuilder.get().put(GobblinYarnEventConstants.EventMetadata.HELIX_INSTANCE_ID, completedInstanceName);
 eventMetadataBuilder.get().put(GobblinYarnEventConstants.EventMetadata.CONTAINER_STATUS_RETRY_ATTEMPT, retryCount + "");
  containerStatus.getContainerId(), completedInstanceName));
this.eventBus.post(new NewContainerRequest(
  shouldStickToTheSameNode(containerStatus.getExitStatus()) ?
    Optional.of(completedContainerEntry.getKey()) : Optional.<Container>absent()));

代码示例来源：origin: apache/incubator-gobblin

public YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
  FileSystem fs, EventBus eventBus) throws Exception {
 this.applicationName = applicationName;
 this.applicationId = applicationId;
 this.config = config;
 this.eventBus = eventBus;
 this.gobblinMetrics = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
   Optional.of(buildGobblinMetrics()) : Optional.<GobblinMetrics>absent();
 this.eventSubmitter = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
   Optional.of(buildEventSubmitter()) : Optional.<EventSubmitter>absent();
 this.yarnConfiguration = yarnConfiguration;
 this.fs = fs;
 this.amrmClientAsync = closer.register(
   AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
 this.amrmClientAsync.init(this.yarnConfiguration);
 this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
 this.nmClientAsync.init(this.yarnConfiguration);
 this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
 this.requestedContainerMemoryMbs = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
 this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
 this.containerHostAffinityEnabled = config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);
 this.helixInstanceMaxRetries = config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);
 this.containerJvmArgs = config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY) ?
   Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)) :
   Optional.<String>absent();
 this.containerLaunchExecutor = Executors.newFixedThreadPool(10,
   ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));
 this.tokens = getSecurityTokens();
}

代码示例来源：origin: apache/incubator-gobblin

private void requestInitialContainers(int containersRequested) {
 for (int i = 0; i < containersRequested; i++) {
  requestContainer(Optional.<String>absent());
 }
}

代码示例来源：origin: apache/incubator-gobblin

@Override
protected void startUp() throws Exception {
 LOGGER.info("Starting the YarnService");
 // Register itself with the EventBus for container-related requests
 this.eventBus.register(this);
 this.amrmClientAsync.start();
 this.nmClientAsync.start();
 // The ApplicationMaster registration response is used to determine the maximum resource capacity of the cluster
 RegisterApplicationMasterResponse response = this.amrmClientAsync.registerApplicationMaster(
   GobblinClusterUtils.getHostname(), -1, "");
 LOGGER.info("ApplicationMaster registration response: " + response);
 this.maxResourceCapacity = Optional.of(response.getMaximumResourceCapability());
 LOGGER.info("Requesting initial containers");
 requestInitialContainers(this.initialContainers);
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

public YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
  FileSystem fs, EventBus eventBus) throws Exception {
 this.applicationName = applicationName;
 this.applicationId = applicationId;
 this.config = config;
 this.eventBus = eventBus;
 this.gobblinMetrics = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
   Optional.of(buildGobblinMetrics()) : Optional.<GobblinMetrics>absent();
 this.eventSubmitter = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
   Optional.of(buildEventSubmitter()) : Optional.<EventSubmitter>absent();
 this.yarnConfiguration = yarnConfiguration;
 this.fs = fs;
 this.amrmClientAsync = closer.register(
   AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
 this.amrmClientAsync.init(this.yarnConfiguration);
 this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
 this.nmClientAsync.init(this.yarnConfiguration);
 this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
 this.requestedContainerMemoryMbs = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
 this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
 this.containerHostAffinityEnabled = config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);
 this.helixInstanceMaxRetries = config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);
 this.containerJvmArgs = config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY) ?
   Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)) :
   Optional.<String>absent();
 this.containerLaunchExecutor = Executors.newFixedThreadPool(10,
   ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));
 this.tokens = getSecurityTokens();
}

代码示例来源：origin: apache/incubator-gobblin

@SuppressWarnings("unused")
@Subscribe
public void handleNewContainerRequest(NewContainerRequest newContainerRequest) {
 if (!this.maxResourceCapacity.isPresent()) {
  LOGGER.error(String.format(
    "Unable to handle new container request as maximum resource capacity is not available: "
      + "[memory (MBs) requested = %d, vcores requested = %d]", this.requestedContainerMemoryMbs,
    this.requestedContainerCores));
  return;
 }
 requestContainer(newContainerRequest.getReplacedContainer().transform(new Function<Container, String>() {
  @Override
  public String apply(Container container) {
   return container.getNodeId().getHost();
  }
 }));
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

@Override
protected void startUp() throws Exception {
 LOGGER.info("Starting the YarnService");
 // Register itself with the EventBus for container-related requests
 this.eventBus.register(this);
 this.amrmClientAsync.start();
 this.nmClientAsync.start();
 // The ApplicationMaster registration response is used to determine the maximum resource capacity of the cluster
 RegisterApplicationMasterResponse response = this.amrmClientAsync.registerApplicationMaster(
   GobblinClusterUtils.getHostname(), -1, "");
 LOGGER.info("ApplicationMaster registration response: " + response);
 this.maxResourceCapacity = Optional.of(response.getMaximumResourceCapability());
 LOGGER.info("Requesting initial containers");
 requestInitialContainers(this.initialContainers);
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
  throws IOException {
 Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
 Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
 Map<String, LocalResource> resourceMap = Maps.newHashMap();
 addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
 addContainerLocalResources(
   new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
 if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
  addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap);
 }
 ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
 containerLaunchContext.setLocalResources(resourceMap);
 containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
 containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
 if (UserGroupInformation.isSecurityEnabled()) {
  containerLaunchContext.setTokens(this.tokens.duplicate());
 }
 return containerLaunchContext;
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

eventMetadataBuilder = Optional.of(buildContainerStatusEventMetadata(containerStatus));
 eventMetadataBuilder.get().put(GobblinYarnEventConstants.EventMetadata.HELIX_INSTANCE_ID, completedInstanceName);
 eventMetadataBuilder.get().put(GobblinYarnEventConstants.EventMetadata.CONTAINER_STATUS_RETRY_ATTEMPT, retryCount + "");
  containerStatus.getContainerId(), completedInstanceName));
this.eventBus.post(new NewContainerRequest(
  shouldStickToTheSameNode(containerStatus.getExitStatus()) ?
    Optional.of(completedContainerEntry.getKey()) : Optional.<Container>absent()));

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

private void requestInitialContainers(int containersRequested) {
 for (int i = 0; i < containersRequested; i++) {
  requestContainer(Optional.<String>absent());
 }
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

/**
 * Build the {@link YarnService} for the Application Master.
 */
private YarnService buildYarnService(Config config, String applicationName, String applicationId,
  YarnConfiguration yarnConfiguration, FileSystem fs)
  throws Exception {
 return new YarnService(config, applicationName, applicationId, yarnConfiguration, fs, this.eventBus);
}

代码示例来源：origin: org.apache.gobblin/gobblin-yarn

@SuppressWarnings("unused")
@Subscribe
public void handleNewContainerRequest(NewContainerRequest newContainerRequest) {
 if (!this.maxResourceCapacity.isPresent()) {
  LOGGER.error(String.format(
    "Unable to handle new container request as maximum resource capacity is not available: "
      + "[memory (MBs) requested = %d, vcores requested = %d]", this.requestedContainerMemoryMbs,
    this.requestedContainerCores));
  return;
 }
 requestContainer(newContainerRequest.getReplacedContainer().transform(new Function<Container, String>() {
  @Override
  public String apply(Container container) {
   return container.getNodeId().getHost();
  }
 }));
}