gpt4 book ai didi

org.apache.samza.job.yarn.YarnContainer类的使用及代码示例

转载 作者:知者 更新时间:2024-03-19 01:49:31 26 4
gpt4 key购买 nike

本文整理了Java中org.apache.samza.job.yarn.YarnContainer类的一些代码示例,展示了YarnContainer类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。YarnContainer类的具体详情如下:
包路径:org.apache.samza.job.yarn.YarnContainer
类名称:YarnContainer

YarnContainer介绍

暂无

代码示例

代码示例来源:origin: org.apache.samza/samza-yarn_2.11

/**
 * Returns the Id of the Samza container that corresponds to the provided Yarn {@link ContainerId}
 * @param containerId the Yarn ContainerId
 * @return the id of the Samza container corresponding to the {@link ContainerId} that is pending launch
 */
private String getPendingSamzaContainerId(ContainerId containerId) {
 for (String samzaContainerId: state.pendingYarnContainers.keySet()) {
  YarnContainer yarnContainer = state.pendingYarnContainers.get(samzaContainerId);
  if (yarnContainer != null && yarnContainer.id().equals(containerId)) {
   return samzaContainerId;
  }
 }
 return null;
}

代码示例来源:origin: org.apache.samza/samza-yarn

@Override
public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) {
 log.info("Received a containerStart notification from the NodeManager for container: {} ", containerId);
 String samzaContainerId = getPendingSamzaContainerId(containerId);
 if (samzaContainerId != null) {
  // 1. Move the container from pending to running state
  final YarnContainer container = state.pendingYarnContainers.remove(samzaContainerId);
  log.info("Samza containerId:{} has started", samzaContainerId);
  state.runningYarnContainers.put(samzaContainerId, container);
  // 2. Invoke the success callback.
  SamzaResource resource = new SamzaResource(container.resource().getVirtualCores(),
    container.resource().getMemory(), container.nodeId().getHost(), containerId.toString());
  clusterManagerCallback.onStreamProcessorLaunchSuccess(resource);
 } else {
  log.info("Got an invalid notification from YARN for container: {}", containerId);
 }
}

代码示例来源:origin: org.apache.samza/samza-yarn_2.11

state.pendingYarnContainers.put(samzaContainerId, new YarnContainer(container));

代码示例来源:origin: org.apache.samza/samza-yarn

state.pendingYarnContainers.put(samzaContainerId, new YarnContainer(container));

代码示例来源:origin: org.apache.samza/samza-yarn_2.11

@Override
public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) {
 log.info("Received a containerStart notification from the NodeManager for container: {} ", containerId);
 String samzaContainerId = getPendingSamzaContainerId(containerId);
 if (samzaContainerId != null) {
  // 1. Move the container from pending to running state
  final YarnContainer container = state.pendingYarnContainers.remove(samzaContainerId);
  log.info("Samza containerId:{} has started", samzaContainerId);
  state.runningYarnContainers.put(samzaContainerId, container);
  // 2. Invoke the success callback.
  SamzaResource resource = new SamzaResource(container.resource().getVirtualCores(),
    container.resource().getMemory(), container.nodeId().getHost(), containerId.toString());
  clusterManagerCallback.onStreamProcessorLaunchSuccess(resource);
 } else {
  log.info("Got an invalid notification from YARN for container: {}", containerId);
 }
}

代码示例来源:origin: org.apache.samza/samza-yarn

/**
 * Returns the Id of the Samza container that corresponds to the provided Yarn {@link ContainerId}
 * @param containerId the Yarn ContainerId
 * @return the id of the Samza container corresponding to the {@link ContainerId} that is pending launch
 */
private String getPendingSamzaContainerId(ContainerId containerId) {
 for (String samzaContainerId: state.pendingYarnContainers.keySet()) {
  YarnContainer yarnContainer = state.pendingYarnContainers.get(samzaContainerId);
  if (yarnContainer != null && yarnContainer.id().equals(containerId)) {
   return samzaContainerId;
  }
 }
 return null;
}

代码示例来源:origin: org.apache.samza/samza-yarn_2.11

@Override
public void onStartContainerError(ContainerId yarnContainerId, Throwable t) {
 log.error(String.format("Yarn Container: %s could not start.", yarnContainerId), t);
 String samzaContainerId = getPendingSamzaContainerId(yarnContainerId);
 if (samzaContainerId != null) {
  YarnContainer container = state.pendingYarnContainers.remove(samzaContainerId);
  log.info("Failed Yarn Container: {} had Samza ContainerId: {} ", yarnContainerId, samzaContainerId);
  SamzaResource resource = new SamzaResource(container.resource().getVirtualCores(),
    container.resource().getMemory(), container.nodeId().getHost(), yarnContainerId.toString());
  log.info("Invoking failure callback for container: {}", yarnContainerId);
  clusterManagerCallback.onStreamProcessorLaunchFailure(resource, new SamzaContainerLaunchException(t));
 } else {
  log.info("Got an invalid notification for container: {}", yarnContainerId);
 }
}

代码示例来源:origin: org.apache.samza/samza-yarn

/**
 * Given a lookupContainerId from Yarn (for example: containerId_app_12345, this method returns the SamzaContainer ID
 * in the range [0,N-1] that maps to it.
 * @param lookupContainerId  the Yarn container ID.
 * @return  the samza container ID.
 */
//TODO: Get rid of the YarnContainer object and just use Container in state.runningYarnContainers hashmap.
//In that case, this scan will turn into a lookup. This change will require changes/testing in the UI files because
//those UI stub templates operate on the YarnContainer object.
private String getIDForContainer(String lookupContainerId) {
 String samzaContainerID = INVALID_YARN_CONTAINER_ID;
 for(Map.Entry<String, YarnContainer> entry : state.runningYarnContainers.entrySet()) {
  String key = entry.getKey();
  YarnContainer yarnContainer = entry.getValue();
  String yarnContainerId = yarnContainer.id().toString();
  if(yarnContainerId.equals(lookupContainerId)) {
   return key;
  }
 }
 return samzaContainerID;
}

代码示例来源:origin: org.apache.samza/samza-yarn

@Override
public void onStartContainerError(ContainerId yarnContainerId, Throwable t) {
 log.error(String.format("Yarn Container: %s could not start.", yarnContainerId), t);
 String samzaContainerId = getPendingSamzaContainerId(yarnContainerId);
 if (samzaContainerId != null) {
  YarnContainer container = state.pendingYarnContainers.remove(samzaContainerId);
  log.info("Failed Yarn Container: {} had Samza ContainerId: {} ", yarnContainerId, samzaContainerId);
  SamzaResource resource = new SamzaResource(container.resource().getVirtualCores(),
    container.resource().getMemory(), container.nodeId().getHost(), yarnContainerId.toString());
  log.info("Invoking failure callback for container: {}", yarnContainerId);
  clusterManagerCallback.onStreamProcessorLaunchFailure(resource, new SamzaContainerLaunchException(t));
 } else {
  log.info("Got an invalid notification for container: {}", yarnContainerId);
 }
}

代码示例来源:origin: org.apache.samza/samza-yarn_2.11

/**
 * Given a lookupContainerId from Yarn (for example: containerId_app_12345, this method returns the SamzaContainer ID
 * in the range [0,N-1] that maps to it.
 * @param lookupContainerId  the Yarn container ID.
 * @return  the samza container ID.
 */
//TODO: Get rid of the YarnContainer object and just use Container in state.runningYarnContainers hashmap.
//In that case, this scan will turn into a lookup. This change will require changes/testing in the UI files because
//those UI stub templates operate on the YarnContainer object.
private String getIDForContainer(String lookupContainerId) {
 String samzaContainerID = INVALID_YARN_CONTAINER_ID;
 for(Map.Entry<String, YarnContainer> entry : state.runningYarnContainers.entrySet()) {
  String key = entry.getKey();
  YarnContainer yarnContainer = entry.getValue();
  String yarnContainerId = yarnContainer.id().toString();
  if(yarnContainerId.equals(lookupContainerId)) {
   return key;
  }
 }
 return samzaContainerID;
}

代码示例来源:origin: org.apache.samza/samza-yarn_2.11

@Override
 protected void doGet(HttpServletRequest req, HttpServletResponse resp)
   throws ServletException, IOException {
  ContainerId yarnContainerId;
  PrintWriter printWriter = resp.getWriter();
  String containerIdParam = req.getParameter(YARN_CONTAINER_ID);
  ContainerHeartbeatResponse response;
  resp.setContentType(APPLICATION_JSON);
  boolean alive = false;
  try {
   yarnContainerId = ContainerId.fromString(containerIdParam);
   for (YarnContainer yarnContainer : yarnAppState.runningYarnContainers.values()) {
    if (yarnContainer.id().compareTo(yarnContainerId) == 0) {
     alive = true;
     break;
    }
   }
   if (!alive) {
    heartbeatsExpiredCount.inc();
   }
   response = new ContainerHeartbeatResponse(alive);
   printWriter.write(mapper.writeValueAsString(response));
  } catch (IllegalArgumentException e) {
   LOG.error("Container ID {} passed is invalid", containerIdParam);
   resp.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage());
  }
 }
}

代码示例来源:origin: org.apache.samza/samza-yarn

@Override
 protected void doGet(HttpServletRequest req, HttpServletResponse resp)
   throws ServletException, IOException {
  ContainerId yarnContainerId;
  PrintWriter printWriter = resp.getWriter();
  String containerIdParam = req.getParameter(YARN_CONTAINER_ID);
  ContainerHeartbeatResponse response;
  resp.setContentType(APPLICATION_JSON);
  boolean alive = false;
  try {
   yarnContainerId = ContainerId.fromString(containerIdParam);
   for (YarnContainer yarnContainer : yarnAppState.runningYarnContainers.values()) {
    if (yarnContainer.id().compareTo(yarnContainerId) == 0) {
     alive = true;
     break;
    }
   }
   if (!alive) {
    heartbeatsExpiredCount.inc();
   }
   response = new ContainerHeartbeatResponse(alive);
   printWriter.write(mapper.writeValueAsString(response));
  } catch (IllegalArgumentException e) {
   LOG.error("Container ID {} passed is invalid", containerIdParam);
   resp.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage());
  }
 }
}

26 4 0
Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
广告合作:1813099741@qq.com 6ren.com