gpt4 book ai didi

co.cask.cdap.api.workflow.WorkflowSpecification.getLocalDatasetSpecs()方法的使用及代码示例

转载 作者:知者 更新时间:2024-03-26 05:21:05 29 4
gpt4 key购买 nike

本文整理了Java中co.cask.cdap.api.workflow.WorkflowSpecification.getLocalDatasetSpecs()方法的一些代码示例,展示了WorkflowSpecification.getLocalDatasetSpecs()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。WorkflowSpecification.getLocalDatasetSpecs()方法的具体详情如下:
包路径:co.cask.cdap.api.workflow.WorkflowSpecification
类名称:WorkflowSpecification
方法名:getLocalDatasetSpecs

WorkflowSpecification.getLocalDatasetSpecs介绍

[英]Return the map of local dataset names and associated specifications required for dataset instance creation.
[中]返回创建数据集实例所需的本地数据集名称和相关规范的映射。

代码示例

代码示例来源:origin: co.cask.cdap/cdap-proto

@Override
public JsonElement serialize(WorkflowSpecification src, Type typeOfSrc, JsonSerializationContext context) {
 JsonObject jsonObj = new JsonObject();
 jsonObj.add("className", new JsonPrimitive(src.getClassName()));
 jsonObj.add("name", new JsonPrimitive(src.getName()));
 jsonObj.add("description", new JsonPrimitive(src.getDescription()));
 jsonObj.add("plugins", serializeMap(src.getPlugins(), context, Plugin.class));
 jsonObj.add("properties", serializeMap(src.getProperties(), context, String.class));
 jsonObj.add("nodes", serializeList(src.getNodes(), context, WorkflowNode.class));
 jsonObj.add("localDatasetSpecs", serializeMap(src.getLocalDatasetSpecs(), context, DatasetCreationSpec.class));
 return jsonObj;
}

代码示例来源:origin: cdapio/cdap

@DELETE
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void deleteWorkflowLocalDatasets(HttpRequest request, HttpResponder responder,
                   @PathParam("namespace-id") String namespaceId,
                   @PathParam("app-id") String applicationId,
                   @PathParam("workflow-id") String workflowId,
                   @PathParam("run-id") String runId) throws NotFoundException {
 WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
 Set<String> errorOnDelete = new HashSet<>();
 for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
  String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
  // try best to delete the local datasets.
  try {
   datasetFramework.deleteInstance(new DatasetId(namespaceId, mappedDatasetName));
  } catch (InstanceNotFoundException e) {
   // Dataset instance is already deleted. so its no-op.
  } catch (Throwable t) {
   errorOnDelete.add(mappedDatasetName);
   LOG.error("Failed to delete the Workflow local dataset {}. Reason - {}", mappedDatasetName, t.getMessage());
  }
 }
 if (errorOnDelete.isEmpty()) {
  responder.sendStatus(HttpResponseStatus.OK);
  return;
 }
 String errorMessage = "Failed to delete Workflow local datasets - " + Joiner.on(",").join(errorOnDelete);
 throw new RuntimeException(errorMessage);
}

代码示例来源:origin: caskdata/cdap

@Override
public JsonElement serialize(WorkflowSpecification src, Type typeOfSrc, JsonSerializationContext context) {
 JsonObject jsonObj = new JsonObject();
 jsonObj.add("className", new JsonPrimitive(src.getClassName()));
 jsonObj.add("name", new JsonPrimitive(src.getName()));
 jsonObj.add("description", new JsonPrimitive(src.getDescription()));
 jsonObj.add("plugins", serializeMap(src.getPlugins(), context, Plugin.class));
 jsonObj.add("properties", serializeMap(src.getProperties(), context, String.class));
 jsonObj.add("nodes", serializeList(src.getNodes(), context, WorkflowNode.class));
 jsonObj.add("localDatasetSpecs", serializeMap(src.getLocalDatasetSpecs(), context, DatasetCreationSpec.class));
 return jsonObj;
}

代码示例来源:origin: co.cask.cdap/cdap-app-fabric

@DELETE
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void deleteWorkflowLocalDatasets(HttpRequest request, HttpResponder responder,
                   @PathParam("namespace-id") String namespaceId,
                   @PathParam("app-id") String applicationId,
                   @PathParam("workflow-id") String workflowId,
                   @PathParam("run-id") String runId) throws NotFoundException {
 WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
 Set<String> errorOnDelete = new HashSet<>();
 for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
  String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
  // try best to delete the local datasets.
  try {
   datasetFramework.deleteInstance(new DatasetId(namespaceId, mappedDatasetName));
  } catch (InstanceNotFoundException e) {
   // Dataset instance is already deleted. so its no-op.
  } catch (Throwable t) {
   errorOnDelete.add(mappedDatasetName);
   LOG.error("Failed to delete the Workflow local dataset {}. Reason - {}", mappedDatasetName, t.getMessage());
  }
 }
 if (errorOnDelete.isEmpty()) {
  responder.sendStatus(HttpResponseStatus.OK);
  return;
 }
 String errorMessage = "Failed to delete Workflow local datasets - " + Joiner.on(",").join(errorOnDelete);
 throw new RuntimeException(errorMessage);
}

代码示例来源:origin: co.cask.cdap/cdap-app-fabric

@GET
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void getWorkflowLocalDatasets(HttpRequest request, HttpResponder responder,
                   @PathParam("namespace-id") String namespaceId,
                   @PathParam("app-id") String applicationId,
                   @PathParam("workflow-id") String workflowId,
                   @PathParam("run-id") String runId)
 throws NotFoundException, DatasetManagementException {
 WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
 Map<String, DatasetSpecificationSummary> localDatasetSummaries = new HashMap<>();
 for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
  String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
  String datasetType = localDatasetEntry.getValue().getTypeName();
  Map<String, String> datasetProperties = localDatasetEntry.getValue().getProperties().getProperties();
  if (datasetFramework.hasInstance(new DatasetId(namespaceId, mappedDatasetName))) {
   localDatasetSummaries.put(localDatasetEntry.getKey(),
                new DatasetSpecificationSummary(mappedDatasetName, datasetType, datasetProperties));
  }
 }
 responder.sendJson(HttpResponseStatus.OK, GSON.toJson(localDatasetSummaries));
}

代码示例来源:origin: cdapio/cdap

@GET
@Path("/apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/localdatasets")
public void getWorkflowLocalDatasets(HttpRequest request, HttpResponder responder,
                   @PathParam("namespace-id") String namespaceId,
                   @PathParam("app-id") String applicationId,
                   @PathParam("workflow-id") String workflowId,
                   @PathParam("run-id") String runId)
 throws NotFoundException, DatasetManagementException {
 WorkflowSpecification workflowSpec = getWorkflowSpecForValidRun(namespaceId, applicationId, workflowId, runId);
 Map<String, DatasetSpecificationSummary> localDatasetSummaries = new HashMap<>();
 for (Map.Entry<String, DatasetCreationSpec> localDatasetEntry : workflowSpec.getLocalDatasetSpecs().entrySet()) {
  String mappedDatasetName = localDatasetEntry.getKey() + "." + runId;
  String datasetType = localDatasetEntry.getValue().getTypeName();
  Map<String, String> datasetProperties = localDatasetEntry.getValue().getProperties().getProperties();
  if (datasetFramework.hasInstance(new DatasetId(namespaceId, mappedDatasetName))) {
   localDatasetSummaries.put(localDatasetEntry.getKey(),
                new DatasetSpecificationSummary(mappedDatasetName, datasetType, datasetProperties));
  }
 }
 responder.sendJson(HttpResponseStatus.OK, GSON.toJson(localDatasetSummaries));
}

代码示例来源:origin: cdapio/cdap

/**
 * Creates a new instance based on the given {@link WorkflowProgramInfo}.
 */
public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework,
                                    WorkflowProgramInfo info,
                                    ApplicationSpecification appSpec) {
 Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet();
 return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId());
}

代码示例来源:origin: co.cask.cdap/cdap-app-fabric

/**
 * Creates a new instance based on the given {@link WorkflowProgramInfo}.
 */
public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework,
                                    WorkflowProgramInfo info,
                                    ApplicationSpecification appSpec) {
 Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet();
 return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId());
}

代码示例来源:origin: cdapio/cdap

private void createLocalDatasets() throws IOException, DatasetManagementException {
 final KerberosPrincipalId principalId = ProgramRunners.getApplicationPrincipal(programOptions);
 for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) {
  final String localInstanceName = entry.getValue();
  final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName);
  final DatasetCreationSpec instanceSpec = workflowSpec.getLocalDatasetSpecs().get(entry.getKey());
  LOG.debug("Adding Workflow local dataset instance: {}", localInstanceName);
  try {
   Retries.callWithRetries(new Retries.Callable<Void, Exception>() {
    @Override
    public Void call() throws Exception {
     DatasetProperties properties = addLocalDatasetProperty(instanceSpec.getProperties(),
                                 keepLocal(entry.getKey()));
     // we have to do this check since addInstance method can only be used when app impersonation is enabled
     if (principalId != null) {
      datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties, principalId);
     } else {
      datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties);
     }
     return null;
    }
   }, RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS));
  } catch (IOException | DatasetManagementException e) {
   throw e;
  } catch (Exception e) {
   // this should never happen
   throw new IllegalStateException(e);
  }
 }
}

代码示例来源:origin: co.cask.cdap/cdap-app-fabric

private void createLocalDatasets() throws IOException, DatasetManagementException {
 final KerberosPrincipalId principalId = ProgramRunners.getApplicationPrincipal(programOptions);
 for (final Map.Entry<String, String> entry : datasetFramework.getDatasetNameMapping().entrySet()) {
  final String localInstanceName = entry.getValue();
  final DatasetId instanceId = new DatasetId(workflowRunId.getNamespace(), localInstanceName);
  final DatasetCreationSpec instanceSpec = workflowSpec.getLocalDatasetSpecs().get(entry.getKey());
  LOG.debug("Adding Workflow local dataset instance: {}", localInstanceName);
  try {
   Retries.callWithRetries(new Retries.Callable<Void, Exception>() {
    @Override
    public Void call() throws Exception {
     DatasetProperties properties = addLocalDatasetProperty(instanceSpec.getProperties(),
                                 keepLocal(entry.getKey()));
     // we have to do this check since addInstance method can only be used when app impersonation is enabled
     if (principalId != null) {
      datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties, principalId);
     } else {
      datasetFramework.addInstance(instanceSpec.getTypeName(), instanceId, properties);
     }
     return null;
    }
   }, RetryStrategies.fixDelay(Constants.Retry.LOCAL_DATASET_OPERATION_RETRY_DELAY_SECONDS, TimeUnit.SECONDS));
  } catch (IOException | DatasetManagementException e) {
   throw e;
  } catch (Exception e) {
   // this should never happen
   throw new IllegalStateException(e);
  }
 }
}

代码示例来源:origin: cdapio/cdap

"SP1")));
Map<String, DatasetCreationSpec> localDatasetSpecs = spec.getLocalDatasetSpecs();
Assert.assertEquals(5, localDatasetSpecs.size());

29 4 0
Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
广告合作:1813099741@qq.com 6ren.com