本文整理了Java中org.apache.gobblin.source.workunit.WorkUnitStream类的一些代码示例，展示了WorkUnitStream类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。WorkUnitStream类的具体详情如下：
包路径：org.apache.gobblin.source.workunit.WorkUnitStream
类名称：WorkUnitStream

WorkUnitStream介绍

[英]A stream of WorkUnits, allows for working with large numbers of work units in a memory-efficient way, as well as processing infinite streams of work units.
[中]工作单元流，允许以内存有效的方式处理大量工作单元，以及处理无限多个工作单元流。

代码示例

代码示例来源：origin: apache/incubator-gobblin

public WriterInitializer getInitializer(State state, WorkUnitStream workUnits, int branches, int branchId) {
  JdbcWriterCommandsFactory factory = new JdbcWriterCommandsFactory();
  if (workUnits.isSafeToMaterialize()) {
   return new JdbcWriterInitializer(state, workUnits.getMaterializedWorkUnitCollection(),
     factory, branches, branchId);
  } else {
   throw new RuntimeException(JdbcWriterBuilder.class.getName() + " does not support work unit streams.");
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

final JobState jobState = this.jobContext.getJobState();
Iterator<WorkUnit> workUnitIterator = workUnitStream.getWorkUnits();
if (!workUnitIterator.hasNext()) {
 LOG.warn("No work units to run");
Iterator<WorkUnit> flattenedWorkUnits = new MultiWorkUnitUnpackingIterator(workUnitStream.getWorkUnits());
Iterator<WorkUnit> workUnitsWithJobState = Iterators.transform(flattenedWorkUnits, new Function<WorkUnit, WorkUnit>() {
 @Override

代码示例来源：origin: apache/incubator-gobblin

/**
 * Materialize a {@link WorkUnitStream} into an in-memory list. Note that infinite work unit streams cannot be materialized.
 */
private List<WorkUnit> materializeWorkUnitList(WorkUnitStream workUnitStream) {
 if (!workUnitStream.isFiniteStream()) {
  throw new UnsupportedOperationException("Cannot materialize an infinite work unit stream.");
 }
 return Lists.newArrayList(workUnitStream.getWorkUnits());
}

代码示例来源：origin: apache/incubator-gobblin

if (workUnitStream == null || workUnitStream.getWorkUnits() == null) {
 this.eventSubmitter.submit(JobEvent.WORK_UNITS_MISSING);
 jobState.setState(JobState.RunningState.FAILED);
if (!workUnitStream.getWorkUnits().hasNext()) {
 this.eventSubmitter.submit(JobEvent.WORK_UNITS_EMPTY);
 LOG.warn("No work units have been created for job " + jobId);
 workUnitStream = workUnitStream.filter(new SkippedWorkUnitsFilter(jobState));
 workUnitStream = workUnitStream.transform(new MultiWorkUnitForEach() {
  @Override
  public void forWorkUnit(WorkUnit workUnit) {
  workUnitStream = workUnitStream.transform(new Function<WorkUnit, WorkUnit>() {
   @Nullable
   @Override

代码示例来源：origin: apache/incubator-gobblin

@Override
public List<WorkUnit> getWorkunits(SourceState state) {
 return Lists.newArrayList(getWorkunitStream(state).getMaterializedWorkUnitCollection());
}

代码示例来源：origin: apache/incubator-gobblin

/**
 * Prepare the flattened {@link WorkUnit}s for execution by populating the job and task IDs.
 */
private WorkUnitStream prepareWorkUnits(WorkUnitStream workUnits, JobState jobState) {
 return workUnits.transform(new WorkUnitPreparator(this.jobContext.getJobId()));
}

代码示例来源：origin: org.apache.gobblin/gobblin-runtime

if (workUnitStream == null || workUnitStream.getWorkUnits() == null) {
 this.eventSubmitter.submit(JobEvent.WORK_UNITS_MISSING);
 jobState.setState(JobState.RunningState.FAILED);
if (!workUnitStream.getWorkUnits().hasNext()) {
 this.eventSubmitter.submit(JobEvent.WORK_UNITS_EMPTY);
 LOG.warn("No work units have been created for job " + jobId);
 workUnitStream = workUnitStream.filter(new SkippedWorkUnitsFilter(jobState));
 workUnitStream = workUnitStream.transform(new MultiWorkUnitForEach() {
  @Override
  public void forWorkUnit(WorkUnit workUnit) {
  workUnitStream = workUnitStream.transform(new Function<WorkUnit, WorkUnit>() {
   @Nullable
   @Override

代码示例来源：origin: org.apache.gobblin/gobblin-data-management

@Override
public List<WorkUnit> getWorkunits(SourceState state) {
 return Lists.newArrayList(getWorkunitStream(state).getMaterializedWorkUnitCollection());
}

代码示例来源：origin: org.apache.gobblin/gobblin-runtime

/**
 * Prepare the flattened {@link WorkUnit}s for execution by populating the job and task IDs.
 */
private WorkUnitStream prepareWorkUnits(WorkUnitStream workUnits, JobState jobState) {
 return workUnits.transform(new WorkUnitPreparator(this.jobContext.getJobId()));
}

代码示例来源：origin: apache/incubator-gobblin

@Override
 public ConverterInitializer getInitializer(State state, WorkUnitStream workUnits, int branches, int branchId) {
  JdbcWriterCommandsFactory factory = new JdbcWriterCommandsFactory();
  if (workUnits.isSafeToMaterialize()) {
   return new AvroToJdbcEntryConverterInitializer(state, workUnits.getMaterializedWorkUnitCollection(),
     factory, branches, branchId);
  } else {
   throw new RuntimeException(AvroToJdbcEntryConverter.class.getName() + " does not support work unit streams.");
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

@Test
public void testNonDrilledDown() {
 Dataset dataset1 = new SimpleDatasetForTesting("dataset1");
 Dataset dataset2 = new SimplePartitionableDatasetForTesting("dataset2", Lists.newArrayList(new SimpleDatasetPartitionForTesting("p1"), new SimpleDatasetPartitionForTesting("p2")));
 Dataset dataset3 = new SimpleDatasetForTesting("dataset3");
 IterableDatasetFinder finder = new StaticDatasetsFinderForTesting(Lists.newArrayList(dataset1, dataset2, dataset3));
 MySource mySource = new MySource(false, finder);
 List<WorkUnit> workUnits = mySource.getWorkunits(new SourceState());
 Assert.assertEquals(workUnits.size(), 3);
 Assert.assertEquals(workUnits.get(0).getProp(DATASET_URN), "dataset1");
 Assert.assertNull(workUnits.get(0).getProp(PARTITION_URN));
 Assert.assertEquals(workUnits.get(1).getProp(DATASET_URN), "dataset2");
 Assert.assertNull(workUnits.get(1).getProp(PARTITION_URN));
 Assert.assertEquals(workUnits.get(2).getProp(DATASET_URN), "dataset3");
 Assert.assertNull(workUnits.get(2).getProp(PARTITION_URN));
 WorkUnitStream workUnitStream = mySource.getWorkunitStream(new SourceState());
 Assert.assertEquals(Lists.newArrayList(workUnitStream.getWorkUnits()), workUnits);
}

代码示例来源：origin: org.apache.gobblin/gobblin-runtime

/**
 * Materialize a {@link WorkUnitStream} into an in-memory list. Note that infinite work unit streams cannot be materialized.
 */
private List<WorkUnit> materializeWorkUnitList(WorkUnitStream workUnitStream) {
 if (!workUnitStream.isFiniteStream()) {
  throw new UnsupportedOperationException("Cannot materialize an infinite work unit stream.");
 }
 return Lists.newArrayList(workUnitStream.getWorkUnits());
}

代码示例来源：origin: apache/incubator-gobblin

if (workUnits.isSafeToMaterialize()) {
 Closer closer = Closer.create();
 Map<String, ParallelRunner> parallelRunners = Maps.newHashMap();
 try {
  for (WorkUnit workUnit : JobLauncherUtils.flattenWorkUnits(workUnits.getMaterializedWorkUnitCollection())) {
   JobLauncherUtils.cleanTaskStagingData(new WorkUnitState(workUnit, jobState), LOG, closer, parallelRunners);

代码示例来源：origin: apache/incubator-gobblin

@Test
public void testDrilledDown() {
 Dataset dataset1 = new SimpleDatasetForTesting("dataset1");
 Dataset dataset2 = new SimplePartitionableDatasetForTesting("dataset2", Lists.newArrayList(new SimpleDatasetPartitionForTesting("p1"), new SimpleDatasetPartitionForTesting("p2")));
 Dataset dataset3 = new SimpleDatasetForTesting("dataset3");
 IterableDatasetFinder finder = new StaticDatasetsFinderForTesting(Lists.newArrayList(dataset1, dataset2, dataset3));
 MySource mySource = new MySource(true, finder);
 List<WorkUnit> workUnits = mySource.getWorkunits(new SourceState());
 Assert.assertEquals(workUnits.size(), 4);
 Assert.assertEquals(workUnits.get(0).getProp(DATASET_URN), "dataset1");
 Assert.assertNull(workUnits.get(0).getProp(PARTITION_URN));
 Assert.assertEquals(workUnits.get(1).getProp(DATASET_URN), "dataset2");
 Assert.assertEquals(workUnits.get(1).getProp(PARTITION_URN), "p1");
 Assert.assertEquals(workUnits.get(2).getProp(DATASET_URN), "dataset2");
 Assert.assertEquals(workUnits.get(2).getProp(PARTITION_URN), "p2");
 Assert.assertEquals(workUnits.get(3).getProp(DATASET_URN), "dataset3");
 Assert.assertNull(workUnits.get(3).getProp(PARTITION_URN));
 WorkUnitStream workUnitStream = mySource.getWorkunitStream(new SourceState());
 Assert.assertEquals(Lists.newArrayList(workUnitStream.getWorkUnits()), workUnits);
}

代码示例来源：origin: org.apache.gobblin/gobblin-sql

public WriterInitializer getInitializer(State state, WorkUnitStream workUnits, int branches, int branchId) {
  JdbcWriterCommandsFactory factory = new JdbcWriterCommandsFactory();
  if (workUnits.isSafeToMaterialize()) {
   return new JdbcWriterInitializer(state, workUnits.getMaterializedWorkUnitCollection(),
     factory, branches, branchId);
  } else {
   throw new RuntimeException(JdbcWriterBuilder.class.getName() + " does not support work unit streams.");
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

List<WorkUnit> workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());

代码示例来源：origin: org.apache.gobblin/gobblin-sql

@Override
 public ConverterInitializer getInitializer(State state, WorkUnitStream workUnits, int branches, int branchId) {
  JdbcWriterCommandsFactory factory = new JdbcWriterCommandsFactory();
  if (workUnits.isSafeToMaterialize()) {
   return new AvroToJdbcEntryConverterInitializer(state, workUnits.getMaterializedWorkUnitCollection(),
     factory, branches, branchId);
  } else {
   throw new RuntimeException(AvroToJdbcEntryConverter.class.getName() + " does not support work unit streams.");
  }
 }
}

代码示例来源：origin: apache/incubator-gobblin

List<WorkUnit> workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());

代码示例来源：origin: org.apache.gobblin/gobblin-runtime

if (workUnits.isSafeToMaterialize()) {
 Closer closer = Closer.create();
 Map<String, ParallelRunner> parallelRunners = Maps.newHashMap();
 try {
  for (WorkUnit workUnit : JobLauncherUtils.flattenWorkUnits(workUnits.getMaterializedWorkUnitCollection())) {
   JobLauncherUtils.cleanTaskStagingData(new WorkUnitState(workUnit, jobState), LOG, closer, parallelRunners);

代码示例来源：origin: apache/incubator-gobblin

sourceState.setProp(ConfigurationKeys.JOB_NAME_KEY, TEST_JOB_NAME_1);
WorkUnitStream workUnitStream = mySource.getWorkunitStream(sourceState, true);
List<WorkUnit> workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());
workUnits = Lists.newArrayList(workUnitStream.getWorkUnits());