gpt4 book ai didi

org.apache.hadoop.hbase.mapreduce.WALPlayer类的使用及代码示例

转载 作者:知者 更新时间:2024-03-27 04:27:05 27 4
gpt4 key购买 nike

本文整理了Java中org.apache.hadoop.hbase.mapreduce.WALPlayer类的一些代码示例,展示了WALPlayer类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。WALPlayer类的具体详情如下:
包路径:org.apache.hadoop.hbase.mapreduce.WALPlayer
类名称:WALPlayer

WALPlayer介绍

[英]A tool to replay WAL files as a M/R job. The WAL can be replayed for a set of tables or all tables, and a time range can be provided (in milliseconds). The WAL is filtered to the passed set of tables and the output can optionally be mapped to another set of tables. WAL replay can also generate HFiles for later bulk importing, in that case the WAL is replayed for a single table only.
[中]作为M/R作业重放WAL文件的工具。WAL可以为一组表或所有表重放,并且可以提供时间范围(以毫秒为单位)。WAL被过滤到传递的一组表,并且输出可以选择性地映射到另一组表。WAL replay还可以为以后的批量导入生成HFiles,在这种情况下,WAL只针对单个表进行重放。

代码示例

代码示例来源:origin: apache/hbase

/**
 * Main entry point.
 * @param args The command line parameters.
 * @throws Exception When running the job fails.
 */
public static void main(String[] args) throws Exception {
 int ret = ToolRunner.run(new WALPlayer(HBaseConfiguration.create()), args);
 System.exit(ret);
}

代码示例来源:origin: apache/hbase

Configuration conf = getConf();
setupTime(conf, WALInputFormat.START_TIME_KEY);
setupTime(conf, WALInputFormat.END_TIME_KEY);
String inputDirs = args[0];
String[] tables = args[1].split(",");
 List<TableName> tableNames = getTableNameList(tables);

代码示例来源:origin: apache/hbase

@Override
 public int run(String[] args) throws Exception {
  if (args.length < 2) {
   usage("Wrong number of arguments: " + args.length);
   System.exit(-1);
  }
  Job job = createSubmittableJob(args);
  return job.waitForCompletion(true) ? 0 : 1;
 }
}

代码示例来源:origin: apache/hbase

WALPlayer player = new WALPlayer(configuration);
String optionName="_test_.name";
configuration.set(optionName, "1000");
player.setupTime(configuration, optionName);
assertEquals(1000,configuration.getLong(optionName,0));
assertEquals(0, ToolRunner.run(configuration, player,

代码示例来源:origin: co.cask.hbase/hbase

Configuration conf = getConf();
setupTime(conf, HLogInputFormat.START_TIME_KEY);
setupTime(conf, HLogInputFormat.END_TIME_KEY);
Path inputDir = new Path(args[0]);
String[] tables = args[1].split(",");

代码示例来源:origin: harbby/presto-connectors

@Override
 public int run(String[] args) throws Exception {
  String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
  if (otherArgs.length < 2) {
   usage("Wrong number of arguments: " + otherArgs.length);
   System.exit(-1);
  }
  Job job = createSubmittableJob(otherArgs);
  return job.waitForCompletion(true) ? 0 : 1;
 }
}

代码示例来源:origin: apache/hbase

@Override
 public Job createSubmittableJob(String[] args) throws IOException {
  Job job = super.createSubmittableJob(args);
  // Call my class instead.
  job.setJarByClass(WALMapperSearcher.class);
  job.setMapperClass(WALMapperSearcher.class);
  job.setOutputFormatClass(NullOutputFormat.class);
  return job;
 }
}

代码示例来源:origin: apache/hbase

/**
 * Test main method
 */
@Test
public void testMainMethod() throws Exception {
 PrintStream oldPrintStream = System.err;
 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
 System.setSecurityManager(newSecurityManager);
 ByteArrayOutputStream data = new ByteArrayOutputStream();
 String[] args = {};
 System.setErr(new PrintStream(data));
 try {
  System.setErr(new PrintStream(data));
  try {
   WALPlayer.main(args);
   fail("should be SecurityException");
  } catch (SecurityException e) {
   assertEquals(-1, newSecurityManager.getExitCode());
   assertTrue(data.toString().contains("ERROR: Wrong number of arguments:"));
   assertTrue(data.toString().contains("Usage: WALPlayer [options] <wal inputdir>" +
     " <tables> [<tableMappings>]"));
   assertTrue(data.toString().contains("-Dwal.bulk.output=/path/for/output"));
  }
 } finally {
  System.setErr(oldPrintStream);
  System.setSecurityManager(SECURITY_MANAGER);
 }
}

代码示例来源:origin: com.aliyun.hbase/alihbase-mapreduce

WALPlayer player = new WALPlayer(configuration);
String optionName="_test_.name";
configuration.set(optionName, "1000");
player.setupTime(configuration, optionName);
assertEquals(1000,configuration.getLong(optionName,0));
assertEquals(0, ToolRunner.run(configuration, player,

代码示例来源:origin: com.aliyun.hbase/alihbase-mapreduce

Configuration conf = getConf();
setupTime(conf, WALInputFormat.START_TIME_KEY);
setupTime(conf, WALInputFormat.END_TIME_KEY);
String inputDirs = args[0];
String[] tables = args[1].split(",");

代码示例来源:origin: co.cask.hbase/hbase

@Override
 public int run(String[] args) throws Exception {
  String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
  if (otherArgs.length < 2) {
   usage("Wrong number of arguments: " + otherArgs.length);
   System.exit(-1);
  }
  Job job = createSubmittableJob(otherArgs);
  return job.waitForCompletion(true) ? 0 : 1;
 }
}

代码示例来源:origin: apache/hbase

@Override
 public Job createSubmittableJob(String[] args) throws IOException {
  Job job = super.createSubmittableJob(args);
  // Call my class instead.
  job.setJarByClass(WALMapperSearcher.class);
  job.setMapperClass(WALMapperSearcher.class);
  job.setOutputFormatClass(NullOutputFormat.class);
  return job;
 }
}

代码示例来源:origin: org.apache.hbase/hbase-mapreduce

/**
 * Test main method
 */
@Test
public void testMainMethod() throws Exception {
 PrintStream oldPrintStream = System.err;
 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
 System.setSecurityManager(newSecurityManager);
 ByteArrayOutputStream data = new ByteArrayOutputStream();
 String[] args = {};
 System.setErr(new PrintStream(data));
 try {
  System.setErr(new PrintStream(data));
  try {
   WALPlayer.main(args);
   fail("should be SecurityException");
  } catch (SecurityException e) {
   assertEquals(-1, newSecurityManager.getExitCode());
   assertTrue(data.toString().contains("ERROR: Wrong number of arguments:"));
   assertTrue(data.toString().contains("Usage: WALPlayer [options] <wal inputdir>" +
     " <tables> [<tableMappings>]"));
   assertTrue(data.toString().contains("-Dwal.bulk.output=/path/for/output"));
  }
 } finally {
  System.setErr(oldPrintStream);
  System.setSecurityManager(SECURITY_MANAGER);
 }
}

代码示例来源:origin: org.apache.hbase/hbase-mapreduce

WALPlayer player = new WALPlayer(configuration);
String optionName="_test_.name";
configuration.set(optionName, "1000");
player.setupTime(configuration, optionName);
assertEquals(1000,configuration.getLong(optionName,0));
assertEquals(0, ToolRunner.run(configuration, player,

代码示例来源:origin: harbby/presto-connectors

Configuration conf = getConf();
setupTime(conf, HLogInputFormat.START_TIME_KEY);
setupTime(conf, HLogInputFormat.END_TIME_KEY);
Path inputDir = new Path(args[0]);
String[] tables = args[1].split(",");

代码示例来源:origin: apache/hbase

protected void walToHFiles(List<String> dirPaths, List<String> tableList) throws IOException {
 Tool player = new WALPlayer();
 // Player reads all files in arbitrary directory structure and creates
 // a Map task for each file. We use ';' as separator
 // because WAL file names contains ','
 String dirs = StringUtils.join(dirPaths, ';');
 String jobname = "Incremental_Backup-" + backupId ;
 Path bulkOutputPath = getBulkOutputDir();
 conf.set(WALPlayer.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString());
 conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";");
 conf.setBoolean(WALPlayer.MULTI_TABLES_SUPPORT, true);
 conf.set(JOB_NAME_CONF_KEY, jobname);
 String[] playerArgs = { dirs, StringUtils.join(tableList, ",") };
 try {
  player.setConf(conf);
  int result = player.run(playerArgs);
  if(result != 0) {
   throw new IOException("WAL Player failed");
  }
  conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY);
  conf.unset(JOB_NAME_CONF_KEY);
 } catch (IOException e) {
  throw e;
 } catch (Exception ee) {
  throw new IOException("Can not convert from directory " + dirs
    + " (check Hadoop, HBase and WALPlayer M/R job logs) ", ee);
 }
}

代码示例来源:origin: com.aliyun.hbase/alihbase-mapreduce

@Override
 public int run(String[] args) throws Exception {
  if (args.length < 2) {
   usage("Wrong number of arguments: " + args.length);
   System.exit(-1);
  }
  Job job = createSubmittableJob(args);
  return job.waitForCompletion(true) ? 0 : 1;
 }
}

代码示例来源:origin: org.apache.hbase/hbase-it

@Override
 public Job createSubmittableJob(String[] args) throws IOException {
  Job job = super.createSubmittableJob(args);
  // Call my class instead.
  job.setJarByClass(WALMapperSearcher.class);
  job.setMapperClass(WALMapperSearcher.class);
  job.setOutputFormatClass(NullOutputFormat.class);
  return job;
 }
}

代码示例来源:origin: com.aliyun.hbase/alihbase-mapreduce

/**
 * Test main method
 */
@Test
public void testMainMethod() throws Exception {
 PrintStream oldPrintStream = System.err;
 SecurityManager SECURITY_MANAGER = System.getSecurityManager();
 LauncherSecurityManager newSecurityManager= new LauncherSecurityManager();
 System.setSecurityManager(newSecurityManager);
 ByteArrayOutputStream data = new ByteArrayOutputStream();
 String[] args = {};
 System.setErr(new PrintStream(data));
 try {
  System.setErr(new PrintStream(data));
  try {
   WALPlayer.main(args);
   fail("should be SecurityException");
  } catch (SecurityException e) {
   assertEquals(-1, newSecurityManager.getExitCode());
   assertTrue(data.toString().contains("ERROR: Wrong number of arguments:"));
   assertTrue(data.toString().contains("Usage: WALPlayer [options] <wal inputdir>" +
     " <tables> [<tableMappings>]"));
   assertTrue(data.toString().contains("-Dwal.bulk.output=/path/for/output"));
  }
 } finally {
  System.setErr(oldPrintStream);
  System.setSecurityManager(SECURITY_MANAGER);
 }
}

代码示例来源:origin: org.apache.hbase/hbase-mapreduce

WALPlayer player = new WALPlayer(configuration);
String optionName="_test_.name";
configuration.set(optionName, "1000");
player.setupTime(configuration, optionName);
assertEquals(1000,configuration.getLong(optionName,0));
assertEquals(0, ToolRunner.run(configuration, player,

27 4 0
Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
广告合作:1813099741@qq.com 6ren.com