本文整理了Java中org.archive.io.warc.WARCWriter类的一些代码示例，展示了WARCWriter类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。WARCWriter类的具体详情如下：
包路径：org.archive.io.warc.WARCWriter
类名称：WARCWriter

WARCWriter介绍

[英]WARC implementation.

Assumption is that the caller is managing access to this WARCWriter ensuring only one thread accessing this WARC instance at any one time.

While being written, WARCs have a '.open' suffix appended.
[中]WARC实现。
假设调用方正在管理对这个WARCWriter的访问，确保在任何时候只有一个线程访问这个WARC实例。
在编写WARC时，WARC有一个。附加了“打开”后缀。

代码示例

代码示例来源：origin: internetarchive/heritrix3

writer = new WARCWriter(
      new AtomicInteger(),
      bos, 
writer.writeWarcinfoRecord(warc.getName(),
  "Made from " + reader.getReaderIdentifier() + " by " +
    this.getClass().getName() + "/" + getRevision());
  l.setLevel(Level.WARNING);
  try {
    writer.close();
  } finally {
    l.setLevel(oldLevel);

代码示例来源：origin: internetarchive/heritrix3

protected URI writeFtpControlConversation(WARCWriter w, String timestamp,
    URI baseid, CrawlURI curi, ANVLRecord headers,
    String controlConversation) throws IOException {
  
  WARCRecordInfo recordInfo = new WARCRecordInfo();
  recordInfo.setCreate14DigitDate(timestamp);
  recordInfo.setUrl(curi.toString());
  recordInfo.setMimetype(FTP_CONTROL_CONVERSATION_MIMETYPE);
  recordInfo.setExtraHeaders(headers);
  recordInfo.setEnforceLength(true);
  recordInfo.setType(WARCRecordType.metadata);
  recordInfo.setRecordId(qualifyRecordID(baseid, TYPE, WARCRecordType.metadata.toString()));
  
  byte[] b = controlConversation.getBytes("UTF-8");
  
  recordInfo.setContentStream(new ByteArrayInputStream(b));
  recordInfo.setContentLength((long) b.length);
  
  w.writeRecord(recordInfo);
  
  return recordInfo.getRecordId();
}

代码示例来源：origin: internetarchive/heritrix3

protected void updateMetadataAfterWrite(final CrawlURI curi,
    WARCWriter writer, long startPosition) {
  if (WARCWriter.getStat(writer.getTmpStats(), WARCWriter.TOTALS, WARCWriter.NUM_RECORDS) > 0l) {
     addStats(writer.getTmpStats());
     urlsWritten.incrementAndGet();
      + WARCWriter.getStat(writer.getTmpStats(), WARCWriter.TOTALS, WARCWriter.SIZE_ON_DISK) 
      + " bytes to " + writer.getFile().getName() + " for " + curi);
  setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - startPosition));
  curi.addExtraInfo("warcFilename", writer.getFilenameWithoutOccupiedSuffix());
  curi.addExtraInfo("warcFileOffset", startPosition);
  curi.getData().put(A_WARC_STATS, copyStats(writer.getTmpStats()));
    history[0].put(A_WRITE_TAG, writer.getFilenameWithoutOccupiedSuffix());
    for (WARCRecordInfo warcRecord: writer.getTmpRecordLog()) {
      if ((warcRecord.getType() == WARCRecordType.response 
          || warcRecord.getType() == WARCRecordType.resource)

代码示例来源：origin: internetarchive/heritrix3

writer.resetTmpStats();
writer.resetTmpRecordLog();
long position = writer.getPosition();
try {
  writer.checkSize();
  if (writer.getPosition() != position) {
      (writer.getPosition() - position));
    addStats(writer.getTmpStats());
    writer.resetTmpStats();
    writer.resetTmpRecordLog();
    position = writer.getPosition();

代码示例来源：origin: org.netpreserve.commons/webarchive-commons

header = createRecordHeader(recordInfo);
long startPosition;
startPosition = getPosition();
try {
  preWriteRecordTasks();
  write(bytes);
  totalBytes += bytes.length;
  write(CRLF_BYTES);
  totalBytes += CRLF_BYTES.length;
    contentBytes += copyFrom(recordInfo.getContentStream(),
        recordInfo.getContentLength(),
        recordInfo.getEnforceLength());
  write(CRLF_BYTES);
  write(CRLF_BYTES);
  totalBytes += 2 * CRLF_BYTES.length;
  recordInfo.setWARCFilename(getFilenameWithoutOccupiedSuffix());
  recordInfo.setWARCFileOffset(startPosition);
  tmpRecordLog.add(recordInfo);
} finally {
  postWriteRecordTasks();
  tally(recordInfo.getType(), contentBytes, totalBytes, getPosition() - startPosition);

代码示例来源：origin: ViDA-NYU/ache

@Override
public boolean insert(Page target) {
  try {
    if (writer == null) {
      createWarcWriter();
    }
    WARCRecordInfo warcRecord = getWarcRecordInfo(target, generator.getRecordID());
    synchronized (writer) {
      writer.checkSize();
      writer.writeRecord(warcRecord);
      writer.resetTmpStats();
      writer.resetTmpRecordLog();
    }
    return true;
  } catch (IOException e) {
    logger.error("Exception thrown while creating a WARC record.", e);
  }
  return false;
}

代码示例来源：origin: iipc/openwayback

private void writeHeaderRecord(File target, File fieldsSrc, String id)
throws IOException {
  WARCWriter writer = null;
  BufferedOutputStream bos =
    new BufferedOutputStream(new FileOutputStream(target));
  FileInputStream is = new FileInputStream(fieldsSrc);
  ANVLRecord ar = ANVLRecord.load(is);
  List<String> metadata = new ArrayList<String>(1);
  metadata.add(ar.toString());
  writer = new WARCWriter(new AtomicInteger(),bos,target,getSettings(true, null, null, metadata));
  // Write a warcinfo record with description about how this WARC
  // was made.
  writer.writeWarcinfoRecord(target.getName(), "Made from "
      + id + " by "
      + this.getClass().getName());
}
private WARCWriterPoolSettings getSettings(final boolean isCompressed,

代码示例来源：origin: org.netpreserve.commons/commons-web

recordInfo.setEnforceLength(true);
recordInfo.setRecordId(generateRecordId(TYPE, WARCRecordType.warcinfo.toString()));
writeRecord(recordInfo);

代码示例来源：origin: org.netpreserve.commons/webarchive-commons

protected WriterPoolMember makeWriter() {
    return new WARCWriter(serialNo, (WARCWriterPoolSettings)settings);
  }
}

代码示例来源：origin: org.netpreserve.commons/webarchive-commons

public URI writeWarcinfoRecord(String filename)
throws IOException {
  return writeWarcinfoRecord(filename, null);
}

代码示例来源：origin: iipc/webarchive-commons

public String getFilenameWithoutOccupiedSuffix() {
  String name = getFile().getName();
  if (name.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) {
    name = name.substring(0, name.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length());
  }
  return name;
}

代码示例来源：origin: org.netpreserve.commons/webarchive-commons

protected String createRecordHeader(WARCRecordInfo metaRecord)
throws IllegalArgumentException {
  final StringBuilder sb =
    new StringBuilder(2048/*A SWAG: TODO: Do analysis.*/);
  sb.append(WARC_ID).append(CRLF);
  sb.append(HEADER_KEY_TYPE).append(COLON_SPACE).append(metaRecord.getType()).
    append(CRLF);
  // Do not write a subject-uri if not one present.
  if (!StringUtils.isEmpty(metaRecord.getUrl())) {
    sb.append(HEADER_KEY_URI).append(COLON_SPACE).
      append(checkHeaderValue(metaRecord.getUrl())).append(CRLF);
  }
  sb.append(HEADER_KEY_DATE).append(COLON_SPACE).
    append(metaRecord.getCreate14DigitDate()).append(CRLF);
  if (metaRecord.getExtraHeaders() != null) {
    for (final Iterator<Element> i = metaRecord.getExtraHeaders().iterator(); i.hasNext();) {
      sb.append(i.next()).append(CRLF);
    }
  }
  sb.append(HEADER_KEY_ID).append(COLON_SPACE).append('<').
    append(metaRecord.getRecordId().toString()).append('>').append(CRLF);
  if (metaRecord.getContentLength() > 0) {
    sb.append(CONTENT_TYPE).append(COLON_SPACE).append(
      checkHeaderLineMimetypeParameter(metaRecord.getMimetype())).append(CRLF);
  }
  sb.append(CONTENT_LENGTH).append(COLON_SPACE).
    append(Long.toString(metaRecord.getContentLength())).append(CRLF);
  
  return sb.toString();
}

代码示例来源：origin: org.netpreserve.commons/webarchive-commons

protected String checkHeaderValue(final String value)
throws IllegalArgumentException {
  for (int i = 0; i < value.length(); i++) {
    final char c = value.charAt(i);
    baseCharacterCheck(c, value);
    if (Character.isWhitespace(c)) {
      throw new IllegalArgumentException("Contains disallowed white space 0x" +
        Integer.toHexString(c) + ": " + value);
    }
  }
  return value;
}

代码示例来源：origin: org.netpreserve.commons/commons-web

header = createRecordHeader(recordInfo);
  startPosition = getPosition();
  preWriteRecordTasks();
  write(bytes);
  totalBytes += bytes.length;
    write(CRLF_BYTES); // TODO: should this be written even for zero-length?
    totalBytes += CRLF_BYTES.length;
    contentBytes += copyFrom(recordInfo.getContentStream(),
        recordInfo.getContentLength(),
        recordInfo.getEnforceLength());
  write(CRLF_BYTES);
  write(CRLF_BYTES);
  totalBytes += 2 * CRLF_BYTES.length;
  tally(recordInfo.getType(), contentBytes, totalBytes, getPosition() - startPosition);
  recordInfo.setWARCFilename(getFilenameWithoutOccupiedSuffix());
  recordInfo.setWARCFileOffset(startPosition);
  tmpRecordLog.add(recordInfo);
} finally {
  postWriteRecordTasks();

代码示例来源：origin: org.archive.heritrix/heritrix-modules

writer.resetTmpStats();
writer.resetTmpRecordLog();
long position = writer.getPosition();
try {
  writer.checkSize();
  if (writer.getPosition() != position) {
      (writer.getPosition() - position));
    addStats(writer.getTmpStats());
    writer.resetTmpStats();
    writer.resetTmpRecordLog();
    position = writer.getPosition();

代码示例来源：origin: org.netpreserve.openwayback/openwayback-core

private void writeHeaderRecord(File target, File fieldsSrc, String id)
throws IOException {
  WARCWriter writer = null;
  BufferedOutputStream bos =
    new BufferedOutputStream(new FileOutputStream(target));
  FileInputStream is = new FileInputStream(fieldsSrc);
  ANVLRecord ar = ANVLRecord.load(is);
  List<String> metadata = new ArrayList<String>(1);
  metadata.add(ar.toString());
  writer = new WARCWriter(new AtomicInteger(),bos,target,getSettings(true, null, null, metadata));
  // Write a warcinfo record with description about how this WARC
  // was made.
  writer.writeWarcinfoRecord(target.getName(), "Made from "
      + id + " by "
      + this.getClass().getName());
}
private WARCWriterPoolSettings getSettings(final boolean isCompressed,

代码示例来源：origin: org.netpreserve.commons/webarchive-commons

recordInfo.setEnforceLength(true);
recordInfo.setRecordId(generateRecordId(TYPE, WARCRecordType.warcinfo.toString()));
writeRecord(recordInfo);

代码示例来源：origin: org.netpreserve.commons/commons-web

protected WriterPoolMember makeWriter() {
    return new WARCWriter(serialNo, (WARCWriterPoolSettings)settings);
  }
}

代码示例来源：origin: org.netpreserve.commons/commons-web

public URI writeWarcinfoRecord(String filename)
throws IOException {
  return writeWarcinfoRecord(filename, null);
}

代码示例来源：origin: org.netpreserve.commons/commons-web

public String getFilenameWithoutOccupiedSuffix() {
  String name = getFile().getName();
  if (name.endsWith(ArchiveFileConstants.OCCUPIED_SUFFIX)) {
    name = name.substring(0, name.length() - ArchiveFileConstants.OCCUPIED_SUFFIX.length());
  }
  return name;
}