本文整理了Java中org.languagetool.tokenizers.WordTokenizer类的一些代码示例，展示了WordTokenizer类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。WordTokenizer类的具体详情如下：
包路径：org.languagetool.tokenizers.WordTokenizer
类名称：WordTokenizer

WordTokenizer介绍

[英]Tokenizes a sentence into words. Punctuation and whitespace gets their own tokens. The tokenizer is a quite simple character-based one, though it knows about urls and will put them in one token, if fully specified including a protocol (like http://foobar.org).
[中]把句子标记成单词。标点和空格都有自己的标记。标记器是一个非常简单的基于字符的标记器，尽管它知道URL，如果完全指定了包含协议（如http://foobar.org).

代码示例

代码示例来源：origin: languagetool-org/languagetool

@Override
public List<String> tokenize(String text) {
 List<String> l = new ArrayList<>();
 StringTokenizer st = new StringTokenizer(text, getTokenizingCharacters(), true);
 while (st.hasMoreElements()) {
  l.add(st.nextToken());
 }
 return joinEMailsAndUrls(l);
}

代码示例来源：origin: languagetool-org/languagetool

private String tokenize(String text) {
 List<String> tokens = wordTokenizer.tokenize(text);
 return String.join("|", tokens);
}

代码示例来源：origin: languagetool-org/languagetool

protected boolean isEMail(String token) {
 return WordTokenizer.isEMail(token);
}

代码示例来源：origin: org.languagetool/language-tl

@Override
 public String getTokenizingCharacters() {
  return super.getTokenizingCharacters() + "-";
 } 
}

代码示例来源：origin: languagetool-org/languagetool

@Test
public void testTokenize() {
 WordTokenizer wordTokenizer = new WordTokenizer();
 List <String> tokens = wordTokenizer.tokenize("This is\u00A0a test");
 assertEquals(tokens.size(), 7);
 assertEquals("[This,  , is, \u00A0, a,  , test]", tokens.toString());
 tokens = wordTokenizer.tokenize("This\rbreaks");
 assertEquals(3, tokens.size());
 assertEquals("[This, \r, breaks]", tokens.toString());
 tokens = wordTokenizer.tokenize("dev.all@languagetool.org");
 assertEquals(1, tokens.size());
 tokens = wordTokenizer.tokenize("dev.all@languagetool.org.");
 assertEquals(2, tokens.size());
 tokens = wordTokenizer.tokenize("dev.all@languagetool.org:");
 assertEquals(2, tokens.size());
 tokens = wordTokenizer.tokenize("Schreiben Sie Hr. Meier (meier@mail.com).");
 assertEquals(tokens.size(), 13);
 tokens = wordTokenizer.tokenize("Get more at languagetool.org/foo, and via twitter");
 assertEquals(14, tokens.size());
 assertTrue(tokens.contains("languagetool.org/foo"));
 tokens = wordTokenizer.tokenize("Get more at sub.languagetool.org/foo, and via twitter");
 assertEquals(14, tokens.size());
 assertTrue(tokens.contains("sub.languagetool.org/foo"));
}

代码示例来源：origin: languagetool-org/languagetool

protected boolean isUrl(String token) {
 return WordTokenizer.isUrl(token);
}

代码示例来源：origin: languagetool-org/languagetool

/**
 * @since 3.0
 */
public static boolean isUrl(String token) {
 for (String protocol : WordTokenizer.getProtocols()) {
  if (token.startsWith(protocol + "://") || token.startsWith("www.")) {
   return true;
  }
 }
 return NO_PROTOCOL_URL.matcher(token).matches();
}

代码示例来源：origin: languagetool-org/languagetool

private boolean urlStartsAt(int i, List<String> l) {
 String token = l.get(i);
 if (isProtocol(token) && l.size() > i + 3) {
  String nToken = l.get(i + 1);
  String nnToken = l.get(i + 2);

代码示例来源：origin: org.languagetool/language-pl

public PolishWordTokenizer() {
 plTokenizing = super.getTokenizingCharacters() + "–";   // n-dash
}

代码示例来源：origin: languagetool-org/languagetool

protected void testPerformance(LuceneLanguageModel model, int ngramLength) throws Exception {
 try (FileInputStream fis = new FileInputStream(FILE)) {
  String content = StringTools.readStream(fis, "UTF-8");
  WordTokenizer wordTokenizer = new WordTokenizer();
  List<String> words = wordTokenizer.tokenize(content);
  String prevPrevWord = null;
  String prevWord = null;

代码示例来源：origin: languagetool-org/languagetool

protected boolean isUrl(String token) {
 return WordTokenizer.isUrl(token);
}

代码示例来源：origin: org.languagetool/languagetool-core

/**
 * @since 3.0
 */
public static boolean isUrl(String token) {
 for (String protocol : WordTokenizer.getProtocols()) {
  if (token.startsWith(protocol + "://") || token.startsWith("www.")) {
   return true;
  }
 }
 return false;
}

代码示例来源：origin: org.languagetool/languagetool-core

private boolean urlStartsAt(int i, List<String> l) {
 String token = l.get(i);
 if (isProtocol(token) && l.size() > i + 3) {
  String nToken = l.get(i + 1);
  String nnToken = l.get(i + 2);
  String nnnToken = l.get(i + 3);
  if (nToken.equals(":") && nnToken.equals("/") && nnnToken.equals("/")) {
   return true;
  }
 }
 if (l.size() > i + 1) {
  String nToken = l.get(i);
  String nnToken = l.get(i + 1);
  if (nToken.equals("www") && nnToken.equals(".")) {
   return true;
  }
 }
 return false;
}

代码示例来源：origin: org.languagetool/languagetool-core

@Override
public List<String> tokenize(String text) {
 List<String> l = new ArrayList<>();
 StringTokenizer st = new StringTokenizer(text, getTokenizingCharacters(), true);
 while (st.hasMoreElements()) {
  l.add(st.nextToken());
 }
 return joinEMailsAndUrls(l);
}

代码示例来源：origin: org.languagetool/language-nl

public DutchWordTokenizer() {
 //remove the apostrophe etc. from the standard tokenizing characters:
 String chars = super.getTokenizingCharacters();
 for (String quote : QUOTES) {
  chars = chars.replace(quote, "");
 }
 nlTokenizingChars = chars;
}

代码示例来源：origin: languagetool-org/languagetool

@Override
 public List<String> tokenize(String text) {
  List<String> tokens = super.tokenize(text);
  String prev = null;
  Stack<String> l = new Stack<>();
  for (String token : tokens) {
   if ("'".equals(prev)) {
    if (token.equals("t")) {
     l.pop();
     l.push("'t");
    } else {
     l.push(token);
    }
   } else {
    l.push(token);
   }
   prev = token;
  }
  return l;
 }
}

代码示例来源：origin: languagetool-org/languagetool

@Test
public void testIsUrl() {
 assertTrue(WordTokenizer.isUrl("www.languagetool.org"));
 assertTrue(WordTokenizer.isUrl("languagetool.org/"));
 assertTrue(WordTokenizer.isUrl("languagetool.org/foo"));
 assertTrue(WordTokenizer.isUrl("subdomain.languagetool.org/"));
 assertTrue(WordTokenizer.isUrl("http://www.languagetool.org"));
 assertTrue(WordTokenizer.isUrl("https://www.languagetool.org"));
 assertFalse(WordTokenizer.isUrl("languagetool.org"));  // not detected yet
 assertFalse(WordTokenizer.isUrl("sub.languagetool.org"));  // not detected yet
 assertFalse(WordTokenizer.isUrl("something-else"));
}

代码示例来源：origin: languagetool-org/languagetool

protected boolean isEMail(String token) {
 return WordTokenizer.isEMail(token);
}

代码示例来源：origin: org.languagetool/language-eo

"(?<!')\\b([a-zA-ZĉĝĥĵŝŭĈĜĤĴŜŬ]+)'(?=[a-zA-ZĉĝĥĵŝŭĈĜĤĴŜŬ-])",
    "$1\u0001\u0001EO@APOS2\u0001\u0001 ");
List<String> tokenList = super.tokenize(replaced);
List<String> tokens = new ArrayList<>();

代码示例来源：origin: languagetool-org/languagetool

WordTokenizer.isUrl(tokens[tokens.length-1].getToken())) {