卢塞恩 4.1 : How split words that contains "dots" when indexing?

Question

我想弄清楚应该如何为包含“.”的关键字编制索引。 .

例如:this.name

我想在我的索引中索引术语:this 和 name。

我使用 StandardAnalyser。我尝试扩展 WhitespaceTokensizer 或扩展 TokenFilter，但我不确定我的方向是否正确。

如果我使用 StandardAnalyser，我将获得“this.name”作为关键字，这不是我想要的，但分析器会为我正确完成其余部分。

Answer 1

您可以在将句点和下划线转换为空格的 StandardTokenizer 前面放置一个 CharFilter。 MappingCharFilter 将起作用。

这是添加到精简 StandardAnalyzer 的 MappingCharFilter(请参阅原始 4.1 版本 here):

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;

import java.io.IOException;
import java.io.Reader;

public final class MyAnalyzer extends StopwordAnalyzerBase {
  private int maxTokenLength = 255;
  public MyAnalyzer() {
    super(Version.LUCENE_41, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
  }

  @Override
  protected TokenStreamComponents createComponents
      (final String fieldName, final Reader reader) {
    final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
    src.setMaxTokenLength(maxTokenLength);
    TokenStream tok = new StandardFilter(matchVersion, src);
    tok = new LowerCaseFilter(matchVersion, tok);
    tok = new StopFilter(matchVersion, tok, stopwords);
    return new TokenStreamComponents(src, tok) {
      @Override
      protected void setReader(final Reader reader) throws IOException {
        src.setMaxTokenLength(MyAnalyzer.this.maxTokenLength);
        super.setReader(reader);
      }
    };
  }

  @Override
  protected Reader initReader(String fieldName, Reader reader) {
    NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
    builder.add(".", " ");
    builder.add("_", " ");
    NormalizeCharMap normMap = builder.build();
    return new MappingCharFilter(normMap, reader);
  }
}

这里有一个快速测试来证明它是有效的:

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;

public class TestMyAnalyzer extends BaseTokenStreamTestCase {
  private Analyzer analyzer = new MyAnalyzer();

  public void testPeriods() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo
    (analyzer, 
     "this.name; here.i.am; sentences ... end with periods.",
     new String[] { "name", "here", "i", "am", "sentences", "end", "periods" } );
  }

  public void testUnderscores() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo
        (analyzer,
         "some_underscore_term _and____ stuff that is_not in it",
         new String[] { "some", "underscore", "term", "stuff" } );
  }
}