Field Detail

• doubleArrayTrie

  protected DoubleArrayTrie doubleArrayTrie

• connectionCosts

  protected ConnectionCosts connectionCosts

• tokenInfoDictionary

  protected TokenInfoDictionary tokenInfoDictionary

• unknownDictionary

  protected UnknownDictionary unknownDictionary

• characterDefinitions

  protected CharacterDefinitions characterDefinitions

• insertedDictionary

  protected InsertedDictionary insertedDictionary

• userDictionary

  protected UserDictionary userDictionary

• mode

  protected TokenizerBase.Mode mode

• split

  protected boolean split

• penalties

  protected List<Integer> penalties

• totalFeatures

  protected int totalFeatures

• readingFeature

  protected int readingFeature

• partOfSpeechFeature

  protected int partOfSpeechFeature

• resolver

  protected ResourceResolver resolver

• tokenFactory

  protected TokenFactory tokenFactory

Constructor Detail

• Builder

  public Builder()

Method Detail

• loadDictionaries

  protected void loadDictionaries()

• build

  public abstract <T extends TokenizerBase> T build()

  Creates a Tokenizer instance defined by this Builder

  Type Parameters:

  T - token type

  Returns:

  Tokenizer instance

• userDictionary

  public TokenizerBase.Builder userDictionary(InputStream input)
                                       throws IOException

  Sets an optional user dictionary as an input stream

  The inpuut stream provided is not closed by this method

  Parameters:

  input - user dictionary as an input stream

  Returns:

  this builder

  Throws:

  IOException - if an error occurs when reading the user dictionary

• userDictionary

  public TokenizerBase.Builder userDictionary(String filename)
                                       throws IOException

  Sets an optional user dictionary filename

  Parameters:

  filename - user dictionary filename

  Returns:

  this builder

  Throws:

  IOException - if an error occurs when reading the user dictionary