public class TokenizerAnnotator
extends JCasAnnotator_ImplBase
Modifier and Type | Field and Description |
---|---|
private UimaContext |
context |
private java.lang.String |
HYPH_FREQ_TABLE_RESRC_KEY |
private Logger |
logger |
static java.lang.String |
PARAM_SEGMENTS_TO_SKIP
Value is "SegmentsToSkip".
|
private java.util.Set<java.lang.String> |
skipSegmentsSet |
static int |
TOKEN_CAP_ALL |
static int |
TOKEN_CAP_FIRST_ONLY |
static int |
TOKEN_CAP_MIXED |
static int |
TOKEN_CAP_NONE |
static int |
TOKEN_NUM_POS_FIRST |
static int |
TOKEN_NUM_POS_LAST |
static int |
TOKEN_NUM_POS_MIDDLE |
static int |
TOKEN_NUM_POS_NONE |
static int |
TOKEN_NUM_TYPE_DECIMAL |
static int |
TOKEN_NUM_TYPE_INTEGER |
static int |
TOKEN_NUM_TYPE_UNKNOWN |
private int |
tokenCount |
private Tokenizer |
tokenizer |
Constructor and Description |
---|
TokenizerAnnotator() |
Modifier and Type | Method and Description |
---|---|
protected void |
annotateRange(JCas jcas,
int beginPos,
int endPos)
A utility method that tokenizes a range of text.
|
private void |
configInit()
Reads configuration parameters.
|
void |
initialize(UimaContext aContext) |
void |
process(JCas jcas)
Entry point for processing.
|
private Logger logger
public static final int TOKEN_CAP_NONE
public static final int TOKEN_CAP_FIRST_ONLY
public static final int TOKEN_CAP_MIXED
public static final int TOKEN_CAP_ALL
public static final int TOKEN_NUM_POS_NONE
public static final int TOKEN_NUM_POS_FIRST
public static final int TOKEN_NUM_POS_MIDDLE
public static final int TOKEN_NUM_POS_LAST
public static final int TOKEN_NUM_TYPE_UNKNOWN
public static final int TOKEN_NUM_TYPE_INTEGER
public static final int TOKEN_NUM_TYPE_DECIMAL
public static final java.lang.String PARAM_SEGMENTS_TO_SKIP
private final java.lang.String HYPH_FREQ_TABLE_RESRC_KEY
private UimaContext context
private java.util.Set<java.lang.String> skipSegmentsSet
private Tokenizer tokenizer
private int tokenCount
public void initialize(UimaContext aContext) throws ResourceInitializationException
ResourceInitializationException
private void configInit() throws ResourceAccessException
ResourceAccessException
public void process(JCas jcas) throws AnalysisEngineProcessException
AnalysisEngineProcessException
protected void annotateRange(JCas jcas, int beginPos, int endPos) throws AnalysisEngineProcessException
AnalysisEngineProcessException