public class TokenizerAnnotatorPTB
extends JCasAnnotator_ImplBase
Modifier and Type | Field and Description |
---|---|
(package private) static char |
CR |
(package private) static char |
LF |
private Logger |
logger |
static java.lang.String |
PARAM_SEGMENTS_TO_SKIP
Value is "SegmentsToSkip".
|
private java.lang.String[] |
skipSegmentsArray |
private java.util.Set<java.lang.String> |
skipSegmentsSet |
private int |
tokenCount |
private TokenizerPTB |
tokenizer |
Constructor and Description |
---|
TokenizerAnnotatorPTB() |
Modifier and Type | Method and Description |
---|---|
protected void |
annotateRange(JCas jcas,
int rangeBegin,
int rangeEnd)
Tokenizes a range of text, adding the tokens to the CAS
Tokenizes one sentence at a time.
|
static AnalysisEngineDescription |
createAnnotatorDescription() |
void |
initialize(UimaContext aContext) |
void |
process(JCas jcas)
Entry point for processing.
|
private Logger logger
public static final java.lang.String PARAM_SEGMENTS_TO_SKIP
private java.lang.String[] skipSegmentsArray
private java.util.Set<java.lang.String> skipSegmentsSet
private TokenizerPTB tokenizer
private int tokenCount
static char CR
static char LF
public void initialize(UimaContext aContext) throws ResourceInitializationException
ResourceInitializationException
public void process(JCas jcas) throws AnalysisEngineProcessException
AnalysisEngineProcessException
protected void annotateRange(JCas jcas, int rangeBegin, int rangeEnd) throws AnalysisEngineProcessException
AnalysisEngineProcessException
public static AnalysisEngineDescription createAnnotatorDescription() throws ResourceInitializationException
ResourceInitializationException