Lucene++ - a full-featured, c++ search engine
API Documentation
Go to the documentation of this file.
7 #ifndef STANDARDTOKENIZERIMPL_H
8 #define STANDARDTOKENIZERIMPL_H
36 static const wchar_t*
ZZ_CMAP();
162 void yybegin(int32_t newState);
static const int32_t * ZZ_ROWMAP()
void getText(const TokenPtr &t)
Fills Lucene token with the current token text.
static const int32_t ZZ_ROWMAP_LENGTH
Definition: StandardTokenizerImpl.h:51
Definition: StandardTokenizerImpl.h:14
static const int32_t * ZZ_ACTION()
static const wchar_t ZZ_ATTRIBUTE_PACKED_0[]
Definition: StandardTokenizerImpl.h:77
#define LUCENE_CLASS(Name)
Definition: LuceneObject.h:24
static const int32_t ZZ_NO_MATCH
Definition: StandardTokenizerImpl.h:70
int32_t yylength()
Returns the length of the matched text region.
void yyclose()
Closes the input stream.
void yybegin(int32_t newState)
Enters a new lexical state.
bool zzAtEOF
zzAtEOF == true if the scanner is at the EOF
Definition: StandardTokenizerImpl.h:125
boost::shared_ptr< Reader > ReaderPtr
Definition: LuceneTypes.h:547
int32_t zzState
The current state of the DFA.
Definition: StandardTokenizerImpl.h:89
int32_t zzMarkedPos
The text position at the last accepting state.
Definition: StandardTokenizerImpl.h:98
static const int32_t ZZ_ROWMAP_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:52
static const wchar_t * ZZ_ERROR_MSG[]
Definition: StandardTokenizerImpl.h:73
int32_t zzEndRead
EndRead marks the last character in the buffer, that has been read from input.
Definition: StandardTokenizerImpl.h:110
static void ZZ_CMAP_INIT()
Translates characters to character classes.
int32_t yystate()
Returns the current lexical state.
int32_t zzPushbackPos
The text position at the last state to be included in yytext.
Definition: StandardTokenizerImpl.h:101
int32_t yycolumn
The number of characters from the last newline up to the start of the matched text.
Definition: StandardTokenizerImpl.h:119
static const int32_t ZZ_ACTION_LENGTH
Definition: StandardTokenizerImpl.h:41
CharArray zzBuffer
This buffer contains the current text to be matched and is the source of the yytext() string.
Definition: StandardTokenizerImpl.h:95
static const int32_t * ZZ_TRANS()
static const wchar_t * ZZ_CMAP()
static void ZZ_ROWMAP_INIT()
Translates a state to a row index in the transition table.
static CharArray _ZZ_CMAP
Translates characters to character classes.
Definition: StandardTokenizerImpl.h:29
int32_t zzLexicalState
The current lexical state.
Definition: StandardTokenizerImpl.h:92
boost::shared_ptr< Token > TokenPtr
Definition: LuceneTypes.h:59
int32_t zzCurrentPos
The current text position in the buffer.
Definition: StandardTokenizerImpl.h:104
static void ZZ_ACTION_INIT()
Translates DFA states to action switch labels.
void zzScanError(int32_t errorCode)
Reports an error that occurred while scanning.
Definition: AbstractAllTermDocs.h:12
static const int32_t ZZ_CMAP_LENGTH
Definition: StandardTokenizerImpl.h:31
static const int32_t ZZ_ATTRIBUTE_LENGTH
Definition: StandardTokenizerImpl.h:78
Base class for all Lucene classes.
Definition: LuceneObject.h:31
void yypushback(int32_t number)
Pushes the specified amount of characters back into the input stream.
static const int32_t ZZ_CMAP_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:32
static const wchar_t ZZ_ACTION_PACKED_0[]
Definition: StandardTokenizerImpl.h:40
static const int32_t ZZ_ATTRIBUTE_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:79
ReaderPtr zzReader
The input device.
Definition: StandardTokenizerImpl.h:86
static IntArray _ZZ_ROWMAP
Translates a state to a row index in the transition table.
Definition: StandardTokenizerImpl.h:49
static const int32_t ZZ_UNKNOWN_ERROR
Definition: StandardTokenizerImpl.h:69
static const int32_t ZZ_TRANS_LENGTH
Definition: StandardTokenizerImpl.h:61
static const wchar_t ZZ_ROWMAP_PACKED_0[]
Definition: StandardTokenizerImpl.h:50
static const int32_t YYINITIAL
Lexical states.
Definition: StandardTokenizerImpl.h:132
static const int32_t * ZZ_ATTRIBUTE()
void yyreset(const ReaderPtr &reader)
Resets the scanner to read from a new input stream. Does not close the old reader.
void reset(const ReaderPtr &r)
Resets the Tokenizer to a new Reader.
static void ZZ_TRANS_INIT()
The transition table of the DFA.
StandardTokenizerImpl(const ReaderPtr &in)
Creates a new scanner.
static const wchar_t ZZ_TRANS_PACKED_0[]
Definition: StandardTokenizerImpl.h:60
wchar_t yycharat(int32_t pos)
Returns the character at position pos from the matched text.
int32_t _yychar
The number of characters up to the start of the matched text.
Definition: StandardTokenizerImpl.h:116
boost::shared_ptr< TermAttribute > TermAttributePtr
Definition: LuceneTypes.h:58
static IntArray _ZZ_ACTION
Translates DFA states to action switch labels.
Definition: StandardTokenizerImpl.h:39
int32_t yyline
Number of newlines encountered up to the start of the matched text.
Definition: StandardTokenizerImpl.h:113
bool zzAtBOL
zzAtBOL == true if the scanner is currently at the beginning of a line
Definition: StandardTokenizerImpl.h:122
static const wchar_t ZZ_CMAP_PACKED[]
Definition: StandardTokenizerImpl.h:30
static void ZZ_ATTRIBUTE_INIT()
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
int32_t zzStartRead
StartRead marks the beginning of the yytext() string in the buffer.
Definition: StandardTokenizerImpl.h:107
int32_t getNextToken()
Resumes scanning until the next regular expression is matched, the end of input is encountered or an ...
static const int32_t YYEOF
This character denotes the end of file.
Definition: StandardTokenizerImpl.h:129
static IntArray _ZZ_TRANS
The transition table of the DFA.
Definition: StandardTokenizerImpl.h:59
virtual ~StandardTokenizerImpl()
bool zzRefill()
Refills the input buffer.
static const int32_t ZZ_ACTION_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:42
static const int32_t ZZ_BUFFERSIZE
Initial size of the lookahead buffer.
Definition: StandardTokenizerImpl.h:22
String yytext()
Returns the text matched by the current regular expression.
static const int32_t ZZ_PUSHBACK_2BIG
Definition: StandardTokenizerImpl.h:71
static const int32_t ZZ_TRANS_PACKED_LENGTH
Definition: StandardTokenizerImpl.h:62
static IntArray _ZZ_ATTRIBUTE
ZZ_ATTRIBUTE[aState] contains the attributes of state aState.
Definition: StandardTokenizerImpl.h:76
clucene.sourceforge.net