Class implements base tokenizer functionality. Can be used as base class only.
You can find more information in comments below. Overrided methods can be found in corresponding base class or interface.
class CTokenizer :public virtual ITokenizer, public CReleasable
{
protected:
IReader *m_pReader; // Pointer to source reader or nullptr
IParser *m_pParserOperator; // Pointer to operator parser or nullptr
IParser *m_pParserString; // Pointer to string parser or nullptr
IParser *m_pParserBinary; // Pointer to binary number parser or nullptr
IParser *m_pParserOctal; // Pointer to octal number parser or nullptr
IParser *m_pParserHexadecimal; // Pointer to hexadecimal number parser or nullptr
IParser *m_pParserNumeric; // Pointer to numerical value parser or nullptr
IParser *m_pParserIdentifier; // Pointer to identifier parser or nullptr
Groups m_aSkipGroups;
CTokenizer(const Groups &skip_groups = {}); // Constructor
public:
void setSkipGroups(const Groups &skip_groups);
// ITokenizer methods
bool Next(size_t &index, Token &token) override;
bool Next(size_t &index, Tokens &tokens) override;
};| Namespace: | nitisa::script |
| Include: | Nitisa/Modules/Script/Core/Tokenizer.h |