Class implements base tokenizer functionality. Can be used as base class only.
You can find more information in comments below. Overrided methods can be found in corresponding base class or interface.
class CTokenizer :public virtual ITokenizer, public CReleasable
{
protected:
IReader *m_pReader; // Pointer to source reader or nullptr
IParser *m_pParserOperator; // Pointer to operator parser or nullptr
IParser *m_pParserString; // Pointer to string parser or nullptr
IParser *m_pParserBinary; // Pointer to binary number parser or nullptr
IParser *m_pParserOctal; // Pointer to octal number parser or nullptr
IParser *m_pParserHexadecimal; // Pointer to hexadecimal number parser or nullptr
IParser *m_pParserNumeric; // Pointer to numerical value parser or nullptr
IParser *m_pParserIdentifier; // Pointer to identifier parser or nullptr
CTokenizer(); // Constructor
public:
// ITokenizer methods
bool Next(size_t &index, Token &token) override;
bool Next(size_t &index, Tokens &tokens) override;
};| Namespace: | nitisa::script |
| Include: | Nitisa/Modules/Script/Core/Tokenizer.h |