Class implements generic (simplified C++ language) tokenizer.
You can find more information in comments below. Overrided methods can be found in corresponding base class or interface.
class CTokenizer :public script::CTokenizer
{
public:
static const Operators Ops; // Operators
// ITokenizer methods
bool Next(size_t &index, Token &token) override; // Changes only parsing order: attempts to parse numerical value first if it starts from "." (like, ".1f")
bool Next(size_t &index, Tokens &tokens) override;
CTokenizer(const String &source);
CTokenizer(const String &source, const size_t min, const size_t max);
};| Namespace: | nitisa::script::generic |
| Include: | Nitisa/Modules/Script/Tokenizers/Generic/Tokenizer.h |