Enumeration defining the token type
type TokenType = enumeration(RealToken, IntegerToken, BooleanToken, StringToken, IdentifierToken, DelimiterToken, NoToken);